From 6d4097bfb7286aeffbe218099ffbe67cfe82eac1 Mon Sep 17 00:00:00 2001 From: cool-developer <51834436+cool-develope@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:34:36 -0400 Subject: [PATCH] feat(store/v2): implement the feature to upgrade the store keys (#20453) Co-authored-by: Matt Kocubinski --- core/store/upgrade.go | 27 +-- runtime/v2/go.mod | 4 +- runtime/v2/go.sum | 4 +- server/v2/cometbft/go.mod | 4 +- server/v2/cometbft/go.sum | 4 +- server/v2/go.mod | 6 +- server/v2/go.sum | 8 +- simapp/v2/go.mod | 4 +- simapp/v2/go.sum | 4 +- store/v2/README.md | 52 ++++- store/v2/commitment/iavl/tree.go | 5 +- store/v2/commitment/iavl/tree_test.go | 31 ++- store/v2/commitment/mem/tree.go | 4 +- store/v2/commitment/metadata.go | 81 +++++++- store/v2/commitment/store.go | 126 ++++++++++-- store/v2/commitment/store_bench_test.go | 2 +- store/v2/commitment/store_test_suite.go | 232 +++++++++++++++++++++- store/v2/commitment/tree.go | 2 +- store/v2/database.go | 10 +- store/v2/go.mod | 10 +- store/v2/go.sum | 16 +- store/v2/internal/encoding/prefix.go | 16 ++ store/v2/internal/encoding/prefix_test.go | 27 +++ store/v2/migration/manager_test.go | 4 +- store/v2/proof/commit_info.go | 14 +- store/v2/pruning/manager.go | 4 +- store/v2/pruning/manager_test.go | 2 +- store/v2/root/factory.go | 44 +++- store/v2/root/migrate_test.go | 4 +- store/v2/root/store.go | 62 +++++- store/v2/root/store_test.go | 14 +- store/v2/root/upgrade_test.go | 157 +++++++++++++++ store/v2/storage/pebbledb/comparator.go | 2 +- store/v2/storage/pebbledb/db.go | 124 +++++++++++- store/v2/storage/rocksdb/db.go | 13 +- store/v2/storage/rocksdb/db_test.go | 1 + store/v2/storage/rocksdb/iterator.go | 4 + store/v2/storage/sqlite/db.go | 73 ++++++- store/v2/storage/storage_test_suite.go | 189 ++++++++++++++++-- store/v2/storage/store.go | 13 ++ store/v2/store.go | 9 +- 41 files changed, 1219 insertions(+), 193 deletions(-) create mode 100644 store/v2/internal/encoding/prefix.go create mode 100644 store/v2/internal/encoding/prefix_test.go create mode 100644 store/v2/root/upgrade_test.go diff --git a/core/store/upgrade.go b/core/store/upgrade.go index 5a943a949235..dbbf599973b1 100644 --- a/core/store/upgrade.go +++ b/core/store/upgrade.go @@ -2,17 +2,8 @@ package store // StoreUpgrades defines a series of transformations to apply the multistore db upon load type StoreUpgrades struct { - Added []string `json:"added"` - Renamed []StoreRename `json:"renamed"` - Deleted []string `json:"deleted"` -} - -// StoreRename defines a name change of a sub-store. -// All data previously under a PrefixStore with OldKey will be copied -// to a PrefixStore with NewKey, then deleted from OldKey store. -type StoreRename struct { - OldKey string `json:"old_key"` - NewKey string `json:"new_key"` + Added []string `json:"added"` + Deleted []string `json:"deleted"` } // IsAdded returns true if the given key should be added @@ -40,17 +31,3 @@ func (s *StoreUpgrades) IsDeleted(key string) bool { } return false } - -// RenamedFrom returns the oldKey if it was renamed -// Returns "" if it was not renamed -func (s *StoreUpgrades) RenamedFrom(key string) string { - if s == nil { - return "" - } - for _, re := range s.Renamed { - if re.NewKey == key { - return re.OldKey - } - } - return "" -} diff --git a/runtime/v2/go.mod b/runtime/v2/go.mod index 931884323c37..dab569cde4e7 100644 --- a/runtime/v2/go.mod +++ b/runtime/v2/go.mod @@ -15,7 +15,7 @@ replace ( require ( cosmossdk.io/api v0.7.5 - cosmossdk.io/core v0.12.1-0.20231114100755-569e3ff6a0d7 + cosmossdk.io/core v0.12.1-0.20240725072823-6a2d039e1212 cosmossdk.io/depinject v1.0.0 cosmossdk.io/log v1.3.1 cosmossdk.io/server/v2/appmanager v0.0.0-00010101000000-000000000000 @@ -44,7 +44,7 @@ require ( github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/cosmos/cosmos-db v1.0.2 // indirect github.com/cosmos/cosmos-proto v1.0.0-beta.5 // indirect - github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 // indirect + github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e // indirect github.com/cosmos/ics23/go v0.10.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/dot v1.6.2 // indirect diff --git a/runtime/v2/go.sum b/runtime/v2/go.sum index 9221f19acb5e..e12620290a0d 100644 --- a/runtime/v2/go.sum +++ b/runtime/v2/go.sum @@ -46,8 +46,8 @@ github.com/cosmos/cosmos-proto v1.0.0-beta.5 h1:eNcayDLpip+zVLRLYafhzLvQlSmyab+R github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRApmI9hjHH05nefC0Ojec= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 h1:wmwDn7V3RodN9auB3FooSQxs46nHVE3u0mb87TJkZFE= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e h1:5bxw1E0peLMrr8ZO9mYT0d9sxy0WgR1ZEWb92yjKnnk= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index 257f70c116c2..7968bfd4175b 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -22,7 +22,7 @@ replace ( require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 cosmossdk.io/api v0.7.5 - cosmossdk.io/core v0.12.1-0.20231114100755-569e3ff6a0d7 + cosmossdk.io/core v0.12.1-0.20240725072823-6a2d039e1212 cosmossdk.io/errors v1.0.1 cosmossdk.io/log v1.3.1 cosmossdk.io/server/v2 v2.0.0-00010101000000-000000000000 @@ -74,7 +74,7 @@ require ( github.com/cosmos/cosmos-db v1.0.2 // indirect github.com/cosmos/crypto v0.1.2 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect - github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 // indirect + github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e // indirect github.com/cosmos/ics23/go v0.10.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect github.com/danieljoos/wincred v1.2.1 // indirect diff --git a/server/v2/cometbft/go.sum b/server/v2/cometbft/go.sum index 133cea6f52bf..1a3add076dda 100644 --- a/server/v2/cometbft/go.sum +++ b/server/v2/cometbft/go.sum @@ -107,8 +107,8 @@ github.com/cosmos/gogogateway v1.2.0 h1:Ae/OivNhp8DqBi/sh2A8a1D0y638GpL3tkmLQAiK github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ4GUkT+tbFI= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 h1:wmwDn7V3RodN9auB3FooSQxs46nHVE3u0mb87TJkZFE= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e h1:5bxw1E0peLMrr8ZO9mYT0d9sxy0WgR1ZEWb92yjKnnk= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/cosmos/ledger-cosmos-go v0.13.3 h1:7ehuBGuyIytsXbd4MP43mLeoN2LTOEnk5nvue4rK+yM= diff --git a/server/v2/go.mod b/server/v2/go.mod index a5b65a1ed4f1..7c577fca5d8b 100644 --- a/server/v2/go.mod +++ b/server/v2/go.mod @@ -15,7 +15,7 @@ replace ( require ( cosmossdk.io/api v0.7.5 - cosmossdk.io/core v0.12.1-0.20231114100755-569e3ff6a0d7 + cosmossdk.io/core v0.12.1-0.20240725072823-6a2d039e1212 cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 cosmossdk.io/log v1.3.1 cosmossdk.io/server/v2/appmanager v0.0.0-00010101000000-000000000000 @@ -56,10 +56,10 @@ require ( github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/cosmos/cosmos-db v1.0.2 // indirect - github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 // indirect + github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e // indirect github.com/cosmos/ics23/go v0.10.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/dot v1.6.1 // indirect + github.com/emicklei/dot v1.6.2 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect diff --git a/server/v2/go.sum b/server/v2/go.sum index 7431c6950e32..1a2c148bff40 100644 --- a/server/v2/go.sum +++ b/server/v2/go.sum @@ -59,8 +59,8 @@ github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 h1:wmwDn7V3RodN9auB3FooSQxs46nHVE3u0mb87TJkZFE= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e h1:5bxw1E0peLMrr8ZO9mYT0d9sxy0WgR1ZEWb92yjKnnk= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -69,8 +69,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/dot v1.6.1 h1:ujpDlBkkwgWUY+qPId5IwapRW/xEoligRSYjioR6DFI= -github.com/emicklei/dot v1.6.1/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= +github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index a310e740891a..5311aa87d950 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -5,7 +5,7 @@ go 1.22.2 require ( cosmossdk.io/api v0.7.5 cosmossdk.io/client/v2 v2.0.0-00010101000000-000000000000 - cosmossdk.io/core v0.12.1-0.20231114100755-569e3ff6a0d7 + cosmossdk.io/core v0.12.1-0.20240725072823-6a2d039e1212 cosmossdk.io/depinject v1.0.0 cosmossdk.io/log v1.3.1 cosmossdk.io/math v1.3.0 @@ -93,7 +93,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogogateway v1.2.0 // indirect github.com/cosmos/gogoproto v1.5.0 // indirect - github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 // indirect + github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e // indirect github.com/cosmos/ics23/go v0.10.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.3 // indirect github.com/creachadair/atomicfile v0.3.4 // indirect diff --git a/simapp/v2/go.sum b/simapp/v2/go.sum index 8a8821e689f7..84efb04e8b12 100644 --- a/simapp/v2/go.sum +++ b/simapp/v2/go.sum @@ -325,8 +325,8 @@ github.com/cosmos/gogogateway v1.2.0/go.mod h1:iQpLkGWxYcnCdz5iAdLcRBSw3h7NXeOkZ github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 h1:wmwDn7V3RodN9auB3FooSQxs46nHVE3u0mb87TJkZFE= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e h1:5bxw1E0peLMrr8ZO9mYT0d9sxy0WgR1ZEWb92yjKnnk= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/cosmos/keyring v1.2.0 h1:8C1lBP9xhImmIabyXW4c3vFjjLiBdGCmfLUfeZlV1Yo= diff --git a/store/v2/README.md b/store/v2/README.md index 70f8dfb9a6e5..213f55d05aeb 100644 --- a/store/v2/README.md +++ b/store/v2/README.md @@ -4,16 +4,6 @@ The `store` package contains the implementation of store/v2, which is the SDK's abstraction around managing historical and committed state. See [ADR-065](../docs/architecture/adr-065-store-v2.md) and [Store v2 Design](https://docs.google.com/document/d/1l6uXIjTPHOOWM5N4sUUmUfCZvePoa5SNfIEtmgvgQSU/edit#heading=h.nz8dqy6wa4g1) for a high-level overview of the design and rationale. -## Migration - - - -## Pruning - -The `root.Store` is NOT responsible for pruning. Rather, pruning is the responsibility -of the underlying SS and SC layers. This means pruning can be implementation specific, -such as being synchronous or asynchronous. - ## Usage The `store` package contains a `root.Store` type which is intended to act as an @@ -29,3 +19,45 @@ from the perspective of `root.Store`, there is no notion of multi or single tree rather these are implementation details of SS and SC. For SS, we utilize store keys to namespace raw key/value pairs. For SC, we utilize an abstraction, `commitment.CommitStore`, to map store keys to a commitment trees. + +## Upgrades + +The `LoadVersionAndUpgrade` API of the `root.store` allows for adding or removing +store keys. This is useful for upgrading the chain with new modules or removing +old ones. The `Rename` feature is not supported in store/v2. + +```mermaid +sequenceDiagram + participant S as Store + participant SS as StateStorage + participant SC as StateCommitment + alt SC is a UpgradeableStore + S->>SC: LoadVersionAndUpgrade + SC->>SC: Mount new store keys + SC->>SC: Prune removed store keys + end + SC->>S: LoadVersion Result + alt SS is a UpgradableDatabase + S->>SS: PruneStoreKeys + end +``` + +`Prune store keys` does not remove the data from the SC and SS instantly. It only +marks the store keys as pruned. The actual data removal is done by the pruning +process of the underlying SS and SC. + +## Migration + + + +## Pruning + +The `root.Store` is NOT responsible for pruning. Rather, pruning is the responsibility +of the underlying SS and SC layers. This means pruning can be implementation specific, +such as being synchronous or asynchronous. + + + +## Test Coverage + +The test coverage of the following logical components should be over 60%: \ No newline at end of file diff --git a/store/v2/commitment/iavl/tree.go b/store/v2/commitment/iavl/tree.go index 5de6a4c86842..5fbff6e32931 100644 --- a/store/v2/commitment/iavl/tree.go +++ b/store/v2/commitment/iavl/tree.go @@ -87,8 +87,9 @@ func (t *IavlTree) Get(version uint64, key []byte) ([]byte, error) { } // GetLatestVersion returns the latest version of the tree. -func (t *IavlTree) GetLatestVersion() uint64 { - return uint64(t.tree.Version()) +func (t *IavlTree) GetLatestVersion() (uint64, error) { + v, err := t.tree.GetLatestVersion() + return uint64(v), err } // SetInitialVersion sets the initial version of the database. diff --git a/store/v2/commitment/iavl/tree_test.go b/store/v2/commitment/iavl/tree_test.go index 7435b4e58dd2..4ea8f0f93ff1 100644 --- a/store/v2/commitment/iavl/tree_test.go +++ b/store/v2/commitment/iavl/tree_test.go @@ -16,14 +16,22 @@ import ( func TestCommitterSuite(t *testing.T) { s := &commitment.CommitStoreTestSuite{ - NewStore: func(db corestore.KVStoreWithBatch, storeKeys []string, logger corelog.Logger) (*commitment.CommitStore, error) { + NewStore: func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*commitment.CommitStore, error) { multiTrees := make(map[string]commitment.Tree) cfg := DefaultConfig() - for _, storeKey := range storeKeys { + mountTreeFn := func(storeKey string) (commitment.Tree, error) { prefixDB := dbm.NewPrefixDB(db, []byte(storeKey)) - multiTrees[storeKey] = NewIavlTree(prefixDB, logger, cfg) + return NewIavlTree(prefixDB, logger, cfg), nil + } + for _, storeKey := range storeKeys { + multiTrees[storeKey], _ = mountTreeFn(storeKey) } - return commitment.NewCommitStore(multiTrees, db, logger) + oldTrees := make(map[string]commitment.Tree) + for _, storeKey := range oldStoreKeys { + oldTrees[storeKey], _ = mountTreeFn(storeKey) + } + + return commitment.NewCommitStore(multiTrees, oldTrees, db, logger) }, } @@ -41,7 +49,8 @@ func TestIavlTree(t *testing.T) { tree := generateTree() require.NotNil(t, tree) - initVersion := tree.GetLatestVersion() + initVersion, err := tree.GetLatestVersion() + require.NoError(t, err) require.Equal(t, uint64(0), initVersion) // write a batch of version 1 @@ -51,14 +60,18 @@ func TestIavlTree(t *testing.T) { workingHash := tree.WorkingHash() require.NotNil(t, workingHash) - require.Equal(t, uint64(0), tree.GetLatestVersion()) + v, err := tree.GetLatestVersion() + require.NoError(t, err) + require.Equal(t, uint64(0), v) // commit the batch commitHash, version, err := tree.Commit() require.NoError(t, err) require.Equal(t, version, uint64(1)) require.Equal(t, workingHash, commitHash) - require.Equal(t, uint64(1), tree.GetLatestVersion()) + v, err = tree.GetLatestVersion() + require.NoError(t, err) + require.Equal(t, uint64(1), v) // ensure we can get expected values bz, err := tree.Get(1, []byte("key1")) @@ -100,7 +113,9 @@ func TestIavlTree(t *testing.T) { // prune version 1 err = tree.Prune(1) require.NoError(t, err) - require.Equal(t, uint64(3), tree.GetLatestVersion()) + v, err = tree.GetLatestVersion() + require.NoError(t, err) + require.Equal(t, uint64(3), v) // async pruning check checkErr := func() bool { if _, err := tree.tree.LoadVersion(1); err != nil { diff --git a/store/v2/commitment/mem/tree.go b/store/v2/commitment/mem/tree.go index 34f26e6a289d..7610ab3bf62d 100644 --- a/store/v2/commitment/mem/tree.go +++ b/store/v2/commitment/mem/tree.go @@ -18,8 +18,8 @@ func (t *Tree) Remove(key []byte) error { return t.MemDB.Delete(key) } -func (t *Tree) GetLatestVersion() uint64 { - return 0 +func (t *Tree) GetLatestVersion() (uint64, error) { + return 0, nil } func (t *Tree) Hash() []byte { diff --git a/store/v2/commitment/metadata.go b/store/v2/commitment/metadata.go index 6b98ef22577d..7462252e962b 100644 --- a/store/v2/commitment/metadata.go +++ b/store/v2/commitment/metadata.go @@ -10,20 +10,24 @@ import ( ) const ( - commitInfoKeyFmt = "c/%d" // c/ - latestVersionKey = "c/latest" + commitInfoKeyFmt = "c/%d" // c/ + latestVersionKey = "c/latest" + removedStoreKeyPrefix = "c/removed/" // c/removed// ) +// MetadataStore is a store for metadata related to the commitment store. type MetadataStore struct { kv corestore.KVStoreWithBatch } +// NewMetadataStore creates a new MetadataStore. func NewMetadataStore(kv corestore.KVStoreWithBatch) *MetadataStore { return &MetadataStore{ kv: kv, } } +// GetLatestVersion returns the latest committed version. func (m *MetadataStore) GetLatestVersion() (uint64, error) { value, err := m.kv.Get([]byte(latestVersionKey)) if err != nil { @@ -41,6 +45,16 @@ func (m *MetadataStore) GetLatestVersion() (uint64, error) { return version, nil } +func (m *MetadataStore) setLatestVersion(version uint64) error { + var buf bytes.Buffer + buf.Grow(encoding.EncodeUvarintSize(version)) + if err := encoding.EncodeUvarint(&buf, version); err != nil { + return err + } + return m.kv.Set([]byte(latestVersionKey), buf.Bytes()) +} + +// GetCommitInfo returns the commit info for the given version. func (m *MetadataStore) GetCommitInfo(version uint64) (*proof.CommitInfo, error) { key := []byte(fmt.Sprintf(commitInfoKeyFmt, version)) value, err := m.kv.Get(key) @@ -90,12 +104,73 @@ func (m *MetadataStore) flushCommitInfo(version uint64, cInfo *proof.CommitInfo) return err } - if err := batch.WriteSync(); err != nil { + if err := batch.Write(); err != nil { return err } return nil } +func (m *MetadataStore) flushRemovedStoreKeys(version uint64, storeKeys []string) (err error) { + batch := m.kv.NewBatch() + defer func() { + err = batch.Close() + }() + + for _, storeKey := range storeKeys { + key := []byte(fmt.Sprintf("%s%s", encoding.BuildPrefixWithVersion(removedStoreKeyPrefix, version), storeKey)) + if err := batch.Set(key, []byte{}); err != nil { + return err + } + } + return batch.Write() +} + +func (m *MetadataStore) GetRemovedStoreKeys(version uint64) (storeKeys [][]byte, err error) { + end := encoding.BuildPrefixWithVersion(removedStoreKeyPrefix, version+1) + iter, err := m.kv.Iterator([]byte(removedStoreKeyPrefix), end) + if err != nil { + return nil, err + } + defer func() { + if ierr := iter.Close(); ierr != nil { + err = ierr + } + }() + + for ; iter.Valid(); iter.Next() { + storeKey := iter.Key()[len(end):] + storeKeys = append(storeKeys, storeKey) + } + return storeKeys, nil +} + +func (m *MetadataStore) deleteRemovedStoreKeys(version uint64, removeStore func(storeKey []byte, version uint64) error) (err error) { + removedStoreKeys, err := m.GetRemovedStoreKeys(version) + if err != nil { + return err + } + if len(removedStoreKeys) == 0 { + return nil + } + + batch := m.kv.NewBatch() + defer func() { + if berr := batch.Close(); berr != nil { + err = berr + } + }() + for _, storeKey := range removedStoreKeys { + if err := removeStore(storeKey, version); err != nil { + return err + } + if err := batch.Delete(storeKey); err != nil { + return err + } + } + + return batch.Write() +} + func (m *MetadataStore) deleteCommitInfo(version uint64) error { cInfoKey := []byte(fmt.Sprintf(commitInfoKeyFmt, version)) return m.kv.Delete(cInfoKey) diff --git a/store/v2/commitment/store.go b/store/v2/commitment/store.go index c814dec70be1..68c83e1c3de1 100644 --- a/store/v2/commitment/store.go +++ b/store/v2/commitment/store.go @@ -5,8 +5,10 @@ import ( "fmt" "io" "math" + "sort" protoio "github.com/cosmos/gogoproto/io" + "golang.org/x/exp/maps" corelog "cosmossdk.io/core/log" corestore "cosmossdk.io/core/store" @@ -20,10 +22,15 @@ import ( var ( _ store.Committer = (*CommitStore)(nil) + _ store.UpgradeableStore = (*CommitStore)(nil) _ snapshots.CommitSnapshotter = (*CommitStore)(nil) _ store.PausablePruner = (*CommitStore)(nil) ) +// MountTreeFn is a function that mounts a tree given a store key. +// It is used to lazily mount trees when needed (e.g. during upgrade or proof generation). +type MountTreeFn func(storeKey string) (Tree, error) + // CommitStore is a wrapper around multiple Tree objects mapped by a unique store // key. Each store key reflects dedicated and unique usage within a module. A caller // can construct a CommitStore with one or more store keys. It is expected that a @@ -33,20 +40,23 @@ type CommitStore struct { logger corelog.Logger metadata *MetadataStore multiTrees map[string]Tree + // oldTrees is a map of store keys to old trees that have been deleted or renamed. + // It is used to get the proof for the old store keys. + oldTrees map[string]Tree } // NewCommitStore creates a new CommitStore instance. -func NewCommitStore(trees map[string]Tree, db corestore.KVStoreWithBatch, logger corelog.Logger) (*CommitStore, error) { +func NewCommitStore(trees, oldTrees map[string]Tree, db corestore.KVStoreWithBatch, logger corelog.Logger) (*CommitStore, error) { return &CommitStore{ logger: logger, multiTrees: trees, + oldTrees: oldTrees, metadata: NewMetadataStore(db), }, nil } func (c *CommitStore) WriteChangeset(cs *corestore.Changeset) error { for _, pairs := range cs.Changes { - key := conv.UnsafeBytesToStr(pairs.Actor) tree, ok := c.multiTrees[key] @@ -90,6 +100,62 @@ func (c *CommitStore) WorkingCommitInfo(version uint64) *proof.CommitInfo { } func (c *CommitStore) LoadVersion(targetVersion uint64) error { + storeKeys := make([]string, 0, len(c.multiTrees)) + for storeKey := range c.multiTrees { + storeKeys = append(storeKeys, storeKey) + } + return c.loadVersion(targetVersion, storeKeys) +} + +// LoadVersionAndUpgrade implements store.UpgradeableStore. +func (c *CommitStore) LoadVersionAndUpgrade(targetVersion uint64, upgrades *corestore.StoreUpgrades) error { + // deterministic iteration order for upgrades (as the underlying store may change and + // upgrades make store changes where the execution order may matter) + storeKeys := maps.Keys(c.multiTrees) + sort.Strings(storeKeys) + + removeTree := func(storeKey string) error { + if oldTree, ok := c.multiTrees[storeKey]; ok { + if err := oldTree.Close(); err != nil { + return err + } + delete(c.multiTrees, storeKey) + } + return nil + } + + newStoreKeys := make([]string, 0, len(c.multiTrees)) + removedStoreKeys := make([]string, 0) + for _, storeKey := range storeKeys { + // If it has been deleted, remove the tree. + if upgrades.IsDeleted(storeKey) { + if err := removeTree(storeKey); err != nil { + return err + } + removedStoreKeys = append(removedStoreKeys, storeKey) + continue + } + + // If it has been added, set the initial version. + if upgrades.IsAdded(storeKey) { + if err := c.multiTrees[storeKey].SetInitialVersion(targetVersion + 1); err != nil { + return err + } + // This is the empty tree, no need to load the version. + continue + } + + newStoreKeys = append(newStoreKeys, storeKey) + } + + if err := c.metadata.flushRemovedStoreKeys(targetVersion, removedStoreKeys); err != nil { + return err + } + + return c.loadVersion(targetVersion, newStoreKeys) +} + +func (c *CommitStore) loadVersion(targetVersion uint64, storeKeys []string) error { // Rollback the metadata to the target version. latestVersion, err := c.GetLatestVersion() if err != nil { @@ -101,22 +167,25 @@ func (c *CommitStore) LoadVersion(targetVersion uint64) error { return err } } + if err := c.metadata.setLatestVersion(targetVersion); err != nil { + return err + } } - for _, tree := range c.multiTrees { - if err := tree.LoadVersion(targetVersion); err != nil { + for _, storeKey := range storeKeys { + if err := c.multiTrees[storeKey].LoadVersion(targetVersion); err != nil { return err } } // If the target version is greater than the latest version, it is the snapshot // restore case, we should create a new commit info for the target version. - var cInfo *proof.CommitInfo if targetVersion > latestVersion { - cInfo = c.WorkingCommitInfo(targetVersion) + cInfo := c.WorkingCommitInfo(targetVersion) + return c.metadata.flushCommitInfo(targetVersion, cInfo) } - return c.metadata.flushCommitInfo(targetVersion, cInfo) + return nil } func (c *CommitStore) Commit(version uint64) (*proof.CommitInfo, error) { @@ -130,7 +199,11 @@ func (c *CommitStore) Commit(version uint64) (*proof.CommitInfo, error) { // will be larger than the RMS's metadata, when the block is replayed, we // should avoid committing that iavl store again. var commitID proof.CommitID - if tree.GetLatestVersion() >= version { + v, err := tree.GetLatestVersion() + if err != nil { + return nil, err + } + if v >= version { commitID.Version = version commitID.Hash = tree.Hash() } else { @@ -175,9 +248,13 @@ func (c *CommitStore) SetInitialVersion(version uint64) error { } func (c *CommitStore) GetProof(storeKey []byte, version uint64, key []byte) ([]proof.CommitmentOp, error) { - tree, ok := c.multiTrees[conv.UnsafeBytesToStr(storeKey)] + rawStoreKey := conv.UnsafeBytesToStr(storeKey) + tree, ok := c.multiTrees[rawStoreKey] if !ok { - return nil, fmt.Errorf("store %s not found", storeKey) + tree, ok = c.oldTrees[rawStoreKey] + if !ok { + return nil, fmt.Errorf("store %s not found", rawStoreKey) + } } iProof, err := tree.GetProof(version, key) @@ -215,21 +292,36 @@ func (c *CommitStore) Get(storeKey []byte, version uint64, key []byte) ([]byte, } // Prune implements store.Pruner. -func (c *CommitStore) Prune(version uint64) (ferr error) { +func (c *CommitStore) Prune(version uint64) error { // prune the metadata for v := version; v > 0; v-- { if err := c.metadata.deleteCommitInfo(v); err != nil { return err } } - + // prune the trees for _, tree := range c.multiTrees { if err := tree.Prune(version); err != nil { - ferr = errors.Join(ferr, err) + return err } } + // prune the removed store keys + if err := c.pruneRemovedStoreKeys(version); err != nil { + return err + } + + return nil +} - return ferr +func (c *CommitStore) pruneRemovedStoreKeys(version uint64) error { + clearKVStore := func(storeKey []byte, version uint64) (err error) { + tree, ok := c.oldTrees[string(storeKey)] + if !ok { + return fmt.Errorf("store %s not found in oldTrees", storeKey) + } + return tree.Prune(version) + } + return c.metadata.deleteRemovedStoreKeys(version, clearKVStore) } // PausePruning implements store.PausablePruner. @@ -402,12 +494,12 @@ func (c *CommitStore) GetLatestVersion() (uint64, error) { return c.metadata.GetLatestVersion() } -func (c *CommitStore) Close() (ferr error) { +func (c *CommitStore) Close() error { for _, tree := range c.multiTrees { if err := tree.Close(); err != nil { - ferr = errors.Join(ferr, err) + return err } } - return ferr + return nil } diff --git a/store/v2/commitment/store_bench_test.go b/store/v2/commitment/store_bench_test.go index 037d211ee719..49513e27bda2 100644 --- a/store/v2/commitment/store_bench_test.go +++ b/store/v2/commitment/store_bench_test.go @@ -66,7 +66,7 @@ func getCommitStore(b *testing.B, db corestore.KVStoreWithBatch) *commitment.Com multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, coretesting.NewNopLogger(), iavl.DefaultConfig()) } - sc, err := commitment.NewCommitStore(multiTrees, db, coretesting.NewNopLogger()) + sc, err := commitment.NewCommitStore(multiTrees, nil, db, coretesting.NewNopLogger()) require.NoError(b, err) return sc diff --git a/store/v2/commitment/store_test_suite.go b/store/v2/commitment/store_test_suite.go index 7e8c3587d34e..edbce27116c9 100644 --- a/store/v2/commitment/store_test_suite.go +++ b/store/v2/commitment/store_test_suite.go @@ -20,18 +20,19 @@ import ( const ( storeKey1 = "store1" storeKey2 = "store2" + storeKey3 = "store3" ) // CommitStoreTestSuite is a test suite to be used for all tree backends. type CommitStoreTestSuite struct { suite.Suite - NewStore func(db corestore.KVStoreWithBatch, storeKeys []string, logger corelog.Logger) (*CommitStore, error) + NewStore func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error) } func (s *CommitStoreTestSuite) TestStore_Snapshotter() { storeKeys := []string{storeKey1, storeKey2} - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(10) @@ -65,7 +66,7 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() { }, } - targetStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, coretesting.NewNopLogger()) + targetStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) chunks := make(chan io.ReadCloser, kvCount*int(latestVersion)) @@ -124,10 +125,65 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() { } } +func (s *CommitStoreTestSuite) TestStore_LoadVersion() { + storeKeys := []string{storeKey1, storeKey2} + mdb := dbm.NewMemDB() + commitStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + s.Require().NoError(err) + + latestVersion := uint64(10) + kvCount := 10 + for i := uint64(1); i <= latestVersion; i++ { + kvPairs := make(map[string]corestore.KVPairs) + for _, storeKey := range storeKeys { + kvPairs[storeKey] = corestore.KVPairs{} + for j := 0; j < kvCount; j++ { + key := []byte(fmt.Sprintf("key-%d-%d", i, j)) + value := []byte(fmt.Sprintf("value-%d-%d", i, j)) + kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value}) + } + } + s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(kvPairs))) + _, err = commitStore.Commit(i) + s.Require().NoError(err) + } + + // load the store with the latest version + targetStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + s.Require().NoError(err) + err = targetStore.LoadVersion(latestVersion) + s.Require().NoError(err) + // check the store + for i := uint64(1); i <= latestVersion; i++ { + commitInfo, _ := targetStore.GetCommitInfo(i) + s.Require().NotNil(commitInfo) + s.Require().Equal(i, commitInfo.Version) + } + + // rollback to a previous version + rollbackVersion := uint64(5) + rollbackStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + s.Require().NoError(err) + err = rollbackStore.LoadVersion(rollbackVersion) + s.Require().NoError(err) + // check the store + v, err := rollbackStore.GetLatestVersion() + s.Require().NoError(err) + s.Require().Equal(rollbackVersion, v) + for i := uint64(1); i <= latestVersion; i++ { + commitInfo, _ := rollbackStore.GetCommitInfo(i) + if i > rollbackVersion { + s.Require().Nil(commitInfo) + } else { + s.Require().NotNil(commitInfo) + } + } +} + func (s *CommitStoreTestSuite) TestStore_Pruning() { storeKeys := []string{storeKey1, storeKey2} pruneOpts := store.NewPruningOptionWithCustom(10, 5) - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(100) @@ -164,3 +220,171 @@ func (s *CommitStoreTestSuite) TestStore_Pruning() { } } } + +func (s *CommitStoreTestSuite) TestStore_Upgrades() { + storeKeys := []string{storeKey1, storeKey2, storeKey3} + commitDB := dbm.NewMemDB() + commitStore, err := s.NewStore(commitDB, storeKeys, nil, coretesting.NewNopLogger()) + s.Require().NoError(err) + + latestVersion := uint64(10) + kvCount := 10 + for i := uint64(1); i <= latestVersion; i++ { + kvPairs := make(map[string]corestore.KVPairs) + for _, storeKey := range storeKeys { + kvPairs[storeKey] = corestore.KVPairs{} + for j := 0; j < kvCount; j++ { + key := []byte(fmt.Sprintf("key-%d-%d", i, j)) + value := []byte(fmt.Sprintf("value-%d-%d", i, j)) + kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value}) + } + } + s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(kvPairs))) + _, err = commitStore.Commit(i) + s.Require().NoError(err) + } + + // create a new commitment store with upgrades + upgrades := &corestore.StoreUpgrades{ + Added: []string{"newStore1", "newStore2"}, + Deleted: []string{storeKey3}, + } + newStoreKeys := []string{storeKey1, storeKey2, storeKey3, "newStore1", "newStore2"} + realStoreKeys := []string{storeKey1, storeKey2, "newStore1", "newStore2"} + oldStoreKeys := []string{storeKey1, storeKey3} + commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) + s.Require().NoError(err) + err = commitStore.LoadVersionAndUpgrade(latestVersion, upgrades) + s.Require().NoError(err) + + // GetProof should work for the old stores + for _, storeKey := range []string{storeKey1, storeKey3} { + for i := uint64(1); i <= latestVersion; i++ { + for j := 0; j < kvCount; j++ { + proof, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } + } + // GetProof should fail for the new stores against the old versions + for _, storeKey := range []string{"newStore1", "newStore2"} { + for i := uint64(1); i <= latestVersion; i++ { + for j := 0; j < kvCount; j++ { + _, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().Error(err) + } + } + } + + // apply the changeset again + for i := latestVersion + 1; i < latestVersion*2; i++ { + kvPairs := make(map[string]corestore.KVPairs) + for _, storeKey := range realStoreKeys { + kvPairs[storeKey] = corestore.KVPairs{} + for j := 0; j < kvCount; j++ { + key := []byte(fmt.Sprintf("key-%d-%d", i, j)) + value := []byte(fmt.Sprintf("value-%d-%d", i, j)) + kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value}) + } + } + s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(kvPairs))) + commitInfo, err := commitStore.Commit(i) + s.Require().NoError(err) + s.Require().NotNil(commitInfo) + s.Require().Equal(len(realStoreKeys), len(commitInfo.StoreInfos)) + for _, storeKey := range realStoreKeys { + s.Require().NotNil(commitInfo.GetStoreCommitID([]byte(storeKey))) + } + } + + // verify new stores + for _, storeKey := range []string{"newStore1", "newStore2"} { + for i := latestVersion + 1; i < latestVersion*2; i++ { + for j := 0; j < kvCount; j++ { + proof, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } + } + + // verify existing store + for i := uint64(1); i < latestVersion*2; i++ { + for j := 0; j < kvCount; j++ { + proof, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } + + // create a new commitment store with one more upgrades + upgrades = &corestore.StoreUpgrades{ + Added: []string{storeKey3}, + Deleted: []string{storeKey2}, + } + newRealStoreKeys := []string{storeKey1, storeKey3, "newStore1", "newStore2"} + oldStoreKeys = []string{storeKey2, storeKey3} + commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) + s.Require().NoError(err) + err = commitStore.LoadVersionAndUpgrade(2*latestVersion-1, upgrades) + s.Require().NoError(err) + + // apply the changeset again + for i := latestVersion * 2; i < latestVersion*3; i++ { + kvPairs := make(map[string]corestore.KVPairs) + for _, storeKey := range newRealStoreKeys { + kvPairs[storeKey] = corestore.KVPairs{} + for j := 0; j < kvCount; j++ { + key := []byte(fmt.Sprintf("key-%d-%d", i, j)) + value := []byte(fmt.Sprintf("value-%d-%d", i, j)) + kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value}) + } + } + s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(kvPairs))) + commitInfo, err := commitStore.Commit(i) + s.Require().NoError(err) + s.Require().NotNil(commitInfo) + s.Require().Equal(len(newRealStoreKeys), len(commitInfo.StoreInfos)) + for _, storeKey := range newRealStoreKeys { + s.Require().NotNil(commitInfo.GetStoreCommitID([]byte(storeKey))) + } + } + + // prune the old stores + s.Require().NoError(commitStore.Prune(latestVersion)) + // GetProof should fail for the old stores + for _, storeKey := range []string{storeKey1, storeKey3} { + for i := uint64(1); i <= latestVersion; i++ { + for j := 0; j < kvCount; j++ { + _, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().Error(err) + } + } + } + // GetProof should not fail for the newly removed store + for i := latestVersion + 1; i < latestVersion*2; i++ { + for j := 0; j < kvCount; j++ { + proof, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } + + s.Require().NoError(commitStore.Prune(latestVersion * 2)) + // GetProof should fail for the newly deleted stores + for i := uint64(1); i < latestVersion*2; i++ { + for j := 0; j < kvCount; j++ { + _, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().Error(err) + } + } + // GetProof should work for the new added store + for i := latestVersion*2 + 1; i < latestVersion*3; i++ { + for j := 0; j < kvCount; j++ { + proof, err := commitStore.GetProof([]byte(storeKey3), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } +} diff --git a/store/v2/commitment/tree.go b/store/v2/commitment/tree.go index 54fe2d60f00e..19b76b34c937 100644 --- a/store/v2/commitment/tree.go +++ b/store/v2/commitment/tree.go @@ -16,7 +16,7 @@ var ErrorExportDone = errors.New("export is complete") type Tree interface { Set(key, value []byte) error Remove(key []byte) error - GetLatestVersion() uint64 + GetLatestVersion() (uint64, error) // Hash returns the hash of the latest saved version of the tree. Hash() []byte diff --git a/store/v2/database.go b/store/v2/database.go index a0466de18dea..58a3ee7ef65c 100644 --- a/store/v2/database.go +++ b/store/v2/database.go @@ -25,6 +25,14 @@ type VersionedDatabase interface { io.Closer } +// UpgradableDatabase defines an API for a versioned database that allows pruning +// deleted storeKeys +type UpgradableDatabase interface { + // PruneStoreKeys prunes all data associated with the given storeKeys whenever + // the given version is pruned. + PruneStoreKeys(storeKeys []string, version uint64) error +} + // Committer defines an API for committing state. type Committer interface { // WriteChangeset writes the changeset to the commitment state. @@ -51,7 +59,7 @@ type Committer interface { // Once migration is complete, this method should be removed and/or not used. Get(storeKey []byte, version uint64, key []byte) ([]byte, error) - // SetInitialVersion sets the initial version of the tree. + // SetInitialVersion sets the initial version of the committer. SetInitialVersion(version uint64) error // GetCommitInfo returns the CommitInfo for the given version. diff --git a/store/v2/go.mod b/store/v2/go.mod index a305cc49e8d0..fc5870aa7836 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -3,14 +3,14 @@ module cosmossdk.io/store/v2 go 1.21 require ( - cosmossdk.io/core v0.12.0 + cosmossdk.io/core v0.12.1-0.20240725072823-6a2d039e1212 cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 cosmossdk.io/log v1.3.1 github.com/cockroachdb/pebble v1.1.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/gogoproto v1.5.0 - github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 + github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e github.com/cosmos/ics23/go v0.10.0 github.com/google/btree v1.1.2 github.com/hashicorp/go-metrics v0.5.3 @@ -19,6 +19,7 @@ require ( github.com/spf13/cast v1.6.0 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.8.0 ) @@ -30,9 +31,9 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/cosmos/cosmos-db v1.0.2 // indirect + github.com/cosmos/cosmos-db v1.0.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/dot v1.6.1 // indirect + github.com/emicklei/dot v1.6.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -58,7 +59,6 @@ require ( github.com/rs/zerolog v1.33.0 // indirect github.com/tidwall/btree v1.7.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/store/v2/go.sum b/store/v2/go.sum index 7e13cab09725..296393501fc9 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -34,14 +34,14 @@ github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cosmos/cosmos-db v1.0.2 h1:hwMjozuY1OlJs/uh6vddqnk9j7VamLv+0DBlbEXbAKs= -github.com/cosmos/cosmos-db v1.0.2/go.mod h1:Z8IXcFJ9PqKK6BIsVOB3QXtkKoqUOp1vRvPT39kOXEA= +github.com/cosmos/cosmos-db v1.0.0 h1:EVcQZ+qYag7W6uorBKFPvX6gRjw6Uq2hIh4hCWjuQ0E= +github.com/cosmos/cosmos-db v1.0.0/go.mod h1:iBvi1TtqaedwLdcrZVYRSSCb6eSy61NLj4UNmdIgs0U= github.com/cosmos/cosmos-proto v1.0.0-beta.5 h1:eNcayDLpip+zVLRLYafhzLvQlSmyab+RC5W7ZfmxJLA= github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRApmI9hjHH05nefC0Ojec= github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179 h1:wmwDn7V3RodN9auB3FooSQxs46nHVE3u0mb87TJkZFE= -github.com/cosmos/iavl v1.2.1-0.20240725141113-7adc688cf179/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e h1:5bxw1E0peLMrr8ZO9mYT0d9sxy0WgR1ZEWb92yjKnnk= +github.com/cosmos/iavl v1.2.1-0.20240731145221-594b181f427e/go.mod h1:GiM43q0pB+uG53mLxLDzimxM9l/5N9UuSY3/D0huuVw= github.com/cosmos/ics23/go v0.10.0 h1:iXqLLgp2Lp+EdpIuwXTYIQU+AiHj9mOC2X9ab++bZDM= github.com/cosmos/ics23/go v0.10.0/go.mod h1:ZfJSmng/TBNTBkFemHHHj5YY7VAU/MBU980F4VU1NG0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -49,8 +49,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/dot v1.6.1 h1:ujpDlBkkwgWUY+qPId5IwapRW/xEoligRSYjioR6DFI= -github.com/emicklei/dot v1.6.1/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= +github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -226,8 +226,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f h1:3CW0unweImhOzd5FmYuRsD4Y4oQFKZIjAnKbjV4WIrw= -golang.org/x/exp v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/store/v2/internal/encoding/prefix.go b/store/v2/internal/encoding/prefix.go new file mode 100644 index 000000000000..f93d7bc5996f --- /dev/null +++ b/store/v2/internal/encoding/prefix.go @@ -0,0 +1,16 @@ +package encoding + +import "encoding/binary" + +const separator = '/' + +// BuildPrefixWithVersion returns a byte slice with the given prefix and BigEndian encoded version. +// It is mainly used to represent the removed store key at the metadata store. +func BuildPrefixWithVersion(prefix string, version uint64) []byte { + n := len(prefix) + buf := make([]byte, n+8+1) + copy(buf, prefix) + binary.BigEndian.PutUint64(buf[n:], version) + buf[n+8] = separator + return buf +} diff --git a/store/v2/internal/encoding/prefix_test.go b/store/v2/internal/encoding/prefix_test.go new file mode 100644 index 000000000000..bcdd74dd781e --- /dev/null +++ b/store/v2/internal/encoding/prefix_test.go @@ -0,0 +1,27 @@ +package encoding + +import ( + "bytes" + "testing" +) + +func TestBuildPrefixWithVersion(t *testing.T) { + testcases := []struct { + prefix string + version uint64 + want []byte + }{ + {"", 0, []byte{0, 0, 0, 0, 0, 0, 0, 0, '/'}}, + {"a", 0, []byte{'a', 0, 0, 0, 0, 0, 0, 0, 0, '/'}}, + {"b", 1, []byte{'b', 0, 0, 0, 0, 0, 0, 0, 1, '/'}}, + {"s/k/removed/", 2, []byte{'s', '/', 'k', '/', 'r', 'e', 'm', 'o', 'v', 'e', 'd', '/', 0, 0, 0, 0, 0, 0, 0, 2, '/'}}, + {"s/k/", 1234567890, []byte{'s', '/', 'k', '/', 0, 0, 0, 0, 73, 150, 2, 210, '/'}}, + } + + for _, tc := range testcases { + got := BuildPrefixWithVersion(tc.prefix, tc.version) + if !bytes.Equal(got, tc.want) { + t.Fatalf("BuildPrefixWithVersion(%q, %d) = %v, want %v", tc.prefix, tc.version, got, tc.want) + } + } +} diff --git a/store/v2/migration/manager_test.go b/store/v2/migration/manager_test.go index d8365ce07579..ae00d5c08bbb 100644 --- a/store/v2/migration/manager_test.go +++ b/store/v2/migration/manager_test.go @@ -28,7 +28,7 @@ func setupMigrationManager(t *testing.T, noCommitStore bool) (*Manager, *commitm multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, coretesting.NewNopLogger(), iavl.DefaultConfig()) } - commitStore, err := commitment.NewCommitStore(multiTrees, db, coretesting.NewNopLogger()) + commitStore, err := commitment.NewCommitStore(multiTrees, nil, db, coretesting.NewNopLogger()) require.NoError(t, err) snapshotsStore, err := snapshots.NewStore(t.TempDir()) @@ -47,7 +47,7 @@ func setupMigrationManager(t *testing.T, noCommitStore bool) (*Manager, *commitm multiTrees1[storeKey] = iavl.NewIavlTree(prefixDB, coretesting.NewNopLogger(), iavl.DefaultConfig()) } - newCommitStore, err := commitment.NewCommitStore(multiTrees1, db1, coretesting.NewNopLogger()) // for store/v2 + newCommitStore, err := commitment.NewCommitStore(multiTrees1, nil, db1, coretesting.NewNopLogger()) // for store/v2 require.NoError(t, err) if noCommitStore { newCommitStore = nil diff --git a/store/v2/proof/commit_info.go b/store/v2/proof/commit_info.go index d95c152b4665..b784676a8d34 100644 --- a/store/v2/proof/commit_info.go +++ b/store/v2/proof/commit_info.go @@ -72,7 +72,8 @@ func (ci *CommitInfo) GetStoreProof(storeKey []byte) ([]byte, *CommitmentOp, err return bytes.Compare(ci.StoreInfos[i].Name, ci.StoreInfos[j].Name) < 0 }) - index := 0 + isEmpty := len(storeKey) == 0 + index := -1 leaves := make([][]byte, len(ci.StoreInfos)) for i, si := range ci.StoreInfos { var err error @@ -80,14 +81,21 @@ func (ci *CommitInfo) GetStoreProof(storeKey []byte) ([]byte, *CommitmentOp, err if err != nil { return nil, nil, err } - if bytes.Equal(si.Name, storeKey) { + if !isEmpty && bytes.Equal(si.Name, storeKey) { index = i } } + if index == -1 { + if isEmpty { + index = 0 + } else { + return nil, nil, fmt.Errorf("store key %s not found", storeKey) + } + } + rootHash, inners := ProofFromByteSlices(leaves, index) commitmentOp := ConvertCommitmentOp(inners, storeKey, ci.StoreInfos[index].GetHash()) - return rootHash, &commitmentOp, nil } diff --git a/store/v2/pruning/manager.go b/store/v2/pruning/manager.go index 424805bde70b..1e36abe7a6ed 100644 --- a/store/v2/pruning/manager.go +++ b/store/v2/pruning/manager.go @@ -1,6 +1,8 @@ package pruning -import "cosmossdk.io/store/v2" +import ( + "cosmossdk.io/store/v2" +) // Manager is a struct that manages the pruning of old versions of the SC and SS. type Manager struct { diff --git a/store/v2/pruning/manager_test.go b/store/v2/pruning/manager_test.go index 607af22bc722..9f96b2a48360 100644 --- a/store/v2/pruning/manager_test.go +++ b/store/v2/pruning/manager_test.go @@ -42,7 +42,7 @@ func (s *PruningManagerTestSuite) SetupTest() { prefixDB := dbm.NewPrefixDB(mdb, []byte(storeKey)) multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, nopLog, iavl.DefaultConfig()) } - s.sc, err = commitment.NewCommitStore(multiTrees, mdb, nopLog) + s.sc, err = commitment.NewCommitStore(multiTrees, nil, mdb, nopLog) s.Require().NoError(err) sqliteDB, err := sqlite.New(s.T().TempDir()) diff --git a/store/v2/root/factory.go b/store/v2/root/factory.go index 563e439815bc..886eafb70bc2 100644 --- a/store/v2/root/factory.go +++ b/store/v2/root/factory.go @@ -109,12 +109,12 @@ func CreateRootStore(opts *FactoryOptions) (store.RootStore, error) { } ss = storage.NewStorageStore(ssDb, opts.Logger) + metadata := commitment.NewMetadataStore(opts.SCRawDB) + latestVersion, err := metadata.GetLatestVersion() + if err != nil { + return nil, err + } if len(opts.StoreKeys) == 0 { - metadata := commitment.NewMetadataStore(opts.SCRawDB) - latestVersion, err := metadata.GetLatestVersion() - if err != nil { - return nil, err - } lastCommitInfo, err := metadata.GetCommitInfo(latestVersion) if err != nil { return nil, err @@ -126,21 +126,43 @@ func CreateRootStore(opts *FactoryOptions) (store.RootStore, error) { opts.StoreKeys = append(opts.StoreKeys, string(si.Name)) } } + removedStoreKeys, err := metadata.GetRemovedStoreKeys(latestVersion) + if err != nil { + return nil, err + } - trees := make(map[string]commitment.Tree) - for _, key := range opts.StoreKeys { + newTreeFn := func(key string) (commitment.Tree, error) { if internal.IsMemoryStoreKey(key) { - trees[key] = mem.New() + return mem.New(), nil } else { switch storeOpts.SCType { case SCTypeIavl: - trees[key] = iavl.NewIavlTree(db.NewPrefixDB(opts.SCRawDB, []byte(key)), opts.Logger, storeOpts.IavlConfig) + return iavl.NewIavlTree(db.NewPrefixDB(opts.SCRawDB, []byte(key)), opts.Logger, storeOpts.IavlConfig), nil case SCTypeIavlV2: - return nil, errors.New("iavl v2 not supported") + return nil, fmt.Errorf("iavl v2 not supported") + default: + return nil, fmt.Errorf("unsupported commitment store type") } } } - sc, err = commitment.NewCommitStore(trees, opts.SCRawDB, opts.Logger) + + trees := make(map[string]commitment.Tree, len(opts.StoreKeys)) + for _, key := range opts.StoreKeys { + tree, err := newTreeFn(key) + if err != nil { + return nil, err + } + trees[key] = tree + } + oldTrees := make(map[string]commitment.Tree, len(opts.StoreKeys)) + for _, key := range removedStoreKeys { + tree, err := newTreeFn(string(key)) + if err != nil { + return nil, err + } + oldTrees[string(key)] = tree + } + sc, err = commitment.NewCommitStore(trees, oldTrees, opts.SCRawDB, opts.Logger) if err != nil { return nil, err } diff --git a/store/v2/root/migrate_test.go b/store/v2/root/migrate_test.go index 4d1b62b878a5..0cc20ae940d9 100644 --- a/store/v2/root/migrate_test.go +++ b/store/v2/root/migrate_test.go @@ -43,7 +43,7 @@ func (s *MigrateStoreTestSuite) SetupTest() { prefixDB := dbm.NewPrefixDB(mdb, []byte(storeKey)) multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, nopLog, iavl.DefaultConfig()) } - orgSC, err := commitment.NewCommitStore(multiTrees, mdb, testLog) + orgSC, err := commitment.NewCommitStore(multiTrees, nil, mdb, testLog) s.Require().NoError(err) // apply changeset against the original store @@ -70,7 +70,7 @@ func (s *MigrateStoreTestSuite) SetupTest() { for _, storeKey := range storeKeys { multiTrees1[storeKey] = iavl.NewIavlTree(dbm.NewMemDB(), nopLog, iavl.DefaultConfig()) } - sc, err := commitment.NewCommitStore(multiTrees1, dbm.NewMemDB(), testLog) + sc, err := commitment.NewCommitStore(multiTrees1, nil, dbm.NewMemDB(), testLog) s.Require().NoError(err) snapshotsStore, err := snapshots.NewStore(s.T().TempDir()) diff --git a/store/v2/root/store.go b/store/v2/root/store.go index 56f722282f45..218af42b6f89 100644 --- a/store/v2/root/store.go +++ b/store/v2/root/store.go @@ -20,7 +20,10 @@ import ( "cosmossdk.io/store/v2/pruning" ) -var _ store.RootStore = (*Store)(nil) +var ( + _ store.RootStore = (*Store)(nil) + _ store.UpgradeableStore = (*Store)(nil) +) // Store defines the SDK's default RootStore implementation. It contains a single // State Storage (SS) backend and a single State Commitment (SC) backend. The SC @@ -225,7 +228,7 @@ func (s *Store) LoadLatestVersion() error { return err } - return s.loadVersion(lv) + return s.loadVersion(lv, nil) } func (s *Store) LoadVersion(version uint64) error { @@ -234,14 +237,57 @@ func (s *Store) LoadVersion(version uint64) error { defer s.telemetry.MeasureSince(now, "root_store", "load_version") } - return s.loadVersion(version) + return s.loadVersion(version, nil) } -func (s *Store) loadVersion(v uint64) error { +// LoadVersionAndUpgrade implements the UpgradeableStore interface. +// +// NOTE: It cannot be called while the store is migrating. +func (s *Store) LoadVersionAndUpgrade(version uint64, upgrades *corestore.StoreUpgrades) error { + if upgrades == nil { + return fmt.Errorf("upgrades cannot be nil") + } + + if s.telemetry != nil { + defer s.telemetry.MeasureSince(time.Now(), "root_store", "load_version_and_upgrade") + } + + if s.isMigrating { + return fmt.Errorf("cannot upgrade while migrating") + } + + if err := s.loadVersion(version, upgrades); err != nil { + return err + } + + // if the state storage implements the UpgradableDatabase interface, prune the + // deleted store keys + upgradableDatabase, ok := s.stateStorage.(store.UpgradableDatabase) + if ok { + if err := upgradableDatabase.PruneStoreKeys(upgrades.Deleted, version); err != nil { + return fmt.Errorf("failed to prune store keys %v: %w", upgrades.Deleted, err) + } + } + + return nil +} + +func (s *Store) loadVersion(v uint64, upgrades *corestore.StoreUpgrades) error { s.logger.Debug("loading version", "version", v) - if err := s.stateCommitment.LoadVersion(v); err != nil { - return fmt.Errorf("failed to load SC version %d: %w", v, err) + if upgrades == nil { + if err := s.stateCommitment.LoadVersion(v); err != nil { + return fmt.Errorf("failed to load SC version %d: %w", v, err) + } + } else { + // if upgrades are provided, we need to load the version and apply the upgrades + upgradeableStore, ok := s.stateCommitment.(store.UpgradeableStore) + if !ok { + return fmt.Errorf("SC store does not support upgrades") + } + if err := upgradeableStore.LoadVersionAndUpgrade(v, upgrades); err != nil { + return fmt.Errorf("failed to load SS version with upgrades %d: %w", v, err) + } } s.commitHeader = nil @@ -301,8 +347,8 @@ func (s *Store) WorkingHash(cs *corestore.Changeset) ([]byte, error) { // Commit commits all state changes to the underlying SS and SC backends. It // writes a batch of the changeset to the SC tree, and retrieves the CommitInfo -// from the SC tree. Finally, it commits the SC tree and returns the hash of the -// CommitInfo. +// from the SC tree. Finally, it commits the SC tree and returns the hash of +// the CommitInfo. func (s *Store) Commit(cs *corestore.Changeset) ([]byte, error) { if s.telemetry != nil { now := time.Now() diff --git a/store/v2/root/store_test.go b/store/v2/root/store_test.go index b2b640feb974..fe7552ee6bd2 100644 --- a/store/v2/root/store_test.go +++ b/store/v2/root/store_test.go @@ -55,7 +55,7 @@ func (s *RootStoreTestSuite) SetupTest() { tree := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) tree2 := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) tree3 := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) - sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree, testStoreKey2: tree2, testStoreKey3: tree3}, dbm.NewMemDB(), noopLog) + sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree, testStoreKey2: tree2, testStoreKey3: tree3}, nil, dbm.NewMemDB(), noopLog) s.Require().NoError(err) pm := pruning.NewManager(sc, ss, nil, nil) @@ -79,7 +79,7 @@ func (s *RootStoreTestSuite) newStoreWithPruneConfig(config *store.PruningOption multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, noopLog, iavl.DefaultConfig()) } - sc, err := commitment.NewCommitStore(multiTrees, dbm.NewMemDB(), noopLog) + sc, err := commitment.NewCommitStore(multiTrees, nil, dbm.NewMemDB(), noopLog) s.Require().NoError(err) pm := pruning.NewManager(sc, ss, config, config) @@ -563,7 +563,7 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { ss := storage.NewStorageStore(sqliteDB, noopLog) tree := iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig()) - sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, mdb2, noopLog) + sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog) s.Require().NoError(err) pm := pruning.NewManager(sc, ss, pruneOpt, pruneOpt) @@ -593,7 +593,7 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { ss = storage.NewStorageStore(sqliteDB, noopLog) tree = iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig()) - sc, err = commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, mdb2, noopLog) + sc, err = commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog) s.Require().NoError(err) pm = pruning.NewManager(sc, ss, pruneOpt, pruneOpt) @@ -624,7 +624,7 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { for v := uint64(1); v <= actualHeightToPrune; v++ { checkErr := func() bool { - if err = s.rootStore.LoadVersion(v); err != nil { + if _, err = s.rootStore.StateAt(v); err != nil { return true } return false @@ -650,7 +650,7 @@ func (s *RootStoreTestSuite) TestMultiStoreRestart() { multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, noopLog, iavl.DefaultConfig()) } - sc, err := commitment.NewCommitStore(multiTrees, mdb2, noopLog) + sc, err := commitment.NewCommitStore(multiTrees, nil, mdb2, noopLog) s.Require().NoError(err) pm := pruning.NewManager(sc, ss, nil, nil) @@ -737,7 +737,7 @@ func (s *RootStoreTestSuite) TestMultiStoreRestart() { multiTrees[storeKey] = iavl.NewIavlTree(prefixDB, noopLog, iavl.DefaultConfig()) } - sc, err = commitment.NewCommitStore(multiTrees, mdb2, noopLog) + sc, err = commitment.NewCommitStore(multiTrees, nil, mdb2, noopLog) s.Require().NoError(err) pm = pruning.NewManager(sc, ss, nil, nil) diff --git a/store/v2/root/upgrade_test.go b/store/v2/root/upgrade_test.go new file mode 100644 index 000000000000..a4aee1d5bcfa --- /dev/null +++ b/store/v2/root/upgrade_test.go @@ -0,0 +1,157 @@ +package root + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/suite" + + corestore "cosmossdk.io/core/store" + coretesting "cosmossdk.io/core/testing" + "cosmossdk.io/log" + "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/commitment" + "cosmossdk.io/store/v2/commitment/iavl" + dbm "cosmossdk.io/store/v2/db" + "cosmossdk.io/store/v2/pruning" + "cosmossdk.io/store/v2/storage" + "cosmossdk.io/store/v2/storage/sqlite" +) + +type UpgradeStoreTestSuite struct { + suite.Suite + + commitDB corestore.KVStoreWithBatch + rootStore store.RootStore +} + +func TestUpgradeStoreTestSuite(t *testing.T) { + suite.Run(t, &UpgradeStoreTestSuite{}) +} + +func (s *UpgradeStoreTestSuite) SetupTest() { + testLog := log.NewTestLogger(s.T()) + nopLog := coretesting.NewNopLogger() + + s.commitDB = dbm.NewMemDB() + multiTrees := make(map[string]commitment.Tree) + newTreeFn := func(storeKey string) (commitment.Tree, error) { + prefixDB := dbm.NewPrefixDB(s.commitDB, []byte(storeKey)) + return iavl.NewIavlTree(prefixDB, nopLog, iavl.DefaultConfig()), nil + } + for _, storeKey := range storeKeys { + multiTrees[storeKey], _ = newTreeFn(storeKey) + } + + // create storage and commitment stores + sqliteDB, err := sqlite.New(s.T().TempDir()) + s.Require().NoError(err) + ss := storage.NewStorageStore(sqliteDB, testLog) + sc, err := commitment.NewCommitStore(multiTrees, nil, s.commitDB, testLog) + s.Require().NoError(err) + pm := pruning.NewManager(sc, ss, nil, nil) + s.rootStore, err = New(testLog, ss, sc, pm, nil, nil) + s.Require().NoError(err) + + // commit changeset + toVersion := uint64(20) + keyCount := 10 + for version := uint64(1); version <= toVersion; version++ { + cs := corestore.NewChangeset() + for _, storeKey := range storeKeys { + for i := 0; i < keyCount; i++ { + cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false) + } + } + _, err = s.rootStore.Commit(cs) + s.Require().NoError(err) + } +} + +func (s *UpgradeStoreTestSuite) loadWithUpgrades(upgrades *corestore.StoreUpgrades) { + testLog := log.NewTestLogger(s.T()) + nopLog := coretesting.NewNopLogger() + + // create a new commitment store + multiTrees := make(map[string]commitment.Tree) + oldTrees := make(map[string]commitment.Tree) + newTreeFn := func(storeKey string) (commitment.Tree, error) { + prefixDB := dbm.NewPrefixDB(s.commitDB, []byte(storeKey)) + return iavl.NewIavlTree(prefixDB, nopLog, iavl.DefaultConfig()), nil + } + for _, storeKey := range storeKeys { + multiTrees[storeKey], _ = newTreeFn(storeKey) + } + for _, added := range upgrades.Added { + multiTrees[added], _ = newTreeFn(added) + } + for _, deleted := range upgrades.Deleted { + oldTrees[deleted], _ = newTreeFn(deleted) + } + + sc, err := commitment.NewCommitStore(multiTrees, oldTrees, s.commitDB, testLog) + s.Require().NoError(err) + pm := pruning.NewManager(sc, s.rootStore.GetStateStorage().(store.Pruner), nil, nil) + s.rootStore, err = New(testLog, s.rootStore.GetStateStorage(), sc, pm, nil, nil) + s.Require().NoError(err) +} + +func (s *UpgradeStoreTestSuite) TestLoadVersionAndUpgrade() { + // upgrade store keys + upgrades := &corestore.StoreUpgrades{ + Added: []string{"newStore1", "newStore2"}, + Deleted: []string{"store3"}, + } + s.loadWithUpgrades(upgrades) + + // load the store with the upgrades + v, err := s.rootStore.GetLatestVersion() + s.Require().NoError(err) + err = s.rootStore.(store.UpgradeableStore).LoadVersionAndUpgrade(v, upgrades) + s.Require().NoError(err) + + keyCount := 10 + // check old store keys are queryable + oldStoreKeys := []string{"store1", "store3"} + for _, storeKey := range oldStoreKeys { + for version := uint64(1); version <= v; version++ { + for i := 0; i < keyCount; i++ { + proof, err := s.rootStore.Query([]byte(storeKey), version, []byte(fmt.Sprintf("key-%d-%d", version, i)), true) + s.Require().NoError(err) + s.Require().NotNil(proof) + } + } + } + + // commit changeset + newStoreKeys := []string{"newStore1", "newStore2"} + toVersion := uint64(40) + for version := v + 1; version <= toVersion; version++ { + cs := corestore.NewChangeset() + for _, storeKey := range newStoreKeys { + for i := 0; i < keyCount; i++ { + cs.Add([]byte(storeKey), []byte(fmt.Sprintf("key-%d-%d", version, i)), []byte(fmt.Sprintf("value-%d-%d", version, i)), false) + } + } + _, err = s.rootStore.Commit(cs) + s.Require().NoError(err) + } + + // check new store keys are queryable + for _, storeKey := range newStoreKeys { + for version := v + 1; version <= toVersion; version++ { + for i := 0; i < keyCount; i++ { + _, err := s.rootStore.Query([]byte(storeKey), version, []byte(fmt.Sprintf("key-%d-%d", version, i)), true) + s.Require().NoError(err) + } + } + } + + // check the original store key is queryable + for version := uint64(1); version <= toVersion; version++ { + for i := 0; i < keyCount; i++ { + _, err := s.rootStore.Query([]byte("store2"), version, []byte(fmt.Sprintf("key-%d-%d", version, i)), true) + s.Require().NoError(err) + } + } +} diff --git a/store/v2/storage/pebbledb/comparator.go b/store/v2/storage/pebbledb/comparator.go index 337ff7698d84..24f5e05a6214 100644 --- a/store/v2/storage/pebbledb/comparator.go +++ b/store/v2/storage/pebbledb/comparator.go @@ -155,7 +155,7 @@ func SplitMVCCKey(mvccKey []byte) (key, version []byte, ok bool) { key = mvccKeyCopy[:n-tsLen] if tsLen > 0 { - version = mvccKeyCopy[n-tsLen+1 : len(mvccKeyCopy)-1] + version = mvccKeyCopy[n-tsLen+1 : n] } return key, version, true diff --git a/store/v2/storage/pebbledb/db.go b/store/v2/storage/pebbledb/db.go index 4e9df737b57e..ebcd242af243 100644 --- a/store/v2/storage/pebbledb/db.go +++ b/store/v2/storage/pebbledb/db.go @@ -13,7 +13,9 @@ import ( corestore "cosmossdk.io/core/store" "cosmossdk.io/store/v2" storeerrors "cosmossdk.io/store/v2/errors" + "cosmossdk.io/store/v2/internal/encoding" "cosmossdk.io/store/v2/storage" + "cosmossdk.io/store/v2/storage/util" ) const ( @@ -21,14 +23,20 @@ const ( // PruneCommitBatchSize defines the size, in number of key/value pairs, to prune // in a single batch. PruneCommitBatchSize = 50 - - StorePrefixTpl = "s/k:%s/" // s/k: - latestVersionKey = "s/_latest" // NB: latestVersionKey key must be lexically smaller than StorePrefixTpl - pruneHeightKey = "s/_prune_height" // NB: pruneHeightKey key must be lexically smaller than StorePrefixTpl - tombstoneVal = "TOMBSTONE" + // batchBufferSize defines the maximum size of a batch before it is committed. + batchBufferSize = 100_000 + + StorePrefixTpl = "s/k:%s/" // s/k: + removedStoreKeyPrefix = "s/_removed_key" // NB: removedStoreKeys key must be lexically smaller than StorePrefixTpl + latestVersionKey = "s/_latest" // NB: latestVersionKey key must be lexically smaller than StorePrefixTpl + pruneHeightKey = "s/_prune_height" // NB: pruneHeightKey key must be lexically smaller than StorePrefixTpl + tombstoneVal = "TOMBSTONE" ) -var _ storage.Database = (*Database)(nil) +var ( + _ storage.Database = (*Database)(nil) + _ store.UpgradableDatabase = (*Database)(nil) +) type Database struct { storage *pebble.DB @@ -252,7 +260,11 @@ func (db *Database) Prune(version uint64) error { prevKey = keyBz prevKeyVersion = keyVersion prevKeyPrefixed = prefixedKey - prevPrefixedVal = slices.Clone(itr.Value()) + value, err := itr.ValueAndErr() + if err != nil { + return err + } + prevPrefixedVal = slices.Clone(value) itr.Next() } @@ -264,6 +276,10 @@ func (db *Database) Prune(version uint64) error { } } + if err := db.deleteRemovedStoreKeys(version); err != nil { + return err + } + return db.setPruneHeight(version) } @@ -315,12 +331,25 @@ func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end return newPebbleDBIterator(itr, storePrefix(storeKey), start, end, version, db.earliestVersion, true), nil } +func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) error { + batch := db.storage.NewBatch() + defer batch.Close() + + for _, storeKey := range storeKeys { + if err := batch.Set([]byte(fmt.Sprintf("%s%s", encoding.BuildPrefixWithVersion(removedStoreKeyPrefix, version), storeKey)), []byte{}, nil); err != nil { + return err + } + } + + return batch.Commit(&pebble.WriteOptions{Sync: db.sync}) +} + func storePrefix(storeKey []byte) []byte { - return append([]byte(StorePrefixTpl), storeKey...) + return []byte(fmt.Sprintf(StorePrefixTpl, storeKey)) } func prependStoreKey(storeKey, key []byte) []byte { - return append(storePrefix(storeKey), key...) + return []byte(fmt.Sprintf("%s%s", storePrefix(storeKey), key)) } func getPruneHeight(storage *pebble.DB) (uint64, error) { @@ -395,5 +424,80 @@ func getMVCCSlice(db *pebble.DB, storeKey, key []byte, version uint64) ([]byte, return nil, fmt.Errorf("key version too large: %d", keyVersion) } - return slices.Clone(itr.Value()), nil + value, err := itr.ValueAndErr() + return slices.Clone(value), err +} + +func (db *Database) deleteRemovedStoreKeys(version uint64) error { + batch := db.storage.NewBatch() + defer batch.Close() + + end := encoding.BuildPrefixWithVersion(removedStoreKeyPrefix, version+1) + storeKeyIter, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: []byte(removedStoreKeyPrefix), UpperBound: end}) + if err != nil { + return err + } + defer storeKeyIter.Close() + + storeKeys := make(map[string]uint64) + prefixLen := len(end) + for storeKeyIter.First(); storeKeyIter.Valid(); storeKeyIter.Next() { + verBz := storeKeyIter.Key()[len(removedStoreKeyPrefix):prefixLen] + v, err := decodeUint64Ascending(verBz) + if err != nil { + return err + } + storeKey := string(storeKeyIter.Key()[prefixLen:]) + if ev, ok := storeKeys[storeKey]; ok { + if ev < v { + storeKeys[storeKey] = v + } + } else { + storeKeys[storeKey] = v + } + if err := batch.Delete(storeKeyIter.Key(), nil); err != nil { + return err + } + } + + for storeKey, v := range storeKeys { + if err := func() error { + storeKey := []byte(storeKey) + itr, err := db.storage.NewIter(&pebble.IterOptions{LowerBound: storePrefix(storeKey), UpperBound: storePrefix(util.CopyIncr(storeKey))}) + if err != nil { + return err + } + defer itr.Close() + + for itr.First(); itr.Valid(); itr.Next() { + itrKey := itr.Key() + _, verBz, ok := SplitMVCCKey(itrKey) + if !ok { + return fmt.Errorf("invalid PebbleDB MVCC key: %s", itrKey) + } + keyVersion, err := decodeUint64Ascending(verBz) + if err != nil { + return err + } + if keyVersion > v { + // skip keys that are newer than the version + continue + } + if err := batch.Delete(itr.Key(), nil); err != nil { + return err + } + if batch.Len() >= batchBufferSize { + if err := batch.Commit(&pebble.WriteOptions{Sync: db.sync}); err != nil { + return err + } + batch.Reset() + } + } + return nil + }(); err != nil { + return err + } + } + + return batch.Commit(&pebble.WriteOptions{Sync: true}) } diff --git a/store/v2/storage/rocksdb/db.go b/store/v2/storage/rocksdb/db.go index 480aeb2cff2f..afac4cc9ff22 100644 --- a/store/v2/storage/rocksdb/db.go +++ b/store/v2/storage/rocksdb/db.go @@ -26,7 +26,8 @@ const ( ) var ( - _ storage.Database = (*Database)(nil) + _ storage.Database = (*Database)(nil) + _ store.UpgradableDatabase = (*Database)(nil) defaultWriteOpts = grocksdb.NewDefaultWriteOptions() defaultReadOpts = grocksdb.NewDefaultReadOptions() @@ -196,6 +197,12 @@ func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end return newRocksDBIterator(itr, prefix, start, end, true), nil } +// PruneStoreKeys will do nothing for RocksDB, it will be pruned by compaction +// when the version is pruned +func (db *Database) PruneStoreKeys(_ []string, _ uint64) error { + return nil +} + // newTSReadOptions returns ReadOptions used in the RocksDB column family read. func newTSReadOptions(version uint64) *grocksdb.ReadOptions { var ts [TimestampSize]byte @@ -208,11 +215,11 @@ func newTSReadOptions(version uint64) *grocksdb.ReadOptions { } func storePrefix(storeKey []byte) []byte { - return append([]byte(StorePrefixTpl), storeKey...) + return []byte(fmt.Sprintf(StorePrefixTpl, storeKey)) } func prependStoreKey(storeKey, key []byte) []byte { - return append(storePrefix(storeKey), key...) + return []byte(fmt.Sprintf("%s%s", storePrefix(storeKey), key)) } // copyAndFreeSlice will copy a given RocksDB slice and free it. If the slice does diff --git a/store/v2/storage/rocksdb/db_test.go b/store/v2/storage/rocksdb/db_test.go index b807f07259d7..a77afbb3a8fd 100644 --- a/store/v2/storage/rocksdb/db_test.go +++ b/store/v2/storage/rocksdb/db_test.go @@ -23,6 +23,7 @@ func TestStorageTestSuite(t *testing.T) { return storage.NewStorageStore(db, coretesting.NewNopLogger()), err }, EmptyBatchSize: 12, + SkipTests: []string{"TestUpgradable_Prune"}, } suite.Run(t, s) } diff --git a/store/v2/storage/rocksdb/iterator.go b/store/v2/storage/rocksdb/iterator.go index 7427e90406f7..9a09dc92c5c8 100644 --- a/store/v2/storage/rocksdb/iterator.go +++ b/store/v2/storage/rocksdb/iterator.go @@ -124,6 +124,10 @@ func (itr *iterator) Value() []byte { return copyAndFreeSlice(itr.source.Value()) } +func (itr *iterator) Timestamp() []byte { + return itr.source.Timestamp().Data() +} + func (itr iterator) Next() { if itr.invalid { return diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index 1ee52583aec4..38750a0ac9a5 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -17,11 +17,12 @@ import ( ) const ( - driverName = "sqlite3" - dbName = "ss.db?cache=shared&mode=rwc&_journal_mode=WAL" - reservedStoreKey = "_RESERVED_" - keyLatestHeight = "latest_height" - keyPruneHeight = "prune_height" + driverName = "sqlite3" + dbName = "ss.db?cache=shared&mode=rwc&_journal_mode=WAL" + reservedStoreKey = "_RESERVED_" + keyLatestHeight = "latest_height" + keyPruneHeight = "prune_height" + valueRemovedStore = "removed_store" reservedUpsertStmt = ` INSERT INTO state_storage(store_key, key, value, version) @@ -43,7 +44,10 @@ const ( ` ) -var _ storage.Database = (*Database)(nil) +var ( + _ storage.Database = (*Database)(nil) + _ store.UpgradableDatabase = (*Database)(nil) +) type Database struct { storage *sql.DB @@ -186,7 +190,13 @@ func (db *Database) Prune(version uint64) error { if err != nil { return fmt.Errorf("failed to create SQL transaction: %w", err) } + defer func() { + if err != nil { + err = tx.Rollback() + } + }() + // prune all keys of old versions pruneStmt := `DELETE FROM state_storage WHERE version < ( SELECT max(version) FROM state_storage t2 WHERE @@ -195,15 +205,34 @@ func (db *Database) Prune(version uint64) error { t2.version <= ? ) AND store_key != ?; ` + if _, err := tx.Exec(pruneStmt, version, reservedStoreKey); err != nil { + return fmt.Errorf("failed to exec SQL statement: %w", err) + } - _, err = tx.Exec(pruneStmt, version, reservedStoreKey) - if err != nil { + // prune removed stores + pruneRemovedStoreKeysStmt := `DELETE FROM state_storage AS s + WHERE EXISTS ( + SELECT 1 FROM + ( + SELECT key, MAX(version) AS max_version + FROM state_storage + WHERE store_key = ? AND value = ? AND version <= ? + GROUP BY key + ) AS t + WHERE s.store_key = t.key AND s.version <= t.max_version LIMIT 1 + ); + ` + if _, err := tx.Exec(pruneRemovedStoreKeysStmt, reservedStoreKey, valueRemovedStore, version, version); err != nil { + return fmt.Errorf("failed to exec SQL statement: %w", err) + } + + // delete the removedKeys + if _, err := tx.Exec("DELETE FROM state_storage WHERE store_key = ? AND value = ? AND version <= ?", reservedStoreKey, valueRemovedStore, version); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } // set the prune height so we can return for queries below this height - _, err = tx.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, version, 0, version) - if err != nil { + if _, err := tx.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, version, 0, version); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } @@ -212,7 +241,6 @@ func (db *Database) Prune(version uint64) error { } db.earliestVersion = version + 1 - return nil } @@ -240,6 +268,29 @@ func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end return newIterator(db, storeKey, version, start, end, true) } +func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err error) { + tx, err := db.storage.Begin() + if err != nil { + return fmt.Errorf("failed to create SQL transaction: %w", err) + } + defer func() { + if err != nil { + err = tx.Rollback() + } + }() + + // flush removed store keys + flushRemovedStoreKeyStmt := `INSERT INTO state_storage(store_key, key, value, version) + VALUES (?, ?, ?, ?)` + for _, storeKey := range storeKeys { + if _, err := tx.Exec(flushRemovedStoreKeyStmt, reservedStoreKey, []byte(storeKey), valueRemovedStore, version); err != nil { + return fmt.Errorf("failed to exec SQL statement: %w", err) + } + } + + return tx.Commit() +} + func (db *Database) PrintRowsDebug() { stmt, err := db.storage.Prepare("SELECT store_key, key, value, version, tombstone FROM state_storage") if err != nil { diff --git a/store/v2/storage/storage_test_suite.go b/store/v2/storage/storage_test_suite.go index 475c529b624f..2bc745221d75 100644 --- a/store/v2/storage/storage_test_suite.go +++ b/store/v2/storage/storage_test_suite.go @@ -411,11 +411,11 @@ func (s *StorageTestSuite) TestDatabaseIterator_SkipVersion() { defer db.Close() - DBApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) - DBApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value000")}) - DBApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value000")}) - DBApplyChangeset(s.T(), db, 58833605, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value004")}) - DBApplyChangeset(s.T(), db, 58833606, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value006")}) + dbApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) + dbApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value000")}) + dbApplyChangeset(s.T(), db, 58827506, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value000")}) + dbApplyChangeset(s.T(), db, 58833605, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value004")}) + dbApplyChangeset(s.T(), db, 58833606, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value006")}) itr, err := db.Iterator(storeKey1Bytes, 58831525, []byte("key"), nil) s.Require().NoError(err) @@ -435,15 +435,15 @@ func (s *StorageTestSuite) TestDatabaseIterator_ForwardIteration() { s.Require().NoError(err) defer db.Close() - DBApplyChangeset(s.T(), db, 8, storeKey1, [][]byte{[]byte("keyA")}, [][]byte{[]byte("value001")}) - DBApplyChangeset(s.T(), db, 9, storeKey1, [][]byte{[]byte("keyB")}, [][]byte{[]byte("value002")}) - DBApplyChangeset(s.T(), db, 10, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) - DBApplyChangeset(s.T(), db, 11, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value004")}) + dbApplyChangeset(s.T(), db, 8, storeKey1, [][]byte{[]byte("keyA")}, [][]byte{[]byte("value001")}) + dbApplyChangeset(s.T(), db, 9, storeKey1, [][]byte{[]byte("keyB")}, [][]byte{[]byte("value002")}) + dbApplyChangeset(s.T(), db, 10, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) + dbApplyChangeset(s.T(), db, 11, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value004")}) - DBApplyChangeset(s.T(), db, 2, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value007")}) - DBApplyChangeset(s.T(), db, 3, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value008")}) - DBApplyChangeset(s.T(), db, 4, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value009")}) - DBApplyChangeset(s.T(), db, 5, storeKey1, [][]byte{[]byte("keyH")}, [][]byte{[]byte("value010")}) + dbApplyChangeset(s.T(), db, 2, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value007")}) + dbApplyChangeset(s.T(), db, 3, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value008")}) + dbApplyChangeset(s.T(), db, 4, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value009")}) + dbApplyChangeset(s.T(), db, 5, storeKey1, [][]byte{[]byte("keyH")}, [][]byte{[]byte("value010")}) itr, err := db.Iterator(storeKey1Bytes, 6, nil, []byte("keyZ")) s.Require().NoError(err) @@ -463,14 +463,14 @@ func (s *StorageTestSuite) TestDatabaseIterator_ForwardIterationHigher() { s.Require().NoError(err) defer db.Close() - DBApplyChangeset(s.T(), db, 9, storeKey1, [][]byte{[]byte("keyB")}, [][]byte{[]byte("value002")}) - DBApplyChangeset(s.T(), db, 10, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) - DBApplyChangeset(s.T(), db, 11, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value004")}) + dbApplyChangeset(s.T(), db, 9, storeKey1, [][]byte{[]byte("keyB")}, [][]byte{[]byte("value002")}) + dbApplyChangeset(s.T(), db, 10, storeKey1, [][]byte{[]byte("keyC")}, [][]byte{[]byte("value003")}) + dbApplyChangeset(s.T(), db, 11, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value004")}) - DBApplyChangeset(s.T(), db, 12, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value007")}) - DBApplyChangeset(s.T(), db, 13, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value008")}) - DBApplyChangeset(s.T(), db, 14, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value009")}) - DBApplyChangeset(s.T(), db, 15, storeKey1, [][]byte{[]byte("keyH")}, [][]byte{[]byte("value010")}) + dbApplyChangeset(s.T(), db, 12, storeKey1, [][]byte{[]byte("keyD")}, [][]byte{[]byte("value007")}) + dbApplyChangeset(s.T(), db, 13, storeKey1, [][]byte{[]byte("keyE")}, [][]byte{[]byte("value008")}) + dbApplyChangeset(s.T(), db, 14, storeKey1, [][]byte{[]byte("keyF")}, [][]byte{[]byte("value009")}) + dbApplyChangeset(s.T(), db, 15, storeKey1, [][]byte{[]byte("keyH")}, [][]byte{[]byte("value010")}) itr, err := db.Iterator(storeKey1Bytes, 6, nil, []byte("keyZ")) s.Require().NoError(err) @@ -640,7 +640,154 @@ func (s *StorageTestSuite) TestDatabase_Prune_KeepRecent() { s.Require().Equal([]byte("val200"), bz) } -func DBApplyChangeset( +func (s *StorageTestSuite) TestUpgradable() { + ss, err := s.NewDB(s.T().TempDir()) + s.Require().NoError(err) + defer ss.Close() + + // Ensure the database is upgradable. + if _, ok := ss.db.(store.UpgradableDatabase); !ok { + s.T().Skip("database is not upgradable") + } + + storeKeys := []string{"store1", "store2", "store3"} + uptoVersion := uint64(50) + keyCount := 10 + for _, storeKey := range storeKeys { + for v := uint64(1); v <= uptoVersion; v++ { + keys := make([][]byte, keyCount) + vals := make([][]byte, keyCount) + for i := 0; i < keyCount; i++ { + keys[i] = []byte(fmt.Sprintf("key%03d", i)) + vals[i] = []byte(fmt.Sprintf("val%03d-%03d", i, v)) + } + dbApplyChangeset(s.T(), ss, v, storeKey, keys, vals) + } + } + + // prune storekeys (`store2`, `store3`) + removedStoreKeys := []string{storeKeys[1], storeKeys[2]} + err = ss.PruneStoreKeys(removedStoreKeys, uptoVersion) + s.Require().NoError(err) + // should be able to query before Prune for removed storeKeys + for _, storeKey := range removedStoreKeys { + for v := uint64(1); v <= uptoVersion; v++ { + for i := 0; i < keyCount; i++ { + bz, err := ss.Get([]byte(storeKey), v, []byte(fmt.Sprintf("key%03d", i))) + s.Require().NoError(err) + s.Require().Equal([]byte(fmt.Sprintf("val%03d-%03d", i, v)), bz) + } + } + } + s.Require().NoError(ss.Prune(uptoVersion)) + // should not be able to query after Prune + // skip the test of RocksDB + if !slices.Contains(s.SkipTests, "TestUpgradable_Prune") { + for _, storeKey := range removedStoreKeys { + // it will return error ErrVersionPruned + for v := uint64(1); v <= uptoVersion; v++ { + for i := 0; i < keyCount; i++ { + _, err := ss.Get([]byte(storeKey), v, []byte(fmt.Sprintf("key%03d", i))) + s.Require().Error(err) + } + } + v := uptoVersion + 1 + for i := 0; i < keyCount; i++ { + val, err := ss.Get([]byte(storeKey), v, []byte(fmt.Sprintf("key%03d", i))) + s.Require().NoError(err) + s.Require().Nil(val) + } + } + } +} + +func (s *StorageTestSuite) TestRemovingOldStoreKey() { + ss, err := s.NewDB(s.T().TempDir()) + s.Require().NoError(err) + defer ss.Close() + + // Ensure the database is upgradable. + if _, ok := ss.db.(store.UpgradableDatabase); !ok { + s.T().Skip("database is not upgradable") + } + + storeKeys := []string{"store1", "store2", "store3"} + uptoVersion := uint64(50) + keyCount := 10 + for _, storeKey := range storeKeys { + for v := uint64(1); v <= uptoVersion; v++ { + keys := make([][]byte, keyCount) + vals := make([][]byte, keyCount) + for i := 0; i < keyCount; i++ { + keys[i] = []byte(fmt.Sprintf("key%03d-%03d", i, v)) + vals[i] = []byte(fmt.Sprintf("val%03d-%03d", i, v)) + } + dbApplyChangeset(s.T(), ss, v, storeKey, keys, vals) + } + } + + // remove `store1` and `store3` + removedStoreKeys := []string{storeKeys[0], storeKeys[2]} + err = ss.PruneStoreKeys(removedStoreKeys, uptoVersion) + s.Require().NoError(err) + // should be able to query before Prune for removed storeKeys + for _, storeKey := range removedStoreKeys { + for v := uint64(1); v <= uptoVersion; v++ { + for i := 0; i < keyCount; i++ { + bz, err := ss.Get([]byte(storeKey), v, []byte(fmt.Sprintf("key%03d-%03d", i, v))) + s.Require().NoError(err) + s.Require().Equal([]byte(fmt.Sprintf("val%03d-%03d", i, v)), bz) + } + } + } + // add `store1` back + newStoreKeys := []string{storeKeys[0], storeKeys[1]} + newVersion := uptoVersion + 10 + for _, storeKey := range newStoreKeys { + for v := uptoVersion + 1; v <= newVersion; v++ { + keys := make([][]byte, keyCount) + vals := make([][]byte, keyCount) + for i := 0; i < keyCount; i++ { + keys[i] = []byte(fmt.Sprintf("key%03d-%03d", i, v)) + vals[i] = []byte(fmt.Sprintf("val%03d-%03d", i, v)) + } + dbApplyChangeset(s.T(), ss, v, storeKey, keys, vals) + } + } + + s.Require().NoError(ss.Prune(newVersion)) + // skip the test of RocksDB + if !slices.Contains(s.SkipTests, "TestUpgradable_Prune") { + for _, storeKey := range removedStoreKeys { + queryVersion := newVersion + 1 + // should not be able to query after Prune during 1 ~ uptoVersion + for v := uint64(1); v <= uptoVersion; v++ { + for i := 0; i < keyCount; i++ { + val, err := ss.Get([]byte(storeKey), queryVersion, []byte(fmt.Sprintf("key%03d", i))) + s.Require().NoError(err) + s.Require().Nil(val) + } + } + // should be able to query after Prune during uptoVersion + 1 ~ newVersion + // for `store1` added back + for v := uptoVersion + 1; v <= newVersion; v++ { + for i := 0; i < keyCount; i++ { + val, err := ss.Get([]byte(storeKey), queryVersion, []byte(fmt.Sprintf("key%03d-%03d", i, v))) + s.Require().NoError(err) + if storeKey == storeKeys[0] { + // `store1` is added back + s.Require().Equal([]byte(fmt.Sprintf("val%03d-%03d", i, v)), val) + } else { + // `store3` is removed + s.Require().Nil(val) + } + } + } + } + } +} + +func dbApplyChangeset( t *testing.T, db store.VersionedDatabase, version uint64, diff --git a/store/v2/storage/store.go b/store/v2/storage/store.go index 25381ee18582..9ea839562847 100644 --- a/store/v2/storage/store.go +++ b/store/v2/storage/store.go @@ -1,6 +1,7 @@ package storage import ( + "errors" "fmt" "cosmossdk.io/core/log" @@ -18,6 +19,7 @@ var ( _ store.VersionedDatabase = (*StorageStore)(nil) _ snapshots.StorageSnapshotter = (*StorageStore)(nil) _ store.Pruner = (*StorageStore)(nil) + _ store.UpgradableDatabase = (*StorageStore)(nil) ) // StorageStore is a wrapper around the store.VersionedDatabase interface. @@ -137,6 +139,17 @@ func (ss *StorageStore) Restore(version uint64, chStorage <-chan *corestore.Stat return nil } +// PruneStoreKeys prunes the store keys which implements the store.UpgradableDatabase +// interface. +func (ss *StorageStore) PruneStoreKeys(storeKeys []string, version uint64) error { + gdb, ok := ss.db.(store.UpgradableDatabase) + if !ok { + return errors.New("db does not implement UpgradableDatabase interface") + } + + return gdb.PruneStoreKeys(storeKeys, version) +} + // Close closes the store. func (ss *StorageStore) Close() error { return ss.db.Close() diff --git a/store/v2/store.go b/store/v2/store.go index 71a3cc48bcad..fee3ad39dbf2 100644 --- a/store/v2/store.go +++ b/store/v2/store.go @@ -72,16 +72,13 @@ type RootStore interface { io.Closer } -// UpgradeableRootStore extends the RootStore interface to support loading versions -// with upgrades. -type UpgradeableRootStore interface { - RootStore - +// UpgradeableStore defines the interface for upgrading store keys. +type UpgradeableStore interface { // LoadVersionAndUpgrade behaves identically to LoadVersion except it also // accepts a StoreUpgrades object that defines a series of transformations to // apply to store keys (if any). // - // Note, handling StoreUpgrades is optional depending on the underlying RootStore + // Note, handling StoreUpgrades is optional depending on the underlying store // implementation. LoadVersionAndUpgrade(version uint64, upgrades *corestore.StoreUpgrades) error }