From 4ed78884ae67ef1282ab4d8c45fe07e541783bdb Mon Sep 17 00:00:00 2001 From: Robert Zaremba Date: Tue, 27 Dec 2022 00:08:56 +0100 Subject: [PATCH 1/7] feat!: remove x/gov/migrations/v046.Migrate_V046_6_To_V046_7 (#14422) --- CHANGELOG.md | 4 ++++ x/gov/migrations/v046/store.go | 14 -------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cf9658293a..d0784642de46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,10 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (x/gov) [#14347](https://github.com/cosmos/cosmos-sdk/pull/14347) Support `v1.Proposal` message in `v1beta1.Proposal.Content`. +### API Breaking + +* (x/gov) [#14422](https://github.com/cosmos/cosmos-sdk/pull/14422) Remove `Migrate_V046_6_To_V046_7` function which shouldn't be used for chains which already migrated to 0.46. + ## [v0.46.7](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.7) - 2022-12-13 ### Features diff --git a/x/gov/migrations/v046/store.go b/x/gov/migrations/v046/store.go index 9fbcf9f3413d..5da5d253fe47 100644 --- a/x/gov/migrations/v046/store.go +++ b/x/gov/migrations/v046/store.go @@ -93,17 +93,3 @@ func MigrateStore(ctx sdk.Context, storeKey storetypes.StoreKey, cdc codec.Binar return migrateProposals(store, cdc) } - -// Migrate_V046_4_To_V046_5 is a helper function to migrate chains from <=v0.46.6 -// to v0.46.7 ONLY. -// -// IMPORTANT: Please do not use this function if you are upgrading to v0.46 -// from <=v0.45. -// -// This function migrates the store in-place by fixing the gov votes weight to -// be stored as decimals strings (instead of the sdk.Dec BigInt representation). -// -// The store is expected to be the gov store, and not any prefixed substore. -func Migrate_V046_6_To_V046_7(store sdk.KVStore, cdc codec.BinaryCodec) error { - return migrateVotes(store, cdc) -} From e7dd5e0bdc8aabadb0d4ae13bdb143f92c1cdf4c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Sun, 1 Jan 2023 15:56:39 +0100 Subject: [PATCH 2/7] fix: return anteEvents when postHandler fail (backport #14448) (#14459) Co-authored-by: yihuang --- CHANGELOG.md | 4 ++++ baseapp/baseapp.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0784642de46..dedbccb2a812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,10 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (x/gov) [#14347](https://github.com/cosmos/cosmos-sdk/pull/14347) Support `v1.Proposal` message in `v1beta1.Proposal.Content`. +### Bug Fixes + +* (ante) [#14448](https://github.com/cosmos/cosmos-sdk/pull/14448) Return anteEvents when postHandler fail. + ### API Breaking * (x/gov) [#14422](https://github.com/cosmos/cosmos-sdk/pull/14422) Remove `Migrate_V046_6_To_V046_7` function which shouldn't be used for chains which already migrated to 0.46. diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go index a094c1be6c34..78d57763726e 100644 --- a/baseapp/baseapp.go +++ b/baseapp/baseapp.go @@ -729,7 +729,7 @@ func (app *BaseApp) runTx(mode runTxMode, txBytes []byte) (gInfo sdk.GasInfo, re newCtx, err := app.postHandler(postCtx, tx, mode == runTxModeSimulate) if err != nil { - return gInfo, nil, nil, priority, err + return gInfo, nil, anteEvents, priority, err } result.Events = append(result.Events, newCtx.EventManager().ABCIEvents()...) From 93fd49d29ded8fd2f04f2a634957846b74d8d212 Mon Sep 17 00:00:00 2001 From: Marko Date: Thu, 5 Jan 2023 14:42:49 +0100 Subject: [PATCH 3/7] chore: backport pr 13881 (#14349) Co-authored-by: yihuang Co-authored-by: Julien Robert Co-authored-by: Matt Kocubinski --- CHANGELOG.md | 5 +- go.mod | 1 + go.sum | 2 + store/cachekv/benchmark_test.go | 161 ++++++++++++++ store/cachekv/internal/btree.go | 80 +++++++ store/cachekv/internal/btree_test.go | 202 ++++++++++++++++++ store/cachekv/internal/memiterator.go | 137 ++++++++++++ store/cachekv/{ => internal}/mergeiterator.go | 53 +++-- store/cachekv/memiterator.go | 57 ----- store/cachekv/search_benchmark_test.go | 4 +- store/cachekv/store.go | 47 ++-- store/cachekv/store_test.go | 146 +++++++++++++ 12 files changed, 782 insertions(+), 113 deletions(-) create mode 100644 store/cachekv/benchmark_test.go create mode 100644 store/cachekv/internal/btree.go create mode 100644 store/cachekv/internal/btree_test.go create mode 100644 store/cachekv/internal/memiterator.go rename store/cachekv/{ => internal}/mergeiterator.go (86%) delete mode 100644 store/cachekv/memiterator.go diff --git a/CHANGELOG.md b/CHANGELOG.md index dedbccb2a812..21526ee3e3cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Improvements +* [#13881](https://github.com/cosmos/cosmos-sdk/pull/13881) Optimize iteration on nested cached KV stores and other operations in general. * (x/gov) [#14347](https://github.com/cosmos/cosmos-sdk/pull/14347) Support `v1.Proposal` message in `v1beta1.Proposal.Content`. ### Bug Fixes @@ -59,13 +60,12 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (deps) Bump Tendermint version to [v0.34.24](https://github.com/tendermint/tendermint/releases/tag/v0.34.24). * [#13651](https://github.com/cosmos/cosmos-sdk/pull/13651) Update `server/config/config.GetConfig` function. -* [#13781](https://github.com/cosmos/cosmos-sdk/pull/13781) Remove `client/keys.KeysCdc`. * [#14175](https://github.com/cosmos/cosmos-sdk/pull/14175) Add `server.DefaultBaseappOptions(appopts)` function to reduce boiler plate in root.go. ### State Machine Breaking * (x/gov) [#14214](https://github.com/cosmos/cosmos-sdk/pull/14214) Fix gov v0.46 migration to v1 votes. - * Also provide a helper function `govv046.Migrate_V0466_To_V0467` for migrating a chain already on v0.46 with versions <=v0.46.6 to the latest v0.46.7 correct state. + * Also provide a helper function `govv046.Migrate_V0466_To_V0467` for migrating a chain already on v0.46 with versions <=v0.46.6 to the latest v0.46.7 correct state. * (x/group) [#14071](https://github.com/cosmos/cosmos-sdk/pull/14071) Don't re-tally proposal after voting period end if they have been marked as ACCEPTED or REJECTED. ### API Breaking Changes @@ -239,6 +239,7 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 * (x/group) [#12888](https://github.com/cosmos/cosmos-sdk/pull/12888) Fix event propagation to the current context of `x/group` message execution `[]sdk.Result`. * (x/upgrade) [#12906](https://github.com/cosmos/cosmos-sdk/pull/12906) Fix upgrade failure by moving downgrade verification logic after store migration. +* (store) [#12945](https://github.com/cosmos/cosmos-sdk/pull/12945) Fix nil end semantics in store/cachekv/iterator when iterating a dirty cache. ## [v0.46.0](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.0) - 2022-07-26 diff --git a/go.mod b/go.mod index af7a597cccc2..dc789b4e79c2 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/tendermint/go-amino v0.16.0 github.com/tendermint/tendermint v0.34.24 github.com/tendermint/tm-db v0.6.7 + github.com/tidwall/btree v1.5.0 golang.org/x/crypto v0.2.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a diff --git a/go.sum b/go.sum index 1a250e71fa80..354198c92ace 100644 --- a/go.sum +++ b/go.sum @@ -952,6 +952,8 @@ github.com/tendermint/tendermint v0.34.24 h1:879MKKJWYYPJEMMKME+DWUTY4V9f/FBpnZD github.com/tendermint/tendermint v0.34.24/go.mod h1:rXVrl4OYzmIa1I91av3iLv2HS0fGSiucyW9J4aMTpKI= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= +github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= +github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= diff --git a/store/cachekv/benchmark_test.go b/store/cachekv/benchmark_test.go new file mode 100644 index 000000000000..2db62ba5d6c6 --- /dev/null +++ b/store/cachekv/benchmark_test.go @@ -0,0 +1,161 @@ +package cachekv_test + +import ( + fmt "fmt" + "testing" + + "github.com/cosmos/cosmos-sdk/store" + storetypes "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + dbm "github.com/tendermint/tm-db" +) + +func DoBenchmarkDeepContextStack(b *testing.B, depth int) { + begin := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + end := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + key := storetypes.NewKVStoreKey("test") + + db := dbm.NewMemDB() + cms := store.NewCommitMultiStore(db) + cms.MountStoreWithDB(key, storetypes.StoreTypeIAVL, db) + cms.LoadLatestVersion() + ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger()) + + var stack ContextStack + stack.Reset(ctx) + + for i := 0; i < depth; i++ { + stack.Snapshot() + + store := stack.CurrentContext().KVStore(key) + store.Set(begin, []byte("value")) + } + + store := stack.CurrentContext().KVStore(key) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + it := store.Iterator(begin, end) + it.Valid() + it.Key() + it.Value() + it.Next() + it.Close() + } +} + +func BenchmarkDeepContextStack1(b *testing.B) { + DoBenchmarkDeepContextStack(b, 1) +} + +func BenchmarkDeepContextStack3(b *testing.B) { + DoBenchmarkDeepContextStack(b, 3) +} +func BenchmarkDeepContextStack10(b *testing.B) { + DoBenchmarkDeepContextStack(b, 10) +} + +func BenchmarkDeepContextStack13(b *testing.B) { + DoBenchmarkDeepContextStack(b, 13) +} + +// cachedContext is a pair of cache context and its corresponding commit method. +// They are obtained from the return value of `context.CacheContext()`. +type cachedContext struct { + ctx sdk.Context + commit func() +} + +// ContextStack manages the initial context and a stack of cached contexts, +// to support the `StateDB.Snapshot` and `StateDB.RevertToSnapshot` methods. +// +// Copied from an old version of ethermint +type ContextStack struct { + // Context of the initial state before transaction execution. + // It's the context used by `StateDB.CommitedState`. + initialCtx sdk.Context + cachedContexts []cachedContext +} + +// CurrentContext returns the top context of cached stack, +// if the stack is empty, returns the initial context. +func (cs *ContextStack) CurrentContext() sdk.Context { + l := len(cs.cachedContexts) + if l == 0 { + return cs.initialCtx + } + return cs.cachedContexts[l-1].ctx +} + +// Reset sets the initial context and clear the cache context stack. +func (cs *ContextStack) Reset(ctx sdk.Context) { + cs.initialCtx = ctx + if len(cs.cachedContexts) > 0 { + cs.cachedContexts = []cachedContext{} + } +} + +// IsEmpty returns true if the cache context stack is empty. +func (cs *ContextStack) IsEmpty() bool { + return len(cs.cachedContexts) == 0 +} + +// Commit commits all the cached contexts from top to bottom in order and clears the stack by setting an empty slice of cache contexts. +func (cs *ContextStack) Commit() { + // commit in order from top to bottom + for i := len(cs.cachedContexts) - 1; i >= 0; i-- { + if cs.cachedContexts[i].commit == nil { + panic(fmt.Sprintf("commit function at index %d should not be nil", i)) + } else { + cs.cachedContexts[i].commit() + } + } + cs.cachedContexts = []cachedContext{} +} + +// CommitToRevision commit the cache after the target revision, +// to improve efficiency of db operations. +func (cs *ContextStack) CommitToRevision(target int) error { + if target < 0 || target >= len(cs.cachedContexts) { + return fmt.Errorf("snapshot index %d out of bound [%d..%d)", target, 0, len(cs.cachedContexts)) + } + + // commit in order from top to bottom + for i := len(cs.cachedContexts) - 1; i > target; i-- { + if cs.cachedContexts[i].commit == nil { + return fmt.Errorf("commit function at index %d should not be nil", i) + } + cs.cachedContexts[i].commit() + } + cs.cachedContexts = cs.cachedContexts[0 : target+1] + + return nil +} + +// Snapshot pushes a new cached context to the stack, +// and returns the index of it. +func (cs *ContextStack) Snapshot() int { + i := len(cs.cachedContexts) + ctx, commit := cs.CurrentContext().CacheContext() + cs.cachedContexts = append(cs.cachedContexts, cachedContext{ctx: ctx, commit: commit}) + return i +} + +// RevertToSnapshot pops all the cached contexts after the target index (inclusive). +// the target should be snapshot index returned by `Snapshot`. +// This function panics if the index is out of bounds. +func (cs *ContextStack) RevertToSnapshot(target int) { + if target < 0 || target >= len(cs.cachedContexts) { + panic(fmt.Errorf("snapshot index %d out of bound [%d..%d)", target, 0, len(cs.cachedContexts))) + } + cs.cachedContexts = cs.cachedContexts[:target] +} + +// RevertAll discards all the cache contexts. +func (cs *ContextStack) RevertAll() { + if len(cs.cachedContexts) > 0 { + cs.RevertToSnapshot(0) + } +} diff --git a/store/cachekv/internal/btree.go b/store/cachekv/internal/btree.go new file mode 100644 index 000000000000..142f754bbd38 --- /dev/null +++ b/store/cachekv/internal/btree.go @@ -0,0 +1,80 @@ +package internal + +import ( + "bytes" + "errors" + + "github.com/tidwall/btree" +) + +const ( + // The approximate number of items and children per B-tree node. Tuned with benchmarks. + // copied from memdb. + bTreeDegree = 32 +) + +var errKeyEmpty = errors.New("key cannot be empty") + +// BTree implements the sorted cache for cachekv store, +// we don't use MemDB here because cachekv is used extensively in sdk core path, +// we need it to be as fast as possible, while `MemDB` is mainly used as a mocking db in unit tests. +// +// We choose tidwall/btree over google/btree here because it provides API to implement step iterator directly. +type BTree struct { + tree btree.BTreeG[item] +} + +// NewBTree creates a wrapper around `btree.BTreeG`. +func NewBTree() *BTree { + return &BTree{tree: *btree.NewBTreeGOptions(byKeys, btree.Options{ + Degree: bTreeDegree, + // Contract: cachekv store must not be called concurrently + NoLocks: true, + })} +} + +func (bt *BTree) Set(key, value []byte) { + bt.tree.Set(newItem(key, value)) +} + +func (bt *BTree) Get(key []byte) []byte { + i, found := bt.tree.Get(newItem(key, nil)) + if !found { + return nil + } + return i.value +} + +func (bt *BTree) Delete(key []byte) { + bt.tree.Delete(newItem(key, nil)) +} + +func (bt *BTree) Iterator(start, end []byte) (*memIterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return NewMemIterator(start, end, bt, make(map[string]struct{}), true), nil +} + +func (bt *BTree) ReverseIterator(start, end []byte) (*memIterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + return NewMemIterator(start, end, bt, make(map[string]struct{}), false), nil +} + +// item is a btree item with byte slices as keys and values +type item struct { + key []byte + value []byte +} + +// byKeys compares the items by key +func byKeys(a, b item) bool { + return bytes.Compare(a.key, b.key) == -1 +} + +// newItem creates a new pair item. +func newItem(key, value []byte) item { + return item{key: key, value: value} +} diff --git a/store/cachekv/internal/btree_test.go b/store/cachekv/internal/btree_test.go new file mode 100644 index 000000000000..f85a8bbaf109 --- /dev/null +++ b/store/cachekv/internal/btree_test.go @@ -0,0 +1,202 @@ +package internal + +import ( + "testing" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/stretchr/testify/require" +) + +func TestGetSetDelete(t *testing.T) { + db := NewBTree() + + // A nonexistent key should return nil. + value := db.Get([]byte("a")) + require.Nil(t, value) + + // Set and get a value. + db.Set([]byte("a"), []byte{0x01}) + db.Set([]byte("b"), []byte{0x02}) + value = db.Get([]byte("a")) + require.Equal(t, []byte{0x01}, value) + + value = db.Get([]byte("b")) + require.Equal(t, []byte{0x02}, value) + + // Deleting a non-existent value is fine. + db.Delete([]byte("x")) + + // Delete a value. + db.Delete([]byte("a")) + + value = db.Get([]byte("a")) + require.Nil(t, value) + + db.Delete([]byte("b")) + + value = db.Get([]byte("b")) + require.Nil(t, value) +} + +func TestDBIterator(t *testing.T) { + db := NewBTree() + + for i := 0; i < 10; i++ { + if i != 6 { // but skip 6. + db.Set(int642Bytes(int64(i)), []byte{}) + } + } + + // Blank iterator keys should error + _, err := db.ReverseIterator([]byte{}, nil) + require.Equal(t, errKeyEmpty, err) + _, err = db.ReverseIterator(nil, []byte{}) + require.Equal(t, errKeyEmpty, err) + + itr, err := db.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator") + + ritr, err := db.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") + + itr, err = db.Iterator(nil, int642Bytes(0)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator to 0") + + ritr, err = db.ReverseIterator(int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator from 10 (ex)") + + itr, err = db.Iterator(int642Bytes(0), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") + + itr, err = db.Iterator(int642Bytes(1), nil) + require.NoError(t, err) + verifyIterator(t, itr, []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") + + ritr, err = db.ReverseIterator(nil, int642Bytes(10)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10 (ex)") + + ritr, err = db.ReverseIterator(nil, int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9 (ex)") + + ritr, err = db.ReverseIterator(nil, int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8 (ex)") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 6") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5}, "forward iterator from 5 to 7") + + itr, err = db.Iterator(int642Bytes(5), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{5, 7}, "forward iterator from 5 to 8") + + itr, err = db.Iterator(int642Bytes(6), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, itr, []int64(nil), "forward iterator from 6 to 7") + + itr, err = db.Iterator(int642Bytes(6), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 6 to 8") + + itr, err = db.Iterator(int642Bytes(7), int642Bytes(8)) + require.NoError(t, err) + verifyIterator(t, itr, []int64{7}, "forward iterator from 7 to 8") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(5)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{4}, "reverse iterator from 5 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 6 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{5, 4}, "reverse iterator from 7 (ex) to 4") + + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(6)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 6 (ex) to 5") + + ritr, err = db.ReverseIterator(int642Bytes(5), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{5}, "reverse iterator from 7 (ex) to 5") + + ritr, err = db.ReverseIterator(int642Bytes(6), int642Bytes(7)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 7 (ex) to 6") + + ritr, err = db.ReverseIterator(int642Bytes(10), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64(nil), "reverse iterator to 10") + + ritr, err = db.ReverseIterator(int642Bytes(6), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7}, "reverse iterator to 6") + + ritr, err = db.ReverseIterator(int642Bytes(5), nil) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{9, 8, 7, 5}, "reverse iterator to 5") + + ritr, err = db.ReverseIterator(int642Bytes(8), int642Bytes(9)) + require.NoError(t, err) + verifyIterator(t, ritr, []int64{8}, "reverse iterator from 9 (ex) to 8") + + ritr, err = db.ReverseIterator(int642Bytes(2), int642Bytes(4)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64{3, 2}, "reverse iterator from 4 (ex) to 2") + + ritr, err = db.ReverseIterator(int642Bytes(4), int642Bytes(2)) + require.NoError(t, err) + verifyIterator(t, ritr, + []int64(nil), "reverse iterator from 2 (ex) to 4") + + // Ensure that the iterators don't panic with an empty database. + db2 := NewBTree() + + itr, err = db2.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, nil, "forward iterator with empty db") + + ritr, err = db2.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, nil, "reverse iterator with empty db") +} + +func verifyIterator(t *testing.T, itr *memIterator, expected []int64, msg string) { + i := 0 + for itr.Valid() { + key := itr.Key() + require.Equal(t, expected[i], bytes2Int64(key), "iterator: %d mismatches", i) + itr.Next() + i++ + } + require.Equal(t, i, len(expected), "expected to have fully iterated over all the elements in iter") + require.NoError(t, itr.Close()) +} + +func int642Bytes(i int64) []byte { + return sdk.Uint64ToBigEndian(uint64(i)) +} + +func bytes2Int64(buf []byte) int64 { + return int64(sdk.BigEndianToUint64(buf)) +} diff --git a/store/cachekv/internal/memiterator.go b/store/cachekv/internal/memiterator.go new file mode 100644 index 000000000000..2bceb8bc77df --- /dev/null +++ b/store/cachekv/internal/memiterator.go @@ -0,0 +1,137 @@ +package internal + +import ( + "bytes" + "errors" + + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/tidwall/btree" +) + +var _ types.Iterator = (*memIterator)(nil) + +// memIterator iterates over iterKVCache items. +// if key is nil, means it was deleted. +// Implements Iterator. +type memIterator struct { + iter btree.GenericIter[item] + + start []byte + end []byte + ascending bool + lastKey []byte + deleted map[string]struct{} + valid bool +} + +func NewMemIterator(start, end []byte, items *BTree, deleted map[string]struct{}, ascending bool) *memIterator { + iter := items.tree.Iter() + var valid bool + if ascending { + if start != nil { + valid = iter.Seek(newItem(start, nil)) + } else { + valid = iter.First() + } + } else { + if end != nil { + valid = iter.Seek(newItem(end, nil)) + if !valid { + valid = iter.Last() + } else { + // end is exclusive + valid = iter.Prev() + } + } else { + valid = iter.Last() + } + } + + mi := &memIterator{ + iter: iter, + start: start, + end: end, + ascending: ascending, + lastKey: nil, + deleted: deleted, + valid: valid, + } + + if mi.valid { + mi.valid = mi.keyInRange(mi.Key()) + } + + return mi +} + +func (mi *memIterator) Domain() (start []byte, end []byte) { + return mi.start, mi.end +} + +func (mi *memIterator) Close() error { + mi.iter.Release() + return nil +} + +func (mi *memIterator) Error() error { + if !mi.Valid() { + return errors.New("invalid memIterator") + } + return nil +} + +func (mi *memIterator) Valid() bool { + return mi.valid +} + +func (mi *memIterator) Next() { + mi.assertValid() + + if mi.ascending { + mi.valid = mi.iter.Next() + } else { + mi.valid = mi.iter.Prev() + } + + if mi.valid { + mi.valid = mi.keyInRange(mi.Key()) + } +} + +func (mi *memIterator) keyInRange(key []byte) bool { + if mi.ascending && mi.end != nil && bytes.Compare(key, mi.end) >= 0 { + return false + } + if !mi.ascending && mi.start != nil && bytes.Compare(key, mi.start) < 0 { + return false + } + return true +} + +func (mi *memIterator) Key() []byte { + return mi.iter.Item().key +} + +func (mi *memIterator) Value() []byte { + item := mi.iter.Item() + key := item.key + // We need to handle the case where deleted is modified and includes our current key + // We handle this by maintaining a lastKey object in the iterator. + // If the current key is the same as the last key (and last key is not nil / the start) + // then we are calling value on the same thing as last time. + // Therefore we don't check the mi.deleted to see if this key is included in there. + if _, ok := mi.deleted[string(key)]; ok { + if mi.lastKey == nil || !bytes.Equal(key, mi.lastKey) { + // not re-calling on old last key + return nil + } + } + mi.lastKey = key + return item.value +} + +func (mi *memIterator) assertValid() { + if err := mi.Error(); err != nil { + panic(err) + } +} diff --git a/store/cachekv/mergeiterator.go b/store/cachekv/internal/mergeiterator.go similarity index 86% rename from store/cachekv/mergeiterator.go rename to store/cachekv/internal/mergeiterator.go index a6c7a035aba0..4186a178a863 100644 --- a/store/cachekv/mergeiterator.go +++ b/store/cachekv/internal/mergeiterator.go @@ -1,4 +1,4 @@ -package cachekv +package internal import ( "bytes" @@ -18,17 +18,20 @@ type cacheMergeIterator struct { parent types.Iterator cache types.Iterator ascending bool + + valid bool } var _ types.Iterator = (*cacheMergeIterator)(nil) -func newCacheMergeIterator(parent, cache types.Iterator, ascending bool) *cacheMergeIterator { +func NewCacheMergeIterator(parent, cache types.Iterator, ascending bool) *cacheMergeIterator { iter := &cacheMergeIterator{ parent: parent, cache: cache, ascending: ascending, } + iter.valid = iter.skipUntilExistsOrInvalid() return iter } @@ -40,42 +43,38 @@ func (iter *cacheMergeIterator) Domain() (start, end []byte) { // Valid implements Iterator. func (iter *cacheMergeIterator) Valid() bool { - return iter.skipUntilExistsOrInvalid() + return iter.valid } // Next implements Iterator func (iter *cacheMergeIterator) Next() { - iter.skipUntilExistsOrInvalid() iter.assertValid() - // If parent is invalid, get the next cache item. - if !iter.parent.Valid() { + switch { + case !iter.parent.Valid(): + // If parent is invalid, get the next cache item. iter.cache.Next() - return - } - - // If cache is invalid, get the next parent item. - if !iter.cache.Valid() { + case !iter.cache.Valid(): + // If cache is invalid, get the next parent item. iter.parent.Next() - return - } - - // Both are valid. Compare keys. - keyP, keyC := iter.parent.Key(), iter.cache.Key() - switch iter.compare(keyP, keyC) { - case -1: // parent < cache - iter.parent.Next() - case 0: // parent == cache - iter.parent.Next() - iter.cache.Next() - case 1: // parent > cache - iter.cache.Next() + default: + // Both are valid. Compare keys. + keyP, keyC := iter.parent.Key(), iter.cache.Key() + switch iter.compare(keyP, keyC) { + case -1: // parent < cache + iter.parent.Next() + case 0: // parent == cache + iter.parent.Next() + iter.cache.Next() + case 1: // parent > cache + iter.cache.Next() + } } + iter.valid = iter.skipUntilExistsOrInvalid() } // Key implements Iterator func (iter *cacheMergeIterator) Key() []byte { - iter.skipUntilExistsOrInvalid() iter.assertValid() // If parent is invalid, get the cache key. @@ -106,7 +105,6 @@ func (iter *cacheMergeIterator) Key() []byte { // Value implements Iterator func (iter *cacheMergeIterator) Value() []byte { - iter.skipUntilExistsOrInvalid() iter.assertValid() // If parent is invalid, get the cache value. @@ -137,11 +135,12 @@ func (iter *cacheMergeIterator) Value() []byte { // Close implements Iterator func (iter *cacheMergeIterator) Close() error { + err1 := iter.cache.Close() if err := iter.parent.Close(); err != nil { return err } - return iter.cache.Close() + return err1 } // Error returns an error if the cacheMergeIterator is invalid defined by the diff --git a/store/cachekv/memiterator.go b/store/cachekv/memiterator.go deleted file mode 100644 index a12ff9acfd11..000000000000 --- a/store/cachekv/memiterator.go +++ /dev/null @@ -1,57 +0,0 @@ -package cachekv - -import ( - "bytes" - - dbm "github.com/tendermint/tm-db" - - "github.com/cosmos/cosmos-sdk/store/types" -) - -// memIterator iterates over iterKVCache items. -// if key is nil, means it was deleted. -// Implements Iterator. -type memIterator struct { - types.Iterator - - lastKey []byte - deleted map[string]struct{} -} - -func newMemIterator(start, end []byte, items *dbm.MemDB, deleted map[string]struct{}, ascending bool) *memIterator { - var ( - iter types.Iterator - err error - ) - - if ascending { - iter, err = items.Iterator(start, end) - } else { - iter, err = items.ReverseIterator(start, end) - } - - if err != nil { - panic(err) - } - - return &memIterator{ - Iterator: iter, - lastKey: nil, - deleted: deleted, - } -} - -func (mi *memIterator) Value() []byte { - key := mi.Iterator.Key() - // We need to handle the case where deleted is modified and includes our current key - // We handle this by maintaining a lastKey object in the iterator. - // If the current key is the same as the last key (and last key is not nil / the start) - // then we are calling value on the same thing as last time. - // Therefore we don't check the mi.deleted to see if this key is included in there. - reCallingOnOldLastKey := (mi.lastKey != nil) && bytes.Equal(key, mi.lastKey) - if _, ok := mi.deleted[string(key)]; ok && !reCallingOnOldLastKey { - return nil - } - mi.lastKey = key - return mi.Iterator.Value() -} diff --git a/store/cachekv/search_benchmark_test.go b/store/cachekv/search_benchmark_test.go index 921bff4e3864..4007c7cda202 100644 --- a/store/cachekv/search_benchmark_test.go +++ b/store/cachekv/search_benchmark_test.go @@ -4,7 +4,7 @@ import ( "strconv" "testing" - db "github.com/tendermint/tm-db" + "github.com/cosmos/cosmos-sdk/store/cachekv/internal" ) func BenchmarkLargeUnsortedMisses(b *testing.B) { @@ -39,6 +39,6 @@ func generateStore() *Store { return &Store{ cache: cache, unsortedCache: unsorted, - sortedCache: db.NewMemDB(), + sortedCache: internal.NewBTree(), } } diff --git a/store/cachekv/store.go b/store/cachekv/store.go index 0ebc52268548..42354fa78be3 100644 --- a/store/cachekv/store.go +++ b/store/cachekv/store.go @@ -9,10 +9,10 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/internal/conv" + "github.com/cosmos/cosmos-sdk/store/cachekv/internal" "github.com/cosmos/cosmos-sdk/store/tracekv" "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/types/kv" - "github.com/tendermint/tendermint/libs/math" ) // cValue represents a cached value. @@ -30,7 +30,7 @@ type Store struct { cache map[string]*cValue deleted map[string]struct{} unsortedCache map[string]struct{} - sortedCache *dbm.MemDB // always ascending sorted + sortedCache *internal.BTree // always ascending sorted parent types.KVStore } @@ -42,7 +42,7 @@ func NewStore(parent types.KVStore) *Store { cache: make(map[string]*cValue), deleted: make(map[string]struct{}), unsortedCache: make(map[string]struct{}), - sortedCache: dbm.NewMemDB(), + sortedCache: internal.NewBTree(), parent: parent, } } @@ -101,6 +101,11 @@ func (store *Store) Write() { store.mtx.Lock() defer store.mtx.Unlock() + if len(store.cache) == 0 && len(store.deleted) == 0 && len(store.unsortedCache) == 0 { + store.sortedCache = internal.NewBTree() + return + } + // We need a copy of all of the keys. // Not the best, but probably not a bottleneck depending. keys := make([]string, 0, len(store.cache)) @@ -144,7 +149,7 @@ func (store *Store) Write() { for key := range store.unsortedCache { delete(store.unsortedCache, key) } - store.sortedCache = dbm.NewMemDB() + store.sortedCache = internal.NewBTree() } // CacheWrap implements CacheWrapper. @@ -183,9 +188,9 @@ func (store *Store) iterator(start, end []byte, ascending bool) types.Iterator { } store.dirtyItems(start, end) - cache = newMemIterator(start, end, store.sortedCache, store.deleted, ascending) + cache = internal.NewMemIterator(start, end, store.sortedCache, store.deleted, ascending) - return newCacheMergeIterator(parent, cache, ascending) + return internal.NewCacheMergeIterator(parent, cache, ascending) } func findStartIndex(strL []string, startQ string) int { @@ -273,7 +278,7 @@ const minSortSize = 1024 // Constructs a slice of dirty items, to use w/ memIterator. func (store *Store) dirtyItems(start, end []byte) { startStr, endStr := conv.UnsafeBytesToStr(start), conv.UnsafeBytesToStr(end) - if startStr > endStr { + if end != nil && startStr > endStr { // Nothing to do here. return } @@ -288,6 +293,7 @@ func (store *Store) dirtyItems(start, end []byte) { // than just not having the cache. if n < minSortSize { for key := range store.unsortedCache { + // dbm.IsKeyInDomain is nil safe and returns true iff key is greater than start if dbm.IsKeyInDomain(conv.UnsafeStrToBytes(key), start, end) { cacheValue := store.cache[key] unsorted = append(unsorted, &kv.Pair{Key: []byte(key), Value: cacheValue.value}) @@ -308,24 +314,18 @@ func (store *Store) dirtyItems(start, end []byte) { // Now find the values within the domain // [start, end) startIndex := findStartIndex(strL, startStr) - endIndex := findEndIndex(strL, endStr) - - if endIndex < 0 { - endIndex = len(strL) - 1 - } if startIndex < 0 { startIndex = 0 } - // Since we spent cycles to sort the values, we should process and remove a reasonable amount - // ensure start to end is at least minSortSize in size - // if below minSortSize, expand it to cover additional values - // this amortizes the cost of processing elements across multiple calls - if endIndex-startIndex < minSortSize { - endIndex = math.MinInt(startIndex+minSortSize, len(strL)-1) - if endIndex-startIndex < minSortSize { - startIndex = math.MaxInt(endIndex-minSortSize, 0) - } + var endIndex int + if end == nil { + endIndex = len(strL) - 1 + } else { + endIndex = findEndIndex(strL, endStr) + } + if endIndex < 0 { + endIndex = len(strL) - 1 } kvL := make([]*kv.Pair, 0) @@ -364,10 +364,7 @@ func (store *Store) clearUnsortedCacheSubset(unsorted []*kv.Pair, sortState sort store.sortedCache.Set(item.Key, []byte{}) continue } - err := store.sortedCache.Set(item.Key, item.Value) - if err != nil { - panic(err) - } + store.sortedCache.Set(item.Key, item.Value) } } diff --git a/store/cachekv/store_test.go b/store/cachekv/store_test.go index d589932d30fc..3ef99fd6f144 100644 --- a/store/cachekv/store_test.go +++ b/store/cachekv/store_test.go @@ -120,6 +120,7 @@ func TestCacheKVIteratorBounds(t *testing.T) { i++ } require.Equal(t, nItems, i) + require.NoError(t, itr.Close()) // iterate over none itr = st.Iterator(bz("money"), nil) @@ -128,6 +129,7 @@ func TestCacheKVIteratorBounds(t *testing.T) { i++ } require.Equal(t, 0, i) + require.NoError(t, itr.Close()) // iterate over lower itr = st.Iterator(keyFmt(0), keyFmt(3)) @@ -139,6 +141,7 @@ func TestCacheKVIteratorBounds(t *testing.T) { i++ } require.Equal(t, 3, i) + require.NoError(t, itr.Close()) // iterate over upper itr = st.Iterator(keyFmt(2), keyFmt(4)) @@ -150,6 +153,64 @@ func TestCacheKVIteratorBounds(t *testing.T) { i++ } require.Equal(t, 4, i) + require.NoError(t, itr.Close()) +} + +func TestCacheKVReverseIteratorBounds(t *testing.T) { + st := newCacheKVStore() + + // set some items + nItems := 5 + for i := 0; i < nItems; i++ { + st.Set(keyFmt(i), valFmt(i)) + } + + // iterate over all of them + itr := st.ReverseIterator(nil, nil) + i := 0 + for ; itr.Valid(); itr.Next() { + k, v := itr.Key(), itr.Value() + require.Equal(t, keyFmt(nItems-1-i), k) + require.Equal(t, valFmt(nItems-1-i), v) + i++ + } + require.Equal(t, nItems, i) + require.NoError(t, itr.Close()) + + // iterate over none + itr = st.ReverseIterator(bz("money"), nil) + i = 0 + for ; itr.Valid(); itr.Next() { + i++ + } + require.Equal(t, 0, i) + require.NoError(t, itr.Close()) + + // iterate over lower + end := 3 + itr = st.ReverseIterator(keyFmt(0), keyFmt(end)) + i = 0 + for ; itr.Valid(); itr.Next() { + i++ + k, v := itr.Key(), itr.Value() + require.Equal(t, keyFmt(end-i), k) + require.Equal(t, valFmt(end-i), v) + } + require.Equal(t, 3, i) + require.NoError(t, itr.Close()) + + // iterate over upper + end = 4 + itr = st.ReverseIterator(keyFmt(2), keyFmt(end)) + i = 0 + for ; itr.Valid(); itr.Next() { + i++ + k, v := itr.Key(), itr.Value() + require.Equal(t, keyFmt(end-i), k) + require.Equal(t, valFmt(end-i), v) + } + require.Equal(t, 2, i) + require.NoError(t, itr.Close()) } func TestCacheKVMergeIteratorBasics(t *testing.T) { @@ -291,6 +352,25 @@ func TestCacheKVMergeIteratorChunks(t *testing.T) { assertIterateDomainCheck(t, st, truth, []keyRange{{0, 15}, {25, 35}, {38, 40}, {45, 80}}) } +func TestCacheKVMergeIteratorDomain(t *testing.T) { + st := newCacheKVStore() + + itr := st.Iterator(nil, nil) + start, end := itr.Domain() + require.Equal(t, start, end) + require.NoError(t, itr.Close()) + + itr = st.Iterator(keyFmt(40), keyFmt(60)) + start, end = itr.Domain() + require.Equal(t, keyFmt(40), start) + require.Equal(t, keyFmt(60), end) + require.NoError(t, itr.Close()) + + start, end = st.ReverseIterator(keyFmt(0), keyFmt(80)).Domain() + require.Equal(t, keyFmt(0), start) + require.Equal(t, keyFmt(80), end) +} + func TestCacheKVMergeIteratorRandom(t *testing.T) { st := newCacheKVStore() truth := dbm.NewMemDB() @@ -306,6 +386,67 @@ func TestCacheKVMergeIteratorRandom(t *testing.T) { } } +func TestNilEndIterator(t *testing.T) { + const SIZE = 3000 + + tests := []struct { + name string + write bool + startIndex int + end []byte + }{ + {name: "write=false, end=nil", write: false, end: nil, startIndex: 1000}, + {name: "write=false, end=nil; full key scan", write: false, end: nil, startIndex: 2000}, + {name: "write=true, end=nil", write: true, end: nil, startIndex: 1000}, + {name: "write=false, end=non-nil", write: false, end: keyFmt(3000), startIndex: 1000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := newCacheKVStore() + + for i := 0; i < SIZE; i++ { + kstr := keyFmt(i) + st.Set(kstr, valFmt(i)) + } + + if tt.write { + st.Write() + } + + itr := st.Iterator(keyFmt(tt.startIndex), tt.end) + i := tt.startIndex + j := 0 + for itr.Valid() { + require.Equal(t, keyFmt(i), itr.Key()) + require.Equal(t, valFmt(i), itr.Value()) + itr.Next() + i++ + j++ + } + + require.Equal(t, SIZE-tt.startIndex, j) + require.NoError(t, itr.Close()) + }) + } +} + +// TestIteratorDeadlock demonstrate the deadlock issue in cache store. +func TestIteratorDeadlock(t *testing.T) { + mem := dbadapter.Store{DB: dbm.NewMemDB()} + store := cachekv.NewStore(mem) + // the channel buffer is 64 and received once, so put at least 66 elements. + for i := 0; i < 66; i++ { + store.Set([]byte(fmt.Sprintf("key%d", i)), []byte{1}) + } + it := store.Iterator(nil, nil) + defer it.Close() + store.Set([]byte("key20"), []byte{1}) + // it'll be blocked here with previous version, or enable lock on btree. + it2 := store.Iterator(nil, nil) + defer it2.Close() +} + //------------------------------------------------------------------------------------------- // do some random ops @@ -388,6 +529,7 @@ func assertIterateDomain(t *testing.T, st types.KVStore, expectedN int) { i++ } require.Equal(t, expectedN, i) + require.NoError(t, itr.Close()) } func assertIterateDomainCheck(t *testing.T, st types.KVStore, mem dbm.DB, r []keyRange) { @@ -419,6 +561,8 @@ func assertIterateDomainCheck(t *testing.T, st types.KVStore, mem dbm.DB, r []ke require.False(t, itr.Valid()) require.False(t, itr2.Valid()) + require.NoError(t, itr.Close()) + require.NoError(t, itr2.Close()) } func assertIterateDomainCompare(t *testing.T, st types.KVStore, mem dbm.DB) { @@ -428,6 +572,8 @@ func assertIterateDomainCompare(t *testing.T, st types.KVStore, mem dbm.DB) { require.NoError(t, err) checkIterators(t, itr, itr2) checkIterators(t, itr2, itr) + require.NoError(t, itr.Close()) + require.NoError(t, itr2.Close()) } func checkIterators(t *testing.T, itr, itr2 types.Iterator) { From 36784a8a7a933c8f31930ad6302321be375a8509 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Sat, 7 Jan 2023 12:19:18 +0000 Subject: [PATCH 4/7] fix: wrong address set in `EventUpdateGroupPolicy` (backport #14526) (#14528) Co-authored-by: Julien Robert --- CHANGELOG.md | 1 + x/group/keeper/keeper_test.go | 22 ++++++++++++++++++++++ x/group/keeper/msg_server.go | 3 +-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21526ee3e3cb..068f02fb5fc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Bug Fixes +* (x/group) [#14526](https://github.com/cosmos/cosmos-sdk/pull/14526) Fix wrong address set in `EventUpdateGroupPolicy`. * (ante) [#14448](https://github.com/cosmos/cosmos-sdk/pull/14448) Return anteEvents when postHandler fail. ### API Breaking diff --git a/x/group/keeper/keeper_test.go b/x/group/keeper/keeper_test.go index 8664dfecddd1..56a748018965 100644 --- a/x/group/keeper/keeper_test.go +++ b/x/group/keeper/keeper_test.go @@ -438,6 +438,9 @@ func (s *TestSuite) TestUpdateGroupMetadata() { res, err := s.keeper.GroupInfo(ctx, &group.QueryGroupInfoRequest{GroupId: groupID}) s.Require().NoError(err) s.Assert().Equal(spec.expStored, res.Info) + + events := sdkCtx.EventManager().ABCIEvents() + s.Require().Len(events, 1) // EventUpdateGroup }) } } @@ -721,6 +724,9 @@ func (s *TestSuite) TestUpdateGroupMembers() { s.Assert().Equal(spec.expMembers[i].Member.AddedAt, loadedMembers[i].Member.AddedAt) s.Assert().Equal(spec.expMembers[i].GroupId, loadedMembers[i].GroupId) } + + events := sdkCtx.EventManager().ABCIEvents() + s.Require().Len(events, 1) // EventUpdateGroup }) } } @@ -1242,6 +1248,22 @@ func (s *TestSuite) TestUpdateGroupPolicyMetadata() { }) s.Require().NoError(err) s.Assert().Equal(spec.expGroupPolicy, res.Info) + + // check events + var hasUpdateGroupPolicyEvent bool + events := s.ctx.(sdk.Context).EventManager().ABCIEvents() + for _, event := range events { + event, err := sdk.ParseTypedEvent(event) + if err == nil { + if e, ok := event.(*group.EventUpdateGroupPolicy); ok { + s.Require().Equal(e.Address, groupPolicyAddr) + hasUpdateGroupPolicyEvent = true + break + } + } + } + + s.Require().True(hasUpdateGroupPolicyEvent) }) } } diff --git a/x/group/keeper/msg_server.go b/x/group/keeper/msg_server.go index 0023c57a90b4..3947f0adad50 100644 --- a/x/group/keeper/msg_server.go +++ b/x/group/keeper/msg_server.go @@ -911,8 +911,7 @@ func (k Keeper) doUpdateGroupPolicy(ctx sdk.Context, groupPolicy string, admin s return err } - err = ctx.EventManager().EmitTypedEvent(&group.EventUpdateGroupPolicy{Address: admin}) - if err != nil { + if err = ctx.EventManager().EmitTypedEvent(&group.EventUpdateGroupPolicy{Address: groupPolicyInfo.Address}); err != nil { return err } From b7a62feda412df868d93531bb79d21c918a4b4c7 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 16:56:19 +0000 Subject: [PATCH 5/7] build: bump `bgentry/speakasy` to pseudo version (backport #14633) (#14638) Co-authored-by: Julien Robert --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index dc789b4e79c2..316a64d82c89 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( cosmossdk.io/math v1.0.0-beta.3 github.com/99designs/keyring v1.2.1 github.com/armon/go-metrics v0.4.0 - github.com/bgentry/speakeasy v0.1.0 + github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 github.com/btcsuite/btcd v0.22.1 github.com/celestiaorg/smt v0.3.0 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e diff --git a/go.sum b/go.sum index 354198c92ace..eb05fbe37bae 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= -github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 h1:41iFGWnSlI2gVpmOtVTJZNodLdLQLn/KsJqFvXwnd/s= +github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= From 89e956c021edb1ec3e9d496b5efbcd11f3997d27 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 18:52:48 +0100 Subject: [PATCH 6/7] refactor: reduce tendermint deps (backport #14616) (#14663) Co-authored-by: Julien Robert --- crypto/armor.go | 2 +- crypto/armor_test.go | 2 +- crypto/keyring/keyring.go | 2 +- crypto/keys/bcrypt/base64.go | 35 ++++ crypto/keys/bcrypt/bcrypt.go | 291 ++++++++++++++++++++++++++++++ crypto/keys/utils.go | 13 ++ crypto/ledger/ledger_mock.go | 16 +- crypto/ledger/ledger_secp256k1.go | 22 ++- go.mod | 8 +- go.sum | 15 +- 10 files changed, 375 insertions(+), 31 deletions(-) create mode 100644 crypto/keys/bcrypt/base64.go create mode 100644 crypto/keys/bcrypt/bcrypt.go create mode 100644 crypto/keys/utils.go diff --git a/crypto/armor.go b/crypto/armor.go index ff265532d13c..1ad0eefe3e7f 100644 --- a/crypto/armor.go +++ b/crypto/armor.go @@ -6,11 +6,11 @@ import ( "fmt" "io" - "github.com/tendermint/crypto/bcrypt" "github.com/tendermint/tendermint/crypto" "golang.org/x/crypto/openpgp/armor" // nolint: staticcheck "github.com/cosmos/cosmos-sdk/codec/legacy" + "github.com/cosmos/cosmos-sdk/crypto/keys/bcrypt" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/cosmos/cosmos-sdk/crypto/xsalsa20symmetric" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" diff --git a/crypto/armor_test.go b/crypto/armor_test.go index cc278aab6c38..5365d89dd239 100644 --- a/crypto/armor_test.go +++ b/crypto/armor_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/crypto/bcrypt" tmcrypto "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/xsalsa20symmetric" @@ -17,6 +16,7 @@ import ( "github.com/cosmos/cosmos-sdk/crypto" "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/cosmos/cosmos-sdk/crypto/keys/bcrypt" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/cosmos/cosmos-sdk/simapp" diff --git a/crypto/keyring/keyring.go b/crypto/keyring/keyring.go index d34bfe3c0369..8d5c707f1a9c 100644 --- a/crypto/keyring/keyring.go +++ b/crypto/keyring/keyring.go @@ -12,13 +12,13 @@ import ( "github.com/99designs/keyring" "github.com/pkg/errors" - "github.com/tendermint/crypto/bcrypt" tmcrypto "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/client/input" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/crypto" "github.com/cosmos/cosmos-sdk/crypto/hd" + "github.com/cosmos/cosmos-sdk/crypto/keys/bcrypt" "github.com/cosmos/cosmos-sdk/crypto/ledger" "github.com/cosmos/cosmos-sdk/crypto/types" sdk "github.com/cosmos/cosmos-sdk/types" diff --git a/crypto/keys/bcrypt/base64.go b/crypto/keys/bcrypt/base64.go new file mode 100644 index 000000000000..fc3116090818 --- /dev/null +++ b/crypto/keys/bcrypt/base64.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bcrypt + +import "encoding/base64" + +const alphabet = "./ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + +var bcEncoding = base64.NewEncoding(alphabet) + +func base64Encode(src []byte) []byte { + n := bcEncoding.EncodedLen(len(src)) + dst := make([]byte, n) + bcEncoding.Encode(dst, src) + for dst[n-1] == '=' { + n-- + } + return dst[:n] +} + +func base64Decode(src []byte) ([]byte, error) { + numOfEquals := 4 - (len(src) % 4) + for i := 0; i < numOfEquals; i++ { + src = append(src, '=') + } + + dst := make([]byte, bcEncoding.DecodedLen(len(src))) + n, err := bcEncoding.Decode(dst, src) + if err != nil { + return nil, err + } + return dst[:n], nil +} diff --git a/crypto/keys/bcrypt/bcrypt.go b/crypto/keys/bcrypt/bcrypt.go new file mode 100644 index 000000000000..4093f8cdb192 --- /dev/null +++ b/crypto/keys/bcrypt/bcrypt.go @@ -0,0 +1,291 @@ +// MODIFIED BY TENDERMINT TO EXPOSE `salt` in `GenerateFromPassword`. +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bcrypt implements Provos and Mazières's bcrypt adaptive hashing +// algorithm. See http://www.usenix.org/event/usenix99/provos/provos.pdf +package bcrypt + +// The code is a port of Provos and Mazières's C implementation. +import ( + "crypto/subtle" + "errors" + "fmt" + "strconv" + + "golang.org/x/crypto/blowfish" //nolint:staticcheck // used in original https://cs.opensource.google/go/x/crypto/+/master:bcrypt/bcrypt.go +) + +const ( + MinCost int = 4 // the minimum allowable cost as passed in to GenerateFromPassword + MaxCost int = 31 // the maximum allowable cost as passed in to GenerateFromPassword + DefaultCost int = 10 // the cost that will actually be set if a cost below MinCost is passed into GenerateFromPassword +) + +// ErrMismatchedHashAndPassword is returned from CompareHashAndPassword when a password and hash do +// not match. +var ErrMismatchedHashAndPassword = errors.New("crypto/bcrypt: hashedPassword is not the hash of the given password") + +// ErrHashTooShort is returned from CompareHashAndPassword when a hash is too short to +// be a bcrypt hash. +var ErrHashTooShort = errors.New("crypto/bcrypt: hashedSecret too short to be a bcrypted password") + +// The error returned from CompareHashAndPassword when a hash was created with +// a bcrypt algorithm newer than this implementation. +type HashVersionTooNewError byte + +func (hv HashVersionTooNewError) Error() string { + return fmt.Sprintf("crypto/bcrypt: bcrypt algorithm version '%c' requested is newer than current version '%c'", byte(hv), majorVersion) +} + +// The error returned from CompareHashAndPassword when a hash starts with something other than '$' +type InvalidHashPrefixError byte + +func (ih InvalidHashPrefixError) Error() string { + return fmt.Sprintf("crypto/bcrypt: bcrypt hashes must start with '$', but hashedSecret started with '%c'", byte(ih)) +} + +type InvalidCostError int + +func (ic InvalidCostError) Error() string { + return fmt.Sprintf("crypto/bcrypt: cost %d is outside allowed range (%d,%d)", int(ic), MinCost, MaxCost) +} + +const ( + majorVersion = '2' + minorVersion = 'a' + maxSaltSize = 16 + maxCryptedHashSize = 23 + encodedSaltSize = 22 + encodedHashSize = 31 + minHashSize = 59 +) + +// magicCipherData is an IV for the 64 Blowfish encryption calls in +// bcrypt(). It's the string "OrpheanBeholderScryDoubt" in big-endian bytes. +var magicCipherData = []byte{ + 0x4f, 0x72, 0x70, 0x68, + 0x65, 0x61, 0x6e, 0x42, + 0x65, 0x68, 0x6f, 0x6c, + 0x64, 0x65, 0x72, 0x53, + 0x63, 0x72, 0x79, 0x44, + 0x6f, 0x75, 0x62, 0x74, +} + +type hashed struct { + hash []byte + salt []byte + cost int // allowed range is MinCost to MaxCost + major byte + minor byte +} + +// GenerateFromPassword returns the bcrypt hash of the password at the given +// cost. If the cost given is less than MinCost, the cost will be set to +// DefaultCost, instead. Use CompareHashAndPassword, as defined in this package, +// to compare the returned hashed password with its cleartext version. +func GenerateFromPassword(salt []byte, password []byte, cost int) ([]byte, error) { + if len(salt) != maxSaltSize { + return nil, fmt.Errorf("salt len must be %v", maxSaltSize) + } + p, err := newFromPassword(salt, password, cost) + if err != nil { + return nil, err + } + return p.Hash(), nil +} + +// CompareHashAndPassword compares a bcrypt hashed password with its possible +// plaintext equivalent. Returns nil on success, or an error on failure. +func CompareHashAndPassword(hashedPassword, password []byte) error { + p, err := newFromHash(hashedPassword) + if err != nil { + return err + } + + otherHash, err := bcrypt(password, p.cost, p.salt) + if err != nil { + return err + } + + otherP := &hashed{otherHash, p.salt, p.cost, p.major, p.minor} + if subtle.ConstantTimeCompare(p.Hash(), otherP.Hash()) == 1 { + return nil + } + + return ErrMismatchedHashAndPassword +} + +// Cost returns the hashing cost used to create the given hashed +// password. When, in the future, the hashing cost of a password system needs +// to be increased in order to adjust for greater computational power, this +// function allows one to establish which passwords need to be updated. +func Cost(hashedPassword []byte) (int, error) { + p, err := newFromHash(hashedPassword) + if err != nil { + return 0, err + } + return p.cost, nil +} + +func newFromPassword(salt []byte, password []byte, cost int) (*hashed, error) { + if cost < MinCost { + cost = DefaultCost + } + p := new(hashed) + p.major = majorVersion + p.minor = minorVersion + + err := checkCost(cost) + if err != nil { + return nil, err + } + p.cost = cost + + p.salt = base64Encode(salt) + hash, err := bcrypt(password, p.cost, p.salt) + if err != nil { + return nil, err + } + p.hash = hash + return p, err +} + +func newFromHash(hashedSecret []byte) (*hashed, error) { + if len(hashedSecret) < minHashSize { + return nil, ErrHashTooShort + } + p := new(hashed) + n, err := p.decodeVersion(hashedSecret) + if err != nil { + return nil, err + } + hashedSecret = hashedSecret[n:] + n, err = p.decodeCost(hashedSecret) + if err != nil { + return nil, err + } + hashedSecret = hashedSecret[n:] + + // The "+2" is here because we'll have to append at most 2 '=' to the salt + // when base64 decoding it in expensiveBlowfishSetup(). + p.salt = make([]byte, encodedSaltSize, encodedSaltSize+2) + copy(p.salt, hashedSecret[:encodedSaltSize]) + + hashedSecret = hashedSecret[encodedSaltSize:] + p.hash = make([]byte, len(hashedSecret)) + copy(p.hash, hashedSecret) + + return p, nil +} + +func bcrypt(password []byte, cost int, salt []byte) ([]byte, error) { + cipherData := make([]byte, len(magicCipherData)) + copy(cipherData, magicCipherData) + + c, err := expensiveBlowfishSetup(password, uint32(cost), salt) + if err != nil { + return nil, err + } + + for i := 0; i < 24; i += 8 { + for j := 0; j < 64; j++ { + c.Encrypt(cipherData[i:i+8], cipherData[i:i+8]) + } + } + + // Bug compatibility with C bcrypt implementations. We only encode 23 of + // the 24 bytes encrypted. + hsh := base64Encode(cipherData[:maxCryptedHashSize]) + return hsh, nil +} + +func expensiveBlowfishSetup(key []byte, cost uint32, salt []byte) (*blowfish.Cipher, error) { + csalt, err := base64Decode(salt) + if err != nil { + return nil, err + } + + // Bug compatibility with C bcrypt implementations. They use the trailing + // NULL in the key string during expansion. + // We copy the key to prevent changing the underlying array. + ckey := append(key[:len(key):len(key)], 0) //nolint:gocritic // used in original https://cs.opensource.google/go/x/crypto/+/master:bcrypt/bcrypt.go + + c, err := blowfish.NewSaltedCipher(ckey, csalt) + if err != nil { + return nil, err + } + + var i, rounds uint64 + rounds = 1 << cost + for i = 0; i < rounds; i++ { + blowfish.ExpandKey(ckey, c) + blowfish.ExpandKey(csalt, c) + } + + return c, nil +} + +func (p *hashed) Hash() []byte { + arr := make([]byte, 60) + arr[0] = '$' + arr[1] = p.major + n := 2 + if p.minor != 0 { + arr[2] = p.minor + n = 3 + } + arr[n] = '$' + n++ + copy(arr[n:], []byte(fmt.Sprintf("%02d", p.cost))) + n += 2 + arr[n] = '$' + n++ + copy(arr[n:], p.salt) + n += encodedSaltSize + copy(arr[n:], p.hash) + n += encodedHashSize + return arr[:n] +} + +func (p *hashed) decodeVersion(sbytes []byte) (int, error) { + if sbytes[0] != '$' { + return -1, InvalidHashPrefixError(sbytes[0]) + } + if sbytes[1] > majorVersion { + return -1, HashVersionTooNewError(sbytes[1]) + } + p.major = sbytes[1] + n := 3 + if sbytes[2] != '$' { + p.minor = sbytes[2] + n++ + } + return n, nil +} + +// sbytes should begin where decodeVersion left off. +func (p *hashed) decodeCost(sbytes []byte) (int, error) { + cost, err := strconv.Atoi(string(sbytes[0:2])) + if err != nil { + return -1, err + } + err = checkCost(cost) + if err != nil { + return -1, err + } + p.cost = cost + return 3, nil +} + +func (p *hashed) String() string { + return fmt.Sprintf("&{hash: %#v, salt: %#v, cost: %d, major: %c, minor: %c}", string(p.hash), p.salt, p.cost, p.major, p.minor) +} + +func checkCost(cost int) error { + if cost < MinCost || cost > MaxCost { + return InvalidCostError(cost) + } + return nil +} diff --git a/crypto/keys/utils.go b/crypto/keys/utils.go new file mode 100644 index 000000000000..2b81337d33c8 --- /dev/null +++ b/crypto/keys/utils.go @@ -0,0 +1,13 @@ +package keys + +import ( + "math/big" + + "github.com/cosmos/cosmos-sdk/crypto/keys/internal/ecdsa" +) + +// Replicates https://github.com/cosmos/cosmos-sdk/blob/44fbb0df9cea049d588e76bf930177d777552cf3/crypto/ledger/ledger_secp256k1.go#L228 +// DO NOT USE. This is a temporary workaround that is cleaned-up in v0.47+ +func IsOverHalfOrder(sigS *big.Int) bool { + return !ecdsa.IsSNormalized(sigS) +} diff --git a/crypto/ledger/ledger_mock.go b/crypto/ledger/ledger_mock.go index 740a773d12ce..97c6f65f43bb 100644 --- a/crypto/ledger/ledger_mock.go +++ b/crypto/ledger/ledger_mock.go @@ -10,7 +10,6 @@ import ( "github.com/pkg/errors" "github.com/cosmos/go-bip39" - secp256k1 "github.com/tendermint/btcd/btcec" "github.com/tendermint/tendermint/crypto" "github.com/cosmos/cosmos-sdk/crypto/hd" @@ -38,11 +37,11 @@ func (mock LedgerSECP256K1Mock) Close() error { // as per the original API, it returns an uncompressed key func (mock LedgerSECP256K1Mock) GetPublicKeySECP256K1(derivationPath []uint32) ([]byte, error) { if derivationPath[0] != 44 { - return nil, errors.New("Invalid derivation path") + return nil, errors.New("invalid derivation path") } if derivationPath[1] != sdk.GetConfig().GetCoinType() { - return nil, errors.New("Invalid derivation path") + return nil, errors.New("invalid derivation path") } seed, err := bip39.NewSeedWithErrorChecking(testdata.TestMnemonic, "") @@ -57,7 +56,7 @@ func (mock LedgerSECP256K1Mock) GetPublicKeySECP256K1(derivationPath []uint32) ( return nil, err } - _, pubkeyObject := secp256k1.PrivKeyFromBytes(secp256k1.S256(), derivedPriv[:]) + _, pubkeyObject := btcec.PrivKeyFromBytes(btcec.S256(), derivedPriv) return pubkeyObject.SerializeUncompressed(), nil } @@ -71,7 +70,7 @@ func (mock LedgerSECP256K1Mock) GetAddressPubKeySECP256K1(derivationPath []uint3 } // re-serialize in the 33-byte compressed format - cmp, err := btcec.ParsePubKey(pk[:], btcec.S256()) + cmp, err := btcec.ParsePubKey(pk, btcec.S256()) if err != nil { return nil, "", fmt.Errorf("error parsing public key: %v", err) } @@ -98,16 +97,13 @@ func (mock LedgerSECP256K1Mock) SignSECP256K1(derivationPath []uint32, message [ return nil, err } - priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), derivedPriv[:]) - + priv, _ := btcec.PrivKeyFromBytes(btcec.S256(), derivedPriv) sig, err := priv.Sign(crypto.Sha256(message)) if err != nil { return nil, err } - // Need to return DER as the ledger does - sig2 := btcec.Signature{R: sig.R, S: sig.S} - return sig2.Serialize(), nil + return sig.Serialize(), nil } // ShowAddressSECP256K1 shows the address for the corresponding bip32 derivation path diff --git a/crypto/ledger/ledger_secp256k1.go b/crypto/ledger/ledger_secp256k1.go index dffded571ab5..fc862c2d08fc 100644 --- a/crypto/ledger/ledger_secp256k1.go +++ b/crypto/ledger/ledger_secp256k1.go @@ -2,14 +2,14 @@ package ledger import ( "fmt" + "math/big" "os" "github.com/btcsuite/btcd/btcec" "github.com/pkg/errors" - tmbtcec "github.com/tendermint/btcd/btcec" - "github.com/cosmos/cosmos-sdk/crypto/hd" + "github.com/cosmos/cosmos-sdk/crypto/keys" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" "github.com/cosmos/cosmos-sdk/crypto/types" ) @@ -173,8 +173,22 @@ func convertDERtoBER(signatureDER []byte) ([]byte, error) { if err != nil { return nil, err } - sigBER := tmbtcec.Signature{R: sigDER.R, S: sigDER.S} - return sigBER.Serialize(), nil + + // based on https://github.com/tendermint/btcd/blob/ec996c5/btcec/signature.go#L33-L50 + // low 'S' malleability breaker + sigS := sigDER.S + if keys.IsOverHalfOrder(sigS) { + sigS = new(big.Int).Sub(btcec.S256().N, sigS) + } + + rBytes := sigDER.R.Bytes() + sBytes := sigS.Bytes() + sigBytes := make([]byte, 64) + // 0 pad the byte arrays from the left if they aren't big enough. + copy(sigBytes[32-len(rBytes):32], rBytes) + copy(sigBytes[64-len(sBytes):64], sBytes) + + return sigBytes, nil } func getDevice() (SECP256K1, error) { diff --git a/go.mod b/go.mod index 316a64d82c89..5ca8411a7bc2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/99designs/keyring v1.2.1 github.com/armon/go-metrics v0.4.0 github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 - github.com/btcsuite/btcd v0.22.1 + github.com/btcsuite/btcd v0.22.2 github.com/celestiaorg/smt v0.3.0 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/cockroachdb/apd/v2 v2.0.2 @@ -19,7 +19,7 @@ require ( github.com/cosmos/cosmos-sdk/db v1.0.0-beta.1 github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.19.4 - github.com/cosmos/ledger-cosmos-go v0.12.1 + github.com/cosmos/ledger-cosmos-go v0.12.2 github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 @@ -48,8 +48,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.8.1 - github.com/tendermint/btcd v0.1.1 - github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 github.com/tendermint/go-amino v0.16.0 github.com/tendermint/tendermint v0.34.24 github.com/tendermint/tm-db v0.6.7 @@ -141,7 +139,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/ulikunitz/xz v0.5.8 // indirect github.com/zondax/hid v0.9.1 // indirect - github.com/zondax/ledger-go v0.14.0 // indirect + github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.2.0 // indirect diff --git a/go.sum b/go.sum index eb05fbe37bae..4e3ac700676a 100644 --- a/go.sum +++ b/go.sum @@ -151,8 +151,9 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv4JPQ3/M+teHz9Bo5jBpkNcP0x6r7rdihlNL/7tTAs= -github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd v0.22.2 h1:vBZ+lGGd1XubpOWO67ITJpAEsICWhA0YzqkcpkgNBfo= +github.com/btcsuite/btcd v0.22.2/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -243,8 +244,8 @@ github.com/cosmos/iavl v0.19.4 h1:t82sN+Y0WeqxDLJRSpNd8YFX5URIrT+p8n6oJbJ2Dok= github.com/cosmos/iavl v0.19.4/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76 h1:DdzS1m6o/pCqeZ8VOAit/gyATedRgjvkVI+UCrLpyuU= github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76/go.mod h1:0mkLWIoZuQ7uBoospo5Q9zIpqq6rYCPJDSUdeCJvPM8= -github.com/cosmos/ledger-cosmos-go v0.12.1 h1:sMBxza5p/rNK/06nBSNmsI/WDqI0pVJFVNihy1Y984w= -github.com/cosmos/ledger-cosmos-go v0.12.1/go.mod h1:dhO6kj+Y+AHIOgAe4L9HL/6NDdyyth4q238I9yFpD2g= +github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= +github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -943,10 +944,6 @@ github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tendermint/btcd v0.1.1 h1:0VcxPfflS2zZ3RiOAHkBiFUcPvbtRj5O7zHmcJWHV7s= -github.com/tendermint/btcd v0.1.1/go.mod h1:DC6/m53jtQzr/NFmMNEu0rxf18/ktVoVtMrnDD5pN+U= -github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 h1:hqAk8riJvK4RMWx1aInLzndwxKalgi5rTqgfXxOxbEI= -github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tendermint v0.34.24 h1:879MKKJWYYPJEMMKME+DWUTY4V9f/FBpnZDI82ky+4k= @@ -996,8 +993,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= -github.com/zondax/ledger-go v0.14.0 h1:dlMC7aO8Wss1CxBq2I96kZ69Nh1ligzbs8UWOtq/AsA= -github.com/zondax/ledger-go v0.14.0/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= +github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c= +github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= From 3a051e1f32bab7ac6a92e269d19c23202fb44e61 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Mon, 23 Jan 2023 19:52:48 +0100 Subject: [PATCH 7/7] chore: prepare v0.46.8 release (#14624) --- CHANGELOG.md | 3 +++ RELEASE_NOTES.md | 17 ++++++----------- go.mod | 2 ++ go.sum | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 068f02fb5fc1..bf9de76481cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,10 +37,13 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +## [v0.46.8](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.8) - 2022-01-23 + ### Improvements * [#13881](https://github.com/cosmos/cosmos-sdk/pull/13881) Optimize iteration on nested cached KV stores and other operations in general. * (x/gov) [#14347](https://github.com/cosmos/cosmos-sdk/pull/14347) Support `v1.Proposal` message in `v1beta1.Proposal.Content`. +* (deps) Use Informal System fork of Tendermint version to [v0.34.24](https://github.com/informalsystems/tendermint/releases/tag/v0.34.24). ### Bug Fixes diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cabe154c0b82..16bfb4e59c67 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,14 +1,13 @@ -# Cosmos SDK v0.46.7 Release Notes +# Cosmos SDK v0.46.8 Release Notes -This release introduces bug fixes and improvements. Notably, the upgrade to Tendermint [v0.34.24](https://github.com/tendermint/tendermint/releases/tag/v0.34.24). +This release introduces bug fixes and improvements. Notably, the SDK have now switched to Informal Systems' Tendermint fork. +Their fork has no changes compared to the upstream Tendermint, but it is now [maintained by Informal Systems](https://twitter.com/informalinc/status/1613580954383040512). Chains are invited to do the same. -Please read the release notes of [v0.46.5](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.46.5) if you are upgrading from `<=0.46.4`. +Moreover, this release contains a store fix. The changes have been tested against a v0.46.x chain mainnet with no issues. However, there is a low probability of an edge case happening. Hence, it is recommended to do a **coordinated upgrade** to avoid any issues. -A critical vulnerability has been fixed in the group module. For safety, `v0.46.5` and `v0.46.6` are retracted, even though chains not using the group module are not affected. When using the group module, please upgrade immediately to `v0.46.7`. +Please see the [CHANGELOG](https://github.com/cosmos/cosmos-sdk/blob/release/v0.46.x/CHANGELOG.md) for an exhaustive list of changes. -An issue has been discovered in the gov module's votes migration. It does not impact proposals and votes tallying, but the gRPC queries on votes are incorrect. This issue is fixed in `v0.46.7`, however: -- if your chain is already on v0.46 using `<= v0.46.6`, a **coordinated upgrade** to v0.46.7 is required. You can use the helper function `govv046.Migrate_V046_6_To_V046_7` for migrating a chain **already on v0.46 with versions <=v0.46.6** to the latest v0.46.7 correct state. -- if your chain is on a previous version <= v0.45, then simply use v0.46.7 when upgrading to v0.46. +Full Commit History: https://github.com/cosmos/cosmos-sdk/compare/v0.46.7...v0.46.8 **NOTE**: The changes mentioned in `v0.46.3` are no longer required. The following replace directive can be removed from the chains. @@ -18,7 +17,3 @@ replace github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8 ``` Instead, `github.com/confio/ics23/go` must be **bumped to `v0.9.0`**. - -Please see the [CHANGELOG](https://github.com/cosmos/cosmos-sdk/blob/release/v0.46.x/CHANGELOG.md) for an exhaustive list of changes. - -Full Commit History: https://github.com/cosmos/cosmos-sdk/compare/v0.46.6...v0.46.7 diff --git a/go.mod b/go.mod index 5ca8411a7bc2..73cbb494d468 100644 --- a/go.mod +++ b/go.mod @@ -164,6 +164,8 @@ replace ( github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/jhump/protoreflect => github.com/jhump/protoreflect v1.9.0 + // use informal system fork of tendermint + github.com/tendermint/tendermint => github.com/informalsystems/tendermint v0.34.24 ) retract ( diff --git a/go.sum b/go.sum index 4e3ac700676a..185ddb5d7f2e 100644 --- a/go.sum +++ b/go.sum @@ -596,6 +596,8 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/informalsystems/tendermint v0.34.24 h1:2beNEg5tp+U5oj/Md+0xDBsMHGbdue31T3OrstS6xS0= +github.com/informalsystems/tendermint v0.34.24/go.mod h1:rXVrl4OYzmIa1I91av3iLv2HS0fGSiucyW9J4aMTpKI= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -946,8 +948,6 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= -github.com/tendermint/tendermint v0.34.24 h1:879MKKJWYYPJEMMKME+DWUTY4V9f/FBpnZDI82ky+4k= -github.com/tendermint/tendermint v0.34.24/go.mod h1:rXVrl4OYzmIa1I91av3iLv2HS0fGSiucyW9J4aMTpKI= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ=