From 5fdfa0a58a6a71492c487f118739fcf55117b6f6 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 01:57:12 +0200 Subject: [PATCH 01/14] WIP WIP --- go.mod | 28 +++--- go.sum | 56 ++++++------ pkg/protocol/engine/engine.go | 1 + .../performance/testsuite_test.go | 6 +- pkg/storage/permanent/settings.go | 70 ++++++++++----- pkg/storage/prunable/bucket_manager.go | 18 ++-- pkg/storage/prunable/epochstore/epoch_kv.go | 49 +++++++++-- pkg/storage/prunable/epochstore/store.go | 67 ++++++++++++--- pkg/storage/prunable/prunable.go | 86 ++++++++++--------- pkg/storage/prunable/prunable_epoch.go | 1 - pkg/storage/storage_prunable.go | 66 ++++++++++---- tools/evil-spammer/go.mod | 28 +++--- tools/evil-spammer/go.sum | 56 ++++++------ tools/gendoc/go.mod | 32 +++---- tools/gendoc/go.sum | 56 ++++++------ tools/genesis-snapshot/go.mod | 27 +++--- tools/genesis-snapshot/go.sum | 57 ++++++------ 17 files changed, 432 insertions(+), 272 deletions(-) diff --git a/go.mod b/go.mod index cfcd26bfe..87ac97ae1 100644 --- a/go.mod +++ b/go.mod @@ -10,19 +10,20 @@ require ( github.com/google/uuid v1.3.1 github.com/gorilla/websocket v1.5.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 - github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b - github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b - github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b + github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af github.com/iotaledger/inx/go v1.0.0-rc.2.0.20231020151337-569450d5bf7d github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd @@ -88,7 +89,6 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 // indirect - github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/go.sum b/go.sum index 39dbfb496..fdc3b1650 100644 --- a/go.sum +++ b/go.sum @@ -277,34 +277,34 @@ github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 h1:dTrD7X2PTNgli6EbS4tV9qu3QAm/kBU3XaYZV2xdzys= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7/go.mod h1:ZRdPu684P0fQ1z8sXz4dj9H5LWHhz4a9oCtvjunkSrw= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b h1:D68khiAFb9DwTvjZc2nc4R0E6wUdKyYCUXkmdaMzuoQ= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b h1:mX3NXaTMLEwZnEs4IlxEvXY0YZo8qbb8M1xM39FS6qY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b h1:HF4e0wz0JMIT4m3saqdQ//T9nWHV9d5sLMtEwNDuykM= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b h1:ZERXxhQBUBV1AqTE6cUI4vTxSx4JrnsMuLZFgj32xLM= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b h1:ZUUqRRO6XnQmVcXlXyx07vqySn28+bln6jp9KagYCjY= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b h1:8b2sH+2Vf0y5BDYTMwKa09iQr3JF9JrzTI64DkXb+9U= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b h1:JJPnr231djUTgTnE4oGz847WE9VA7Py6E6fgZwT5TQo= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b h1:LusmtjpfG/q8lc15Fp9W3kABbN3tArKx/zw2ibdY1DU= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b h1:UvFWI8wQJS/XQOeWHpPsaFVeS2nxJ7nIGFr+IFjrnVw= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 h1:6HIBNKOWLXY5Gk1s+q93D6IsTcqwANigfMO3ePtV3HQ= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b h1:EhVgAU/f2J3VYZwP60dRdyfAeDU3c/gBzX9blKtQGKA= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b h1:O68POYIqBLnoHN+HIszc58QwAI2qocYq0WKGfVrXmMg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b h1:zaXZn9yV/95SRDkgCZQeBbSbmcJTKSZbCB7oBd71Qwg= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b h1:MDZhTZTVDiydXcW5j4TA7HixVCyAdToIMPhHfJee7cE= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 h1:EOfxTuAiBmED1VHuVh7/UIeB27cCRe13gdSzyioNMBw= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 h1:xAER9M9Uoz2EOWT43E9wmXRe+RmAk8OBSUoboH4Su8M= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 h1:drmpgLlJy7kZ09Dt1qKSnbILU+27Qu2jp4VdPDNwbFk= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 h1:BC5GkIHyXdoJGdw6Tu5ds2kjw9grFLtwQiuMaKfdLU8= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 h1:r8TkdQJB7/bJd8cF8z5GQ+rX/7JpbPdPoN7wMoV1OCM= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 h1:ytzZZPtclAzLfjxv26frbinCGx3Z6ouUENbx5U7lFGg= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 h1:QMxd32Y/veVhTDPCiOFgetjUbG7sr9MryF29/rSPkMA= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 h1:/xPwStUckZ2V0XPoY496cXU+c5elpHyvYoT6JAmuvRY= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 h1:AvNLzONVMspwx7nD/NyYUgb5Hi7/zgzIOegr1uRD/M8= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 h1:e1uJAlXE3zeXpa+c4uFOG+/AMFbUlLt2mcrSK5NMxVs= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 h1:7wjs4t1snBDJ8LOTl+tZhr2ORywSOTgJMppxiIAMA0A= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 h1:1QMJ39qXIx/IZVzus3+97IV7Pa++e+d340TvbMjhiBU= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 h1:hZli4E9kJUAEQ7gzZR1XbPcpgqvqMPYq8YBPMbrBuos= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 h1:OlDhgvJ48bZxcvTeebJ1b96xtNnJAddejd2Q4rlH1mU= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af h1:3+heabXNGjv3Sx1sg0qG+QdqeEWIpf7jM3T5cPiVmSA= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af/go.mod h1:3BZ5r/MhswUZwXHDzMZacOhm6DWpUdGjefNRNshfwiM= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20231020151337-569450d5bf7d h1:PeJNFBkR28hdQCxUyC/cbOlDjFm3D1U7O5bxi+qsOvQ= diff --git a/pkg/protocol/engine/engine.go b/pkg/protocol/engine/engine.go index a3ea1b44c..7733ba618 100644 --- a/pkg/protocol/engine/engine.go +++ b/pkg/protocol/engine/engine.go @@ -216,6 +216,7 @@ func New( } } }, + (*Engine).Reset, (*Engine).TriggerInitialized, ) } diff --git a/pkg/protocol/sybilprotection/sybilprotectionv1/performance/testsuite_test.go b/pkg/protocol/sybilprotection/sybilprotectionv1/performance/testsuite_test.go index 9ce7a9f9a..08cbc3766 100644 --- a/pkg/protocol/sybilprotection/sybilprotectionv1/performance/testsuite_test.go +++ b/pkg/protocol/sybilprotection/sybilprotectionv1/performance/testsuite_test.go @@ -70,9 +70,9 @@ func (t *TestSuite) InitPerformanceTracker() { return p, nil } - rewardsStore := epochstore.NewEpochKVStore(kvstore.Realm{}, kvstore.Realm{}, mapdb.NewMapDB(), 0) - poolStatsStore := epochstore.NewStore(kvstore.Realm{}, kvstore.Realm{}, mapdb.NewMapDB(), 0, (*model.PoolsStats).Bytes, model.PoolsStatsFromBytes) - committeeStore := epochstore.NewStore(kvstore.Realm{}, kvstore.Realm{}, mapdb.NewMapDB(), 0, (*account.Accounts).Bytes, account.AccountsFromBytes) + rewardsStore := epochstore.NewEpochKVStore(kvstore.Realm{}, mapdb.NewMapDB(), 0) + poolStatsStore := epochstore.NewStore(kvstore.Realm{}, mapdb.NewMapDB(), 0, (*model.PoolsStats).Bytes, model.PoolsStatsFromBytes) + committeeStore := epochstore.NewStore(kvstore.Realm{}, mapdb.NewMapDB(), 0, (*account.Accounts).Bytes, account.AccountsFromBytes) t.Instance = NewTracker(rewardsStore.GetEpoch, poolStatsStore, committeeStore, performanceFactorFunc, t.latestCommittedEpoch, api.SingleVersionProvider(t.api), func(err error) {}) } diff --git a/pkg/storage/permanent/settings.go b/pkg/storage/permanent/settings.go index d1ad5fe82..60f175b2c 100644 --- a/pkg/storage/permanent/settings.go +++ b/pkg/storage/permanent/settings.go @@ -22,6 +22,7 @@ const ( snapshotImportedKey = iota latestCommitmentKey latestFinalizedSlotKey + latestStoredSlotKey protocolVersionEpochMappingKey futureProtocolParametersKey protocolParametersKey @@ -34,6 +35,7 @@ type Settings struct { storeSnapshotImported *kvstore.TypedValue[bool] storeLatestCommitment *kvstore.TypedValue[*model.Commitment] storeLatestFinalizedSlot *kvstore.TypedValue[iotago.SlotIndex] + storeLatestProcessedSlot *kvstore.TypedValue[iotago.SlotIndex] storeLatestIssuedValidationBlock *kvstore.TypedValue[*model.Block] storeProtocolVersionEpochMapping *kvstore.TypedStore[iotago.Version, iotago.EpochIndex] storeFutureProtocolParameters *kvstore.TypedStore[iotago.Version, *types.Tuple[iotago.EpochIndex, iotago.Identifier]] @@ -83,6 +85,12 @@ func NewSettings(store kvstore.KVStore, opts ...options.Option[api.EpochBasedPro iotago.SlotIndex.Bytes, iotago.SlotIndexFromBytes, ), + storeLatestProcessedSlot: kvstore.NewTypedValue( + store, + []byte{latestStoredSlotKey}, + iotago.SlotIndex.Bytes, + iotago.SlotIndexFromBytes, + ), storeLatestIssuedValidationBlock: kvstore.NewTypedValue( store, []byte{latestIssuedValidationBlock}, @@ -285,7 +293,7 @@ func (s *Settings) LatestFinalizedSlot() iotago.SlotIndex { s.mutex.RLock() defer s.mutex.RUnlock() - return s.latestFinalizedSlot() + return read(s.storeLatestFinalizedSlot) } func (s *Settings) SetLatestFinalizedSlot(slot iotago.SlotIndex) (err error) { @@ -295,23 +303,42 @@ func (s *Settings) SetLatestFinalizedSlot(slot iotago.SlotIndex) (err error) { return s.storeLatestFinalizedSlot.Set(slot) } -func (s *Settings) latestFinalizedSlot() iotago.SlotIndex { - latestFinalizedSlot, err := s.storeLatestFinalizedSlot.Get() - if err != nil { - if ierrors.Is(err, kvstore.ErrKeyNotFound) { - return 0 +func (s *Settings) LatestStoredSlot() iotago.SlotIndex { + s.mutex.RLock() + defer s.mutex.RUnlock() + + return read(s.storeLatestProcessedSlot) +} + +func (s *Settings) SetLatestStoredSlot(slot iotago.SlotIndex) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.storeLatestProcessedSlot.Set(slot) +} + +func (s *Settings) AdvanceLatestStoredSlot(slot iotago.SlotIndex) (err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + if _, err = s.storeLatestProcessedSlot.Compute(func(latestStoredSlot iotago.SlotIndex, _ bool) (newValue iotago.SlotIndex, err error) { + if latestStoredSlot >= slot { + return latestStoredSlot, errLatestStoredSlotUnchanged } - panic(err) + + return slot, nil + }); err != nil && !ierrors.Is(err, errLatestStoredSlotUnchanged) { + return ierrors.Wrap(err, "failed to advance latest stored slot") } - return latestFinalizedSlot + return nil } func (s *Settings) LatestIssuedValidationBlock() *model.Block { s.mutex.RLock() defer s.mutex.RUnlock() - return s.latestIssuedValidationBlock() + return read(s.storeLatestIssuedValidationBlock) } func (s *Settings) SetLatestIssuedValidationBlock(block *model.Block) (err error) { @@ -321,18 +348,6 @@ func (s *Settings) SetLatestIssuedValidationBlock(block *model.Block) (err error return s.storeLatestIssuedValidationBlock.Set(block) } -func (s *Settings) latestIssuedValidationBlock() *model.Block { - block, err := s.storeLatestIssuedValidationBlock.Get() - if err != nil { - if ierrors.Is(err, kvstore.ErrKeyNotFound) { - return nil - } - panic(err) - } - - return block -} - func (s *Settings) Export(writer io.WriteSeeker, targetCommitment *iotago.Commitment) error { var commitmentBytes []byte var err error @@ -579,7 +594,7 @@ func (s *Settings) String() string { builder := stringify.NewStructBuilder("Settings") builder.AddField(stringify.NewStructField("IsSnapshotImported", lo.PanicOnErr(s.storeSnapshotImported.Has()))) builder.AddField(stringify.NewStructField("LatestCommitment", s.latestCommitment())) - builder.AddField(stringify.NewStructField("LatestFinalizedSlot", s.latestFinalizedSlot())) + builder.AddField(stringify.NewStructField("LatestFinalizedSlot", read(s.storeLatestFinalizedSlot))) if err := s.storeProtocolParameters.Iterate(kvstore.EmptyPrefix, func(version iotago.Version, protocolParams iotago.ProtocolParameters) bool { builder.AddField(stringify.NewStructField(fmt.Sprintf("ProtocolParameters(%d)", version), protocolParams)) @@ -590,3 +605,14 @@ func (s *Settings) String() string { return builder.String() } + +func read[T any](typedValue *kvstore.TypedValue[T]) (value T) { + value, err := typedValue.Get() + if err != nil && !ierrors.Is(err, kvstore.ErrKeyNotFound) { + panic(err) + } + + return value +} + +var errLatestStoredSlotUnchanged = ierrors.New("latest stored slot did not change") diff --git a/pkg/storage/prunable/bucket_manager.go b/pkg/storage/prunable/bucket_manager.go index 980d9d6b6..907fd054a 100644 --- a/pkg/storage/prunable/bucket_manager.go +++ b/pkg/storage/prunable/bucket_manager.go @@ -236,18 +236,22 @@ func (b *BucketManager) DeleteBucket(epoch iotago.EpochIndex) (deleted bool) { return true } -// RollbackBucket removes data in the bucket in slots [targetSlotIndex+1; epochEndSlot]. -func (b *BucketManager) RollbackBucket(epoch iotago.EpochIndex, targetSlot, epochEndSlot iotago.SlotIndex) error { - oldBucketKvStore := b.getDBInstance(epoch).KVStore() - for clearSlot := targetSlot + 1; clearSlot <= epochEndSlot; clearSlot++ { - // delete slot prefix from forkedPrunable storage that will be eventually copied into the new engine - if err := oldBucketKvStore.DeletePrefix(clearSlot.MustBytes()); err != nil { - return ierrors.Wrapf(err, "error while clearing slot %d in bucket for epoch %d", clearSlot, epoch) +// PruneSlots prunes the data of all slots in the range [from, to] in the given epoch. +func (b *BucketManager) PruneSlots(epoch iotago.EpochIndex, pruningRange [2]iotago.SlotIndex) error { + epochStore := b.getDBInstance(epoch).KVStore() + + for slot := pruningRange[0]; slot <= pruningRange[1]; slot++ { + if err := epochStore.DeletePrefix(slot.MustBytes()); err != nil { + return ierrors.Wrapf(err, "error while clearing slot %d in bucket for epoch %d", slot, epoch) } } + // shutting down the storage does not prevent this storage from being used again and only forces a flush. + b.Shutdown() + return nil } + func (b *BucketManager) Flush() error { b.openDBsMutex.RLock() defer b.openDBsMutex.RUnlock() diff --git a/pkg/storage/prunable/epochstore/epoch_kv.go b/pkg/storage/prunable/epochstore/epoch_kv.go index 3fd580eb7..014bcacc4 100644 --- a/pkg/storage/prunable/epochstore/epoch_kv.go +++ b/pkg/storage/prunable/epochstore/epoch_kv.go @@ -14,16 +14,18 @@ type EpochKVStore struct { kv kvstore.KVStore pruningDelay iotago.EpochIndex - lastPrunedEpoch *model.PruningIndex + lastAccessedEpoch *kvstore.TypedValue[iotago.EpochIndex] + lastPrunedEpoch *model.PruningIndex } -func NewEpochKVStore(storeRealm, pruningRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay iotago.EpochIndex) *EpochKVStore { +func NewEpochKVStore(storeRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay iotago.EpochIndex) *EpochKVStore { return &EpochKVStore{ - realm: storeRealm, - kv: lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), - pruningDelay: pruningDelay, - lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(pruningRealm)), storeRealm), + realm: storeRealm, + kv: lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), + pruningDelay: pruningDelay, + lastAccessedEpoch: kvstore.NewTypedValue(kv, kvstore.Realm{lastAccessedEpochKey}, iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes), + lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(kvstore.Realm{lastPrunedEpochKey})), storeRealm), } } @@ -37,11 +39,31 @@ func (e *EpochKVStore) RestoreLastPrunedEpoch() error { return e.lastPrunedEpoch.RestoreFromDisk() } +func (e *EpochKVStore) LastAccessedEpoch() (lastAccessedEpoch iotago.EpochIndex, err error) { + if lastAccessedEpoch, err = e.lastAccessedEpoch.Get(); err != nil { + if !ierrors.Is(err, kvstore.ErrKeyNotFound) { + err = ierrors.Wrap(err, "failed to get last accessed epoch") + } else { + err = nil + } + } + + return lastAccessedEpoch, err +} + func (e *EpochKVStore) LastPrunedEpoch() (iotago.EpochIndex, bool) { return e.lastPrunedEpoch.Index() } func (e *EpochKVStore) GetEpoch(epoch iotago.EpochIndex) (kvstore.KVStore, error) { + _, _ = e.lastAccessedEpoch.Compute(func(lastAccessedEpoch iotago.EpochIndex, exists bool) (newValue iotago.EpochIndex, err error) { + if lastAccessedEpoch >= epoch { + return lastAccessedEpoch, kvstore.ErrTypedValueNotChanged + } + + return epoch, nil + }) + if e.isTooOld(epoch) { return nil, ierrors.Wrapf(database.ErrEpochPruned, "epoch %d is too old", epoch) } @@ -81,3 +103,18 @@ func (e *EpochKVStore) Prune(epoch iotago.EpochIndex, defaultPruningDelay iotago return nil } + +func (e *EpochKVStore) RollbackEpochs(epoch iotago.EpochIndex) (lastPrunedEpoch iotago.EpochIndex, err error) { + lastAccessedEpoch, err := e.LastAccessedEpoch() + if err != nil { + return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed epoch") + } + + for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + if err = e.DeleteEpoch(epochToPrune); err != nil { + return epochToPrune, ierrors.Wrapf(err, "error while deleting epoch %d", epochToPrune) + } + } + + return lastAccessedEpoch, nil +} diff --git a/pkg/storage/prunable/epochstore/store.go b/pkg/storage/prunable/epochstore/store.go index f93d71832..a8ef66e68 100644 --- a/pkg/storage/prunable/epochstore/store.go +++ b/pkg/storage/prunable/epochstore/store.go @@ -9,33 +9,47 @@ import ( iotago "github.com/iotaledger/iota.go/v4" ) +const ( + entriesKey byte = iota + lastAccessedEpochKey + lastPrunedEpochKey +) + type Store[V any] struct { realm kvstore.Realm kv *kvstore.TypedStore[iotago.EpochIndex, V] pruningDelay iotago.EpochIndex - lastPrunedEpoch *model.PruningIndex + lastAccessedEpoch *kvstore.TypedValue[iotago.EpochIndex] + lastPrunedEpoch *model.PruningIndex } -func NewStore[V any](storeRealm, pruningRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay iotago.EpochIndex, vToBytes kvstore.ObjectToBytes[V], bytesToV kvstore.BytesToObject[V]) *Store[V] { +func NewStore[V any](storeRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay iotago.EpochIndex, vToBytes kvstore.ObjectToBytes[V], bytesToV kvstore.BytesToObject[V]) *Store[V] { return &Store[V]{ - realm: storeRealm, - kv: kvstore.NewTypedStore(lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes, vToBytes, bytesToV), - pruningDelay: pruningDelay, - lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(pruningRealm)), storeRealm), + realm: storeRealm, + kv: kvstore.NewTypedStore(lo.PanicOnErr(kv.WithExtendedRealm(append(storeRealm, entriesKey))), iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes, vToBytes, bytesToV), + pruningDelay: pruningDelay, + lastAccessedEpoch: kvstore.NewTypedValue(kv, append(storeRealm, lastAccessedEpochKey), iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes), + lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), kvstore.Realm{lastPrunedEpochKey}), } } -func (s *Store[V]) isTooOld(epoch iotago.EpochIndex) bool { - prunedEpoch, hasPruned := s.lastPrunedEpoch.Index() - - return hasPruned && epoch <= prunedEpoch -} - func (s *Store[V]) RestoreLastPrunedEpoch() error { return s.lastPrunedEpoch.RestoreFromDisk() } +func (s *Store[V]) LastAccessedEpoch() (lastAccessedEpoch iotago.EpochIndex, err error) { + if lastAccessedEpoch, err = s.lastAccessedEpoch.Get(); err != nil { + if !ierrors.Is(err, kvstore.ErrKeyNotFound) { + err = ierrors.Wrap(err, "failed to get last accessed epoch") + } else { + err = nil + } + } + + return lastAccessedEpoch, err +} + func (s *Store[V]) LastPrunedEpoch() (iotago.EpochIndex, bool) { return s.lastPrunedEpoch.Index() } @@ -61,6 +75,14 @@ func (s *Store[V]) Load(epoch iotago.EpochIndex) (V, error) { } func (s *Store[V]) Store(epoch iotago.EpochIndex, value V) error { + _, _ = s.lastAccessedEpoch.Compute(func(lastAccessedEpoch iotago.EpochIndex, exists bool) (newValue iotago.EpochIndex, err error) { + if lastAccessedEpoch >= epoch { + return lastAccessedEpoch, kvstore.ErrTypedValueNotChanged + } + + return epoch, nil + }) + if s.isTooOld(epoch) { return ierrors.Wrapf(database.ErrEpochPruned, "epoch %d is too old", epoch) } @@ -133,3 +155,24 @@ func (s *Store[V]) Prune(epoch iotago.EpochIndex, defaultPruningDelay iotago.Epo return nil } + +func (s *Store[V]) RollbackEpochs(epoch iotago.EpochIndex) (lastPrunedEpoch iotago.EpochIndex, err error) { + lastAccessedEpoch, err := s.LastAccessedEpoch() + if err != nil { + return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed epoch") + } + + for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + if err = s.DeleteEpoch(epochToPrune); err != nil { + return epochToPrune, ierrors.Wrapf(err, "error while deleting epoch %d", epochToPrune) + } + } + + return lastAccessedEpoch, nil +} + +func (s *Store[V]) isTooOld(epoch iotago.EpochIndex) bool { + prunedEpoch, hasPruned := s.lastPrunedEpoch.Index() + + return hasPruned && epoch <= prunedEpoch +} diff --git a/pkg/storage/prunable/prunable.go b/pkg/storage/prunable/prunable.go index 1fc392877..11bb3ebda 100644 --- a/pkg/storage/prunable/prunable.go +++ b/pkg/storage/prunable/prunable.go @@ -41,10 +41,10 @@ func New(dbConfig database.Config, apiProvider iotago.APIProvider, errorHandler semiPermanentDBConfig: semiPermanentDBConfig, semiPermanentDB: semiPermanentDB, - decidedUpgradeSignals: epochstore.NewStore(kvstore.Realm{epochPrefixDecidedUpgradeSignals}, kvstore.Realm{lastPrunedEpochKey}, semiPermanentDB.KVStore(), pruningDelayDecidedUpgradeSignals, model.VersionAndHash.Bytes, model.VersionAndHashFromBytes), - poolRewards: epochstore.NewEpochKVStore(kvstore.Realm{epochPrefixPoolRewards}, kvstore.Realm{lastPrunedEpochKey}, semiPermanentDB.KVStore(), pruningDelayPoolRewards), - poolStats: epochstore.NewStore(kvstore.Realm{epochPrefixPoolStats}, kvstore.Realm{lastPrunedEpochKey}, semiPermanentDB.KVStore(), pruningDelayPoolStats, (*model.PoolsStats).Bytes, model.PoolsStatsFromBytes), - committee: epochstore.NewStore(kvstore.Realm{epochPrefixCommittee}, kvstore.Realm{lastPrunedEpochKey}, semiPermanentDB.KVStore(), pruningDelayCommittee, (*account.Accounts).Bytes, account.AccountsFromBytes), + decidedUpgradeSignals: epochstore.NewStore(kvstore.Realm{epochPrefixDecidedUpgradeSignals}, semiPermanentDB.KVStore(), pruningDelayDecidedUpgradeSignals, model.VersionAndHash.Bytes, model.VersionAndHashFromBytes), + poolRewards: epochstore.NewEpochKVStore(kvstore.Realm{epochPrefixPoolRewards}, semiPermanentDB.KVStore(), pruningDelayPoolRewards), + poolStats: epochstore.NewStore(kvstore.Realm{epochPrefixPoolStats}, semiPermanentDB.KVStore(), pruningDelayPoolStats, (*model.PoolsStats).Bytes, model.PoolsStatsFromBytes), + committee: epochstore.NewStore(kvstore.Realm{epochPrefixCommittee}, semiPermanentDB.KVStore(), pruningDelayCommittee, (*account.Accounts).Bytes, account.AccountsFromBytes), } } @@ -142,54 +142,60 @@ func (p *Prunable) Flush() { } } -func (p *Prunable) Rollback(targetSlot iotago.SlotIndex) error { - timeProvider := p.apiProvider.APIForSlot(targetSlot).TimeProvider() - targetSlotEpoch := timeProvider.EpochFromSlot(targetSlot) - lastCommittedEpoch := targetSlotEpoch - // if the target index is the last slot of the epoch, the epoch was committed - if timeProvider.EpochEnd(targetSlotEpoch) != targetSlot { - lastCommittedEpoch-- +func (p *Prunable) Rollback(targetEpoch iotago.EpochIndex, pruningRange [2]iotago.SlotIndex) error { + if err := p.prunableSlotStore.PruneSlots(targetEpoch, pruningRange); err != nil { + return ierrors.Wrapf(err, "failed to prune slots in range [%d, %d] from target epoch %d", pruningRange[0], pruningRange[1], targetEpoch) } - if err := p.prunableSlotStore.RollbackBucket(targetSlotEpoch, targetSlot, timeProvider.EpochEnd(targetSlotEpoch)); err != nil { - return ierrors.Wrapf(err, "error while rolling back slots in a bucket for epoch %d", targetSlotEpoch) + var lastPrunedEpoch iotago.EpochIndex + if lastPrunedCommitteeEpoch, err := p.rollbackCommitteeEpochs(targetEpoch, pruningRange[0]-1); err != nil { + return ierrors.Wrapf(err, "failed to rollback committee epochs to target epoch %d", targetEpoch) + } else { + lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedCommitteeEpoch) } - // Shut down the prunableSlotStore in order to flush and get consistent state on disk after reopening. - p.prunableSlotStore.Shutdown() + if lastPrunedPoolStatsEpoch, err := p.poolStats.RollbackEpochs(targetEpoch); err != nil { + return ierrors.Wrapf(err, "failed to rollback pool stats epochs to target epoch %d", targetEpoch) + } else { + lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedPoolStatsEpoch) + } - // Removed entries that belong to the old fork and cannot be re-used. - for epoch := lastCommittedEpoch + 1; ; epoch++ { - if epoch > targetSlotEpoch { - shouldRollback, err := p.shouldRollbackCommittee(epoch, targetSlot) - if err != nil { - return ierrors.Wrapf(err, "error while checking if committee for epoch %d should be rolled back", epoch) - } + if lastPrunedDecidedUpgradeSignalsEpoch, err := p.decidedUpgradeSignals.RollbackEpochs(targetEpoch); err != nil { + return ierrors.Wrapf(err, "failed to rollback decided upgrade signals epochs to target epoch %d", targetEpoch) + } else { + lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedDecidedUpgradeSignalsEpoch) + } - if shouldRollback { - if err := p.committee.DeleteEpoch(epoch); err != nil { - return ierrors.Wrapf(err, "error while deleting committee for epoch %d", epoch) - } - } + if lastPrunedPoolRewardsEpoch, err := p.poolRewards.RollbackEpochs(targetEpoch); err != nil { + return ierrors.Wrapf(err, "failed to rollback pool rewards epochs to target epoch %d", targetEpoch) + } else { + lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedPoolRewardsEpoch) + } - if deleted := p.prunableSlotStore.DeleteBucket(epoch); !deleted { - break - } - } + for epochToPrune := targetEpoch + 1; epochToPrune <= lastPrunedEpoch; epochToPrune++ { + p.prunableSlotStore.DeleteBucket(epochToPrune) + } - if err := p.poolRewards.DeleteEpoch(epoch); err != nil { - return ierrors.Wrapf(err, "error while deleting pool rewards for epoch %d", epoch) - } - if err := p.poolStats.DeleteEpoch(epoch); err != nil { - return ierrors.Wrapf(err, "error while deleting pool stats for epoch %d", epoch) - } + return nil +} - if err := p.decidedUpgradeSignals.DeleteEpoch(epoch); err != nil { - return ierrors.Wrapf(err, "error while deleting decided upgrade signals for epoch %d", epoch) +func (p *Prunable) rollbackCommitteeEpochs(epoch iotago.EpochIndex, targetSlot iotago.SlotIndex) (lastPrunedEpoch iotago.EpochIndex, err error) { + lastAccessedEpoch, err := p.committee.LastAccessedEpoch() + if err != nil { + return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed committee epoch") + } + + for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + if shouldRollback, rollbackErr := p.shouldRollbackCommittee(epochToPrune, targetSlot); rollbackErr != nil { + return epochToPrune, ierrors.Wrapf(rollbackErr, "error while checking if committee for epoch %d should be rolled back", epochToPrune) + } else if shouldRollback { + if err = p.committee.DeleteEpoch(epochToPrune); err != nil { + return epochToPrune, ierrors.Wrapf(err, "error while deleting committee for epoch %d", epochToPrune) + } } } - return nil + return lastAccessedEpoch, nil } // Remove committee for the next epoch only if forking point is before point of no return and committee is reused. diff --git a/pkg/storage/prunable/prunable_epoch.go b/pkg/storage/prunable/prunable_epoch.go index f09d3d075..0c2186dd0 100644 --- a/pkg/storage/prunable/prunable_epoch.go +++ b/pkg/storage/prunable/prunable_epoch.go @@ -13,7 +13,6 @@ const ( epochPrefixPoolRewards epochPrefixPoolStats epochPrefixCommittee - lastPrunedEpochKey ) const ( diff --git a/pkg/storage/storage_prunable.go b/pkg/storage/storage_prunable.go index 6ee3c4ead..eb62eac4d 100644 --- a/pkg/storage/storage_prunable.go +++ b/pkg/storage/storage_prunable.go @@ -40,50 +40,70 @@ func (s *Storage) Blocks(slot iotago.SlotIndex) (*slotstore.Blocks, error) { // Reset resets the component to a clean state as if it was created at the last commitment. func (s *Storage) Reset() { - s.lastAccessedBlocks.Compute(func(lastAccessedBlocks iotago.SlotIndex) iotago.SlotIndex { - latestCommittedSlot := s.Settings().LatestCommitment().Slot() - - for slot := latestCommittedSlot + 1; slot <= lastAccessedBlocks; slot++ { - if blocksForSlot, err := s.prunable.Blocks(slot); err != nil { - s.errorHandler(ierrors.Wrapf(err, "failed to clear blocks at slot %d", slot)) - } else if err = blocksForSlot.Clear(); err != nil { - s.errorHandler(ierrors.Wrapf(err, "failed to clear blocks at slot %d", slot)) - } - } - - return latestCommittedSlot - }) + s.Rollback(s.Settings().LatestCommitment().Slot()) } func (s *Storage) RootBlocks(slot iotago.SlotIndex) (*slotstore.Store[iotago.BlockID, iotago.CommitmentID], error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing root blocks") + } + return s.prunable.RootBlocks(slot) } func (s *Storage) Mutations(slot iotago.SlotIndex) (kvstore.KVStore, error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing mutations") + } + return s.prunable.Mutations(slot) } func (s *Storage) Attestations(slot iotago.SlotIndex) (kvstore.KVStore, error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing attestations") + } + return s.prunable.Attestations(slot) } func (s *Storage) AccountDiffs(slot iotago.SlotIndex) (*slotstore.AccountDiffs, error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing account diffs") + } + return s.prunable.AccountDiffs(slot) } func (s *Storage) ValidatorPerformances(slot iotago.SlotIndex) (*slotstore.Store[iotago.AccountID, *model.ValidatorPerformance], error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing validator performances") + } + return s.prunable.ValidatorPerformances(slot) } func (s *Storage) UpgradeSignals(slot iotago.SlotIndex) (*slotstore.Store[account.SeatIndex, *model.SignaledBlock], error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing upgrade signals") + } + return s.prunable.UpgradeSignals(slot) } func (s *Storage) Roots(slot iotago.SlotIndex) (*slotstore.Store[iotago.CommitmentID, *iotago.Roots], error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing roots") + } + return s.prunable.Roots(slot) } func (s *Storage) Retainer(slot iotago.SlotIndex) (*slotstore.Retainer, error) { + if err := s.permanent.Settings().AdvanceLatestStoredSlot(slot); err != nil { + return nil, ierrors.Wrap(err, "failed to advance latest stored slot when accessing retainer") + } + return s.prunable.Retainer(slot) } @@ -100,6 +120,22 @@ func (s *Storage) RestoreFromDisk() { s.lastPrunedEpoch.MarkEvicted(lastPrunedEpoch) } -func (s *Storage) RollbackPrunable(targetIndex iotago.SlotIndex) error { - return s.prunable.Rollback(targetIndex) +func (s *Storage) Rollback(targetSlot iotago.SlotIndex) error { + if err := s.prunable.Rollback(s.pruningRange(targetSlot)); err != nil { + return ierrors.Wrapf(err, "failed to rollback prunable storage to slot %d", targetSlot) + } + + return nil +} + +func (s *Storage) pruningRange(targetSlot iotago.SlotIndex) (targetEpoch iotago.EpochIndex, pruneRange [2]iotago.SlotIndex) { + epochOfSlot := func(slot iotago.SlotIndex) iotago.EpochIndex { + return s.Settings().APIProvider().APIForSlot(slot).TimeProvider().EpochFromSlot(slot) + } + + if targetEpoch, pruneRange = epochOfSlot(targetSlot), [2]iotago.SlotIndex{targetSlot + 1, s.Settings().LatestStoredSlot()}; epochOfSlot(pruneRange[0]) > targetEpoch { + pruneRange[1] = s.Settings().APIProvider().APIForEpoch(targetEpoch).TimeProvider().EpochEnd(targetEpoch) + } + + return targetEpoch, pruneRange } diff --git a/tools/evil-spammer/go.mod b/tools/evil-spammer/go.mod index c959ca7a6..22a5cf5ac 100644 --- a/tools/evil-spammer/go.mod +++ b/tools/evil-spammer/go.mod @@ -10,13 +10,13 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/ethereum/go-ethereum v1.13.4 github.com/google/martian v2.1.0+incompatible - github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b - github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b + github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota-core/tools/genesis-snapshot v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd @@ -38,13 +38,13 @@ require ( github.com/holiman/uint256 v1.2.3 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 // indirect - github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b // indirect + github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect diff --git a/tools/evil-spammer/go.sum b/tools/evil-spammer/go.sum index 4fa058b2b..f43719e7a 100644 --- a/tools/evil-spammer/go.sum +++ b/tools/evil-spammer/go.sum @@ -173,34 +173,34 @@ github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJ github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 h1:dTrD7X2PTNgli6EbS4tV9qu3QAm/kBU3XaYZV2xdzys= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7/go.mod h1:ZRdPu684P0fQ1z8sXz4dj9H5LWHhz4a9oCtvjunkSrw= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b h1:D68khiAFb9DwTvjZc2nc4R0E6wUdKyYCUXkmdaMzuoQ= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b h1:mX3NXaTMLEwZnEs4IlxEvXY0YZo8qbb8M1xM39FS6qY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b h1:HF4e0wz0JMIT4m3saqdQ//T9nWHV9d5sLMtEwNDuykM= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b h1:ZERXxhQBUBV1AqTE6cUI4vTxSx4JrnsMuLZFgj32xLM= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b h1:ZUUqRRO6XnQmVcXlXyx07vqySn28+bln6jp9KagYCjY= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b h1:8b2sH+2Vf0y5BDYTMwKa09iQr3JF9JrzTI64DkXb+9U= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b h1:JJPnr231djUTgTnE4oGz847WE9VA7Py6E6fgZwT5TQo= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b h1:LusmtjpfG/q8lc15Fp9W3kABbN3tArKx/zw2ibdY1DU= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b h1:UvFWI8wQJS/XQOeWHpPsaFVeS2nxJ7nIGFr+IFjrnVw= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 h1:6HIBNKOWLXY5Gk1s+q93D6IsTcqwANigfMO3ePtV3HQ= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b h1:EhVgAU/f2J3VYZwP60dRdyfAeDU3c/gBzX9blKtQGKA= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b h1:O68POYIqBLnoHN+HIszc58QwAI2qocYq0WKGfVrXmMg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b h1:zaXZn9yV/95SRDkgCZQeBbSbmcJTKSZbCB7oBd71Qwg= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b h1:MDZhTZTVDiydXcW5j4TA7HixVCyAdToIMPhHfJee7cE= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 h1:EOfxTuAiBmED1VHuVh7/UIeB27cCRe13gdSzyioNMBw= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 h1:xAER9M9Uoz2EOWT43E9wmXRe+RmAk8OBSUoboH4Su8M= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 h1:drmpgLlJy7kZ09Dt1qKSnbILU+27Qu2jp4VdPDNwbFk= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 h1:BC5GkIHyXdoJGdw6Tu5ds2kjw9grFLtwQiuMaKfdLU8= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 h1:r8TkdQJB7/bJd8cF8z5GQ+rX/7JpbPdPoN7wMoV1OCM= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 h1:ytzZZPtclAzLfjxv26frbinCGx3Z6ouUENbx5U7lFGg= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 h1:QMxd32Y/veVhTDPCiOFgetjUbG7sr9MryF29/rSPkMA= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 h1:/xPwStUckZ2V0XPoY496cXU+c5elpHyvYoT6JAmuvRY= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 h1:AvNLzONVMspwx7nD/NyYUgb5Hi7/zgzIOegr1uRD/M8= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 h1:e1uJAlXE3zeXpa+c4uFOG+/AMFbUlLt2mcrSK5NMxVs= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 h1:7wjs4t1snBDJ8LOTl+tZhr2ORywSOTgJMppxiIAMA0A= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 h1:1QMJ39qXIx/IZVzus3+97IV7Pa++e+d340TvbMjhiBU= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 h1:hZli4E9kJUAEQ7gzZR1XbPcpgqvqMPYq8YBPMbrBuos= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 h1:OlDhgvJ48bZxcvTeebJ1b96xtNnJAddejd2Q4rlH1mU= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd h1:FKiEzy3gOyWDGXs6mTMS9qFHGXSxH97N5w+08ljbEtU= github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd/go.mod h1:XFwcDfAF0hv6wB/trCsyDmQyPLjEZ60yTrllJ2bprN8= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/tools/gendoc/go.mod b/tools/gendoc/go.mod index 6e52b1d6a..b7a50cb8d 100644 --- a/tools/gendoc/go.mod +++ b/tools/gendoc/go.mod @@ -5,7 +5,7 @@ go 1.21 replace github.com/iotaledger/iota-core => ../../ require ( - github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b + github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 github.com/iotaledger/hive.go/apputils v0.0.0-20230829152614-7afc7a4d89b3 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 ) @@ -18,7 +18,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -58,19 +57,19 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 // indirect - github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 // indirect - github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b // indirect + github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 // indirect github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af // indirect github.com/iotaledger/inx/go v1.0.0-rc.2.0.20231020151337-569450d5bf7d // indirect github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd // indirect @@ -135,7 +134,6 @@ require ( github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20230904192822-1876fd5063bc // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pokt-network/smt v0.6.1 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.17.0 // indirect @@ -151,7 +149,6 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.4 // indirect github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect @@ -181,6 +178,5 @@ require ( google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/tools/gendoc/go.sum b/tools/gendoc/go.sum index 6d3b7c621..adddbda7f 100644 --- a/tools/gendoc/go.sum +++ b/tools/gendoc/go.sum @@ -279,36 +279,36 @@ github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 h1:dTrD7X2PTNgli6EbS4tV9qu3QAm/kBU3XaYZV2xdzys= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7/go.mod h1:ZRdPu684P0fQ1z8sXz4dj9H5LWHhz4a9oCtvjunkSrw= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b h1:D68khiAFb9DwTvjZc2nc4R0E6wUdKyYCUXkmdaMzuoQ= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b h1:mX3NXaTMLEwZnEs4IlxEvXY0YZo8qbb8M1xM39FS6qY= -github.com/iotaledger/hive.go/app v0.0.0-20231020115340-13da292c580b/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 h1:EOfxTuAiBmED1VHuVh7/UIeB27cCRe13gdSzyioNMBw= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42 h1:xAER9M9Uoz2EOWT43E9wmXRe+RmAk8OBSUoboH4Su8M= +github.com/iotaledger/hive.go/app v0.0.0-20231027195901-620bd7470e42/go.mod h1:8ZbIKR84oQd/3iQ5eeT7xpudO9/ytzXP7veIYnk7Orc= github.com/iotaledger/hive.go/apputils v0.0.0-20230829152614-7afc7a4d89b3 h1:4aVJTc0KS77uEw0Tny4r0n1ORwcbAQDECaCclgf/6lE= github.com/iotaledger/hive.go/apputils v0.0.0-20230829152614-7afc7a4d89b3/go.mod h1:TZeAqieDu+xDOZp2e9+S+8pZp1PrfgcwLUnxmd8IgLU= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b h1:HF4e0wz0JMIT4m3saqdQ//T9nWHV9d5sLMtEwNDuykM= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b h1:ZERXxhQBUBV1AqTE6cUI4vTxSx4JrnsMuLZFgj32xLM= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b h1:ZUUqRRO6XnQmVcXlXyx07vqySn28+bln6jp9KagYCjY= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b h1:8b2sH+2Vf0y5BDYTMwKa09iQr3JF9JrzTI64DkXb+9U= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b h1:JJPnr231djUTgTnE4oGz847WE9VA7Py6E6fgZwT5TQo= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b h1:LusmtjpfG/q8lc15Fp9W3kABbN3tArKx/zw2ibdY1DU= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b h1:UvFWI8wQJS/XQOeWHpPsaFVeS2nxJ7nIGFr+IFjrnVw= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 h1:6HIBNKOWLXY5Gk1s+q93D6IsTcqwANigfMO3ePtV3HQ= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b h1:EhVgAU/f2J3VYZwP60dRdyfAeDU3c/gBzX9blKtQGKA= -github.com/iotaledger/hive.go/logger v0.0.0-20231020115340-13da292c580b/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b h1:O68POYIqBLnoHN+HIszc58QwAI2qocYq0WKGfVrXmMg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b h1:zaXZn9yV/95SRDkgCZQeBbSbmcJTKSZbCB7oBd71Qwg= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b h1:MDZhTZTVDiydXcW5j4TA7HixVCyAdToIMPhHfJee7cE= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 h1:drmpgLlJy7kZ09Dt1qKSnbILU+27Qu2jp4VdPDNwbFk= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 h1:BC5GkIHyXdoJGdw6Tu5ds2kjw9grFLtwQiuMaKfdLU8= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 h1:r8TkdQJB7/bJd8cF8z5GQ+rX/7JpbPdPoN7wMoV1OCM= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 h1:ytzZZPtclAzLfjxv26frbinCGx3Z6ouUENbx5U7lFGg= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 h1:QMxd32Y/veVhTDPCiOFgetjUbG7sr9MryF29/rSPkMA= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 h1:/xPwStUckZ2V0XPoY496cXU+c5elpHyvYoT6JAmuvRY= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 h1:AvNLzONVMspwx7nD/NyYUgb5Hi7/zgzIOegr1uRD/M8= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 h1:e1uJAlXE3zeXpa+c4uFOG+/AMFbUlLt2mcrSK5NMxVs= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 h1:7wjs4t1snBDJ8LOTl+tZhr2ORywSOTgJMppxiIAMA0A= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 h1:1QMJ39qXIx/IZVzus3+97IV7Pa++e+d340TvbMjhiBU= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 h1:hZli4E9kJUAEQ7gzZR1XbPcpgqvqMPYq8YBPMbrBuos= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 h1:OlDhgvJ48bZxcvTeebJ1b96xtNnJAddejd2Q4rlH1mU= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af h1:3+heabXNGjv3Sx1sg0qG+QdqeEWIpf7jM3T5cPiVmSA= github.com/iotaledger/inx-app v1.0.0-rc.3.0.20231020152103-b6ea7ff7a4af/go.mod h1:3BZ5r/MhswUZwXHDzMZacOhm6DWpUdGjefNRNshfwiM= github.com/iotaledger/inx/go v1.0.0-rc.2.0.20231020151337-569450d5bf7d h1:PeJNFBkR28hdQCxUyC/cbOlDjFm3D1U7O5bxi+qsOvQ= diff --git a/tools/genesis-snapshot/go.mod b/tools/genesis-snapshot/go.mod index 0fc07ce2a..31ec44009 100644 --- a/tools/genesis-snapshot/go.mod +++ b/tools/genesis-snapshot/go.mod @@ -5,10 +5,10 @@ go 1.21 replace github.com/iotaledger/iota-core => ../../ require ( - github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b - github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b + github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 + github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 github.com/iotaledger/iota-core v0.0.0-00010101000000-000000000000 github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd github.com/mr-tron/base58 v1.2.0 @@ -26,14 +26,15 @@ require ( github.com/holiman/uint256 v1.2.3 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 // indirect - github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 // indirect - github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b // indirect - github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b // indirect + github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 // indirect + github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/text v0.2.0 // indirect @@ -59,6 +60,8 @@ require ( github.com/wollac/iota-crypto-demo v0.0.0-20221117162917-b10619eccb98 // indirect github.com/zyedidia/generic v1.2.1 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/tools/genesis-snapshot/go.sum b/tools/genesis-snapshot/go.sum index 6c4a41287..b41c4bdae 100644 --- a/tools/genesis-snapshot/go.sum +++ b/tools/genesis-snapshot/go.sum @@ -28,30 +28,32 @@ github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJ github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7 h1:dTrD7X2PTNgli6EbS4tV9qu3QAm/kBU3XaYZV2xdzys= github.com/iotaledger/grocksdb v1.7.5-0.20230220105546-5162e18885c7/go.mod h1:ZRdPu684P0fQ1z8sXz4dj9H5LWHhz4a9oCtvjunkSrw= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b h1:D68khiAFb9DwTvjZc2nc4R0E6wUdKyYCUXkmdaMzuoQ= -github.com/iotaledger/hive.go/ads v0.0.0-20231020115340-13da292c580b/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b h1:HF4e0wz0JMIT4m3saqdQ//T9nWHV9d5sLMtEwNDuykM= -github.com/iotaledger/hive.go/constraints v0.0.0-20231020115340-13da292c580b/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b h1:ZERXxhQBUBV1AqTE6cUI4vTxSx4JrnsMuLZFgj32xLM= -github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231020115340-13da292c580b/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b h1:ZUUqRRO6XnQmVcXlXyx07vqySn28+bln6jp9KagYCjY= -github.com/iotaledger/hive.go/crypto v0.0.0-20231020115340-13da292c580b/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b h1:8b2sH+2Vf0y5BDYTMwKa09iQr3JF9JrzTI64DkXb+9U= -github.com/iotaledger/hive.go/ds v0.0.0-20231020115340-13da292c580b/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b h1:JJPnr231djUTgTnE4oGz847WE9VA7Py6E6fgZwT5TQo= -github.com/iotaledger/hive.go/ierrors v0.0.0-20231020115340-13da292c580b/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b h1:LusmtjpfG/q8lc15Fp9W3kABbN3tArKx/zw2ibdY1DU= -github.com/iotaledger/hive.go/kvstore v0.0.0-20231020115340-13da292c580b/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b h1:UvFWI8wQJS/XQOeWHpPsaFVeS2nxJ7nIGFr+IFjrnVw= -github.com/iotaledger/hive.go/lo v0.0.0-20231020115340-13da292c580b/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38 h1:6HIBNKOWLXY5Gk1s+q93D6IsTcqwANigfMO3ePtV3HQ= -github.com/iotaledger/hive.go/log v0.0.0-20231019113503-7986872a7a38/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b h1:O68POYIqBLnoHN+HIszc58QwAI2qocYq0WKGfVrXmMg= -github.com/iotaledger/hive.go/runtime v0.0.0-20231020115340-13da292c580b/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b h1:zaXZn9yV/95SRDkgCZQeBbSbmcJTKSZbCB7oBd71Qwg= -github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231020115340-13da292c580b/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b h1:MDZhTZTVDiydXcW5j4TA7HixVCyAdToIMPhHfJee7cE= -github.com/iotaledger/hive.go/stringify v0.0.0-20231020115340-13da292c580b/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42 h1:EOfxTuAiBmED1VHuVh7/UIeB27cCRe13gdSzyioNMBw= +github.com/iotaledger/hive.go/ads v0.0.0-20231027195901-620bd7470e42/go.mod h1:IFh0gDfeMgZtfCo+5afK59IDR4xXh+cTR9YtLnZPcbY= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42 h1:drmpgLlJy7kZ09Dt1qKSnbILU+27Qu2jp4VdPDNwbFk= +github.com/iotaledger/hive.go/constraints v0.0.0-20231027195901-620bd7470e42/go.mod h1:dOBOM2s4se3HcWefPe8sQLUalGXJ8yVXw58oK8jke3s= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42 h1:BC5GkIHyXdoJGdw6Tu5ds2kjw9grFLtwQiuMaKfdLU8= +github.com/iotaledger/hive.go/core v1.0.0-rc.3.0.20231027195901-620bd7470e42/go.mod h1:Mc+ACqBGPxrPMIPUBOm6/HL0J6m0iVMwjtIEKW3uow8= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42 h1:r8TkdQJB7/bJd8cF8z5GQ+rX/7JpbPdPoN7wMoV1OCM= +github.com/iotaledger/hive.go/crypto v0.0.0-20231027195901-620bd7470e42/go.mod h1:h3o6okvMSEK3KOX6pOp3yq1h9ohTkTfo6X8MzEadeb0= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42 h1:ytzZZPtclAzLfjxv26frbinCGx3Z6ouUENbx5U7lFGg= +github.com/iotaledger/hive.go/ds v0.0.0-20231027195901-620bd7470e42/go.mod h1:3XkUSKfHaVxGbT0XAvjNlVYqPzhfLTGhDtdNA5UBPco= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42 h1:QMxd32Y/veVhTDPCiOFgetjUbG7sr9MryF29/rSPkMA= +github.com/iotaledger/hive.go/ierrors v0.0.0-20231027195901-620bd7470e42/go.mod h1:HcE8B5lP96enc/OALTb2/rIIi+yOLouRoHOKRclKmC8= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42 h1:/xPwStUckZ2V0XPoY496cXU+c5elpHyvYoT6JAmuvRY= +github.com/iotaledger/hive.go/kvstore v0.0.0-20231027195901-620bd7470e42/go.mod h1:O/U3jtiUDeqqM0MZQFu2UPqS9fUm0C5hNISxlmg/thE= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42 h1:AvNLzONVMspwx7nD/NyYUgb5Hi7/zgzIOegr1uRD/M8= +github.com/iotaledger/hive.go/lo v0.0.0-20231027195901-620bd7470e42/go.mod h1:s4kzx9QY1MVWHJralj+3q5kI0eARtrJhphYD/iBbPfo= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42 h1:e1uJAlXE3zeXpa+c4uFOG+/AMFbUlLt2mcrSK5NMxVs= +github.com/iotaledger/hive.go/log v0.0.0-20231027195901-620bd7470e42/go.mod h1:JvokzmpmFZPDskMlUqqjgHtD8usVJU4nAY/TNMGge8M= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42 h1:7wjs4t1snBDJ8LOTl+tZhr2ORywSOTgJMppxiIAMA0A= +github.com/iotaledger/hive.go/logger v0.0.0-20231027195901-620bd7470e42/go.mod h1:aBfAfIB2GO/IblhYt5ipCbyeL9bXSNeAwtYVA3hZaHg= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42 h1:1QMJ39qXIx/IZVzus3+97IV7Pa++e+d340TvbMjhiBU= +github.com/iotaledger/hive.go/runtime v0.0.0-20231027195901-620bd7470e42/go.mod h1:jRw8yFipiPaqmTPHh7hTcxAP9u6pjRGpByS3REJKkbY= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42 h1:hZli4E9kJUAEQ7gzZR1XbPcpgqvqMPYq8YBPMbrBuos= +github.com/iotaledger/hive.go/serializer/v2 v2.0.0-rc.1.0.20231027195901-620bd7470e42/go.mod h1:SdK26z8/VhWtxaqCuQrufm80SELgowQPmu9T/8eUQ8g= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42 h1:OlDhgvJ48bZxcvTeebJ1b96xtNnJAddejd2Q4rlH1mU= +github.com/iotaledger/hive.go/stringify v0.0.0-20231027195901-620bd7470e42/go.mod h1:FTo/UWzNYgnQ082GI9QVM9HFDERqf9rw9RivNpqrnTs= github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd h1:FKiEzy3gOyWDGXs6mTMS9qFHGXSxH97N5w+08ljbEtU= github.com/iotaledger/iota.go/v4 v4.0.0-20231019174124-aa2290512bcd/go.mod h1:XFwcDfAF0hv6wB/trCsyDmQyPLjEZ60yTrllJ2bprN8= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -113,6 +115,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -124,6 +127,12 @@ github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XD github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= From 95eb8d4b451a6f0a37aa04798766ee7e69936338 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 02:09:41 +0200 Subject: [PATCH 02/14] Fix: committed missing changes --- pkg/protocol/engine_manager.go | 14 +++++++------- pkg/protocol/protocol.go | 5 ----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/pkg/protocol/engine_manager.go b/pkg/protocol/engine_manager.go index 3c3fb9251..42195dba6 100644 --- a/pkg/protocol/engine_manager.go +++ b/pkg/protocol/engine_manager.go @@ -107,7 +107,7 @@ func (e *EngineManager) ForkAtSlot(slot iotago.SlotIndex) (*engine.Engine, error // remove commitments that after forking point. latestCommitment := newStorage.Settings().LatestCommitment() - if err := newStorage.Commitments().Rollback(slot, latestCommitment.Slot()); err != nil { + if err = newStorage.Commitments().Rollback(slot, latestCommitment.Slot()); err != nil { return nil, ierrors.Wrap(err, "failed to rollback commitments") } // create temporary components and rollback their permanent state, which will be reflected on disk. @@ -118,14 +118,14 @@ func (e *EngineManager) ForkAtSlot(slot iotago.SlotIndex) (*engine.Engine, error accountsManager := accountsledger.New(newStorage.Settings().APIProvider(), blockCache.Block, newStorage.AccountDiffs, newStorage.Accounts()) accountsManager.SetLatestCommittedSlot(latestCommitment.Slot()) - if err := accountsManager.Rollback(slot); err != nil { + if err = accountsManager.Rollback(slot); err != nil { return nil, ierrors.Wrap(err, "failed to rollback accounts manager") } - if err := evictionState.Rollback(newStorage.Settings().LatestFinalizedSlot(), slot); err != nil { + if err = evictionState.Rollback(newStorage.Settings().LatestFinalizedSlot(), slot); err != nil { return nil, ierrors.Wrap(err, "failed to rollback eviction state") } - if err := newStorage.Ledger().Rollback(slot); err != nil { + if err = newStorage.Ledger().Rollback(slot); err != nil { return nil, err } @@ -134,18 +134,18 @@ func (e *EngineManager) ForkAtSlot(slot iotago.SlotIndex) (*engine.Engine, error return nil, ierrors.Wrapf(err, "error while retrieving commitment for target index %d", slot) } - if err := newStorage.Settings().Rollback(targetCommitment); err != nil { + if err = newStorage.Settings().Rollback(targetCommitment); err != nil { return nil, err } - if err := newStorage.RollbackPrunable(slot); err != nil { + if err = newStorage.Rollback(slot); err != nil { return nil, err } candidateEngine := e.loadEngineInstanceWithStorage(newEngineAlias, newStorage) // rollback attestations already on created engine instance, because this action modifies the in-memory storage. - if err := candidateEngine.Attestations.Rollback(slot); err != nil { + if err = candidateEngine.Attestations.Rollback(slot); err != nil { return nil, ierrors.Wrap(err, "error while rolling back attestations storage on candidate engine") } diff --git a/pkg/protocol/protocol.go b/pkg/protocol/protocol.go index cd52b3020..bb6dd21e0 100644 --- a/pkg/protocol/protocol.go +++ b/pkg/protocol/protocol.go @@ -2,7 +2,6 @@ package protocol import ( "context" - "fmt" "sync" "github.com/libp2p/go-libp2p/core/peer" @@ -93,19 +92,15 @@ func (p *Protocol) IssueBlock(block *model.Block) error { // Run starts the protocol. func (p *Protocol) Run(ctx context.Context) error { - fmt.Println("STARTING UP") - p.waitEngineInitialized() p.Initialized.Trigger() <-ctx.Done() - fmt.Println("SHUTTING DOWN") p.Shutdown.Trigger() p.Workers.Shutdown() p.Stopped.Trigger() - fmt.Println("SHUTTING DOWN DONE") return ctx.Err() } From 8a458ef17b58acc527fa409294a735cff3e34cf6 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 02:17:25 +0200 Subject: [PATCH 03/14] Fix: addressed linter errors --- pkg/protocol/chain.go | 8 +++++--- pkg/protocol/network_clock.go | 4 ++-- pkg/storage/prunable/prunable.go | 28 ++++++++++++++-------------- pkg/storage/storage_prunable.go | 4 +++- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 8d376c405..01690a697 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -229,11 +229,13 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (success bool) { func (c *Chain) earliestUncommittedSlot() iotago.SlotIndex { if latestVerifiedCommitment := c.LatestVerifiedCommitment.Get(); latestVerifiedCommitment != nil { return latestVerifiedCommitment.Slot() + 1 - } else if forkingPoint := c.ForkingPoint.Get(); forkingPoint != nil { + } + + if forkingPoint := c.ForkingPoint.Get(); forkingPoint != nil { return forkingPoint.Slot() - } else { - return 0 } + + return 0 } func (c *Chain) InSyncRange(slot iotago.SlotIndex) bool { diff --git a/pkg/protocol/network_clock.go b/pkg/protocol/network_clock.go index 51e879f20..33a80cb8a 100644 --- a/pkg/protocol/network_clock.go +++ b/pkg/protocol/network_clock.go @@ -22,9 +22,9 @@ func NewNetworkClock(protocol *Protocol) *NetworkClock { Variable: reactive.NewVariable[time.Time](func(currentValue time.Time, newValue time.Time) time.Time { if newValue.Before(currentValue) || newValue.After(time.Now()) { return currentValue - } else { - return newValue } + + return newValue }), } diff --git a/pkg/storage/prunable/prunable.go b/pkg/storage/prunable/prunable.go index 11bb3ebda..87e04a3ab 100644 --- a/pkg/storage/prunable/prunable.go +++ b/pkg/storage/prunable/prunable.go @@ -147,32 +147,32 @@ func (p *Prunable) Rollback(targetEpoch iotago.EpochIndex, pruningRange [2]iotag return ierrors.Wrapf(err, "failed to prune slots in range [%d, %d] from target epoch %d", pruningRange[0], pruningRange[1], targetEpoch) } - var lastPrunedEpoch iotago.EpochIndex - if lastPrunedCommitteeEpoch, err := p.rollbackCommitteeEpochs(targetEpoch, pruningRange[0]-1); err != nil { + lastPrunedCommitteeEpoch, err := p.rollbackCommitteeEpochs(targetEpoch, pruningRange[0]-1) + if err != nil { return ierrors.Wrapf(err, "failed to rollback committee epochs to target epoch %d", targetEpoch) - } else { - lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedCommitteeEpoch) } - if lastPrunedPoolStatsEpoch, err := p.poolStats.RollbackEpochs(targetEpoch); err != nil { + lastPrunedPoolStatsEpoch, err := p.poolStats.RollbackEpochs(targetEpoch) + if err != nil { return ierrors.Wrapf(err, "failed to rollback pool stats epochs to target epoch %d", targetEpoch) - } else { - lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedPoolStatsEpoch) } - if lastPrunedDecidedUpgradeSignalsEpoch, err := p.decidedUpgradeSignals.RollbackEpochs(targetEpoch); err != nil { + lastPrunedDecidedUpgradeSignalsEpoch, err := p.decidedUpgradeSignals.RollbackEpochs(targetEpoch) + if err != nil { return ierrors.Wrapf(err, "failed to rollback decided upgrade signals epochs to target epoch %d", targetEpoch) - } else { - lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedDecidedUpgradeSignalsEpoch) } - if lastPrunedPoolRewardsEpoch, err := p.poolRewards.RollbackEpochs(targetEpoch); err != nil { + lastPrunedPoolRewardsEpoch, err := p.poolRewards.RollbackEpochs(targetEpoch) + if err != nil { return ierrors.Wrapf(err, "failed to rollback pool rewards epochs to target epoch %d", targetEpoch) - } else { - lastPrunedEpoch = max(lastPrunedEpoch, lastPrunedPoolRewardsEpoch) } - for epochToPrune := targetEpoch + 1; epochToPrune <= lastPrunedEpoch; epochToPrune++ { + for epochToPrune := targetEpoch + 1; epochToPrune <= max( + lastPrunedCommitteeEpoch, + lastPrunedPoolStatsEpoch, + lastPrunedDecidedUpgradeSignalsEpoch, + lastPrunedPoolRewardsEpoch, + ); epochToPrune++ { p.prunableSlotStore.DeleteBucket(epochToPrune) } diff --git a/pkg/storage/storage_prunable.go b/pkg/storage/storage_prunable.go index eb62eac4d..51307cff1 100644 --- a/pkg/storage/storage_prunable.go +++ b/pkg/storage/storage_prunable.go @@ -40,7 +40,9 @@ func (s *Storage) Blocks(slot iotago.SlotIndex) (*slotstore.Blocks, error) { // Reset resets the component to a clean state as if it was created at the last commitment. func (s *Storage) Reset() { - s.Rollback(s.Settings().LatestCommitment().Slot()) + if err := s.Rollback(s.Settings().LatestCommitment().Slot()); err != nil { + s.errorHandler(ierrors.Wrap(err, "failed to reset prunable storage")) + } } func (s *Storage) RootBlocks(slot iotago.SlotIndex) (*slotstore.Store[iotago.BlockID, iotago.CommitmentID], error) { From fc0da15da695ca399a3790052f8c7d6c74706d16 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 02:22:21 +0200 Subject: [PATCH 04/14] Feat: addressed more linter warnings --- pkg/protocol/chain.go | 12 ------------ pkg/protocol/chain_manager.go | 7 ++++--- pkg/protocol/commitment_verifier.go | 2 +- pkg/protocol/protocol_warp_sync.go | 2 +- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 01690a697..8d13709f3 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -226,18 +226,6 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (success bool) { return success } -func (c *Chain) earliestUncommittedSlot() iotago.SlotIndex { - if latestVerifiedCommitment := c.LatestVerifiedCommitment.Get(); latestVerifiedCommitment != nil { - return latestVerifiedCommitment.Slot() + 1 - } - - if forkingPoint := c.ForkingPoint.Get(); forkingPoint != nil { - return forkingPoint.Slot() - } - - return 0 -} - func (c *Chain) InSyncRange(slot iotago.SlotIndex) bool { if latestVerifiedCommitment := c.LatestVerifiedCommitment.Get(); latestVerifiedCommitment != nil { return slot > latestVerifiedCommitment.Slot() && slot < c.SyncThreshold.Get() diff --git a/pkg/protocol/chain_manager.go b/pkg/protocol/chain_manager.go index 4eef9391d..5910232ab 100644 --- a/pkg/protocol/chain_manager.go +++ b/pkg/protocol/chain_manager.go @@ -262,11 +262,12 @@ func (c *ChainManager) publishEngineCommitments(chain *Chain) { return engine.LatestCommitment.OnUpdate(func(_, latestCommitment *model.Commitment) { for latestPublishedSlot < latestCommitment.Slot() { - if commitmentToPublish, err := engine.Storage.Commitments().Load(latestPublishedSlot + 1); err != nil { + commitmentToPublish, err := engine.Storage.Commitments().Load(latestPublishedSlot + 1) + if err != nil { panic(err) // this should never happen, but we panic to get a stack trace if it does - } else { - publishCommitment(commitmentToPublish) } + + publishCommitment(commitmentToPublish) } }) }) diff --git a/pkg/protocol/commitment_verifier.go b/pkg/protocol/commitment_verifier.go index 2fa667638..8e49f4c42 100644 --- a/pkg/protocol/commitment_verifier.go +++ b/pkg/protocol/commitment_verifier.go @@ -37,7 +37,7 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio } } - if !iotago.VerifyProof(merkleProof, iotago.Identifier(tree.Root()), commitment.RootsID()) { + if !iotago.VerifyProof(merkleProof, tree.Root(), commitment.RootsID()) { return nil, 0, ierrors.Errorf("invalid merkle proof for attestations for commitment %s", commitment.ID()) } diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 9c81c07db..6a2d8dbbd 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -110,7 +110,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo _ = acceptedTransactionIDs.Add(transactionID) // a mapdb can never return an error } - if !iotago.VerifyProof(mutationProof, iotago.Identifier(acceptedTransactionIDs.Root()), commitment.RootsID()) { + if !iotago.VerifyProof(mutationProof, acceptedTransactionIDs.Root(), commitment.RootsID()) { w.LogError("failed to verify mutations proof", "commitment", commitment.LogName(), "transactionIDs", transactionIDs, "proof", mutationProof, "fromPeer", from) return false From a62763c7d200ed0772de8e2dfb2a600b5b050d1b Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:58:11 +0200 Subject: [PATCH 05/14] Refactor: addressed linter warning --- pkg/protocol/protocol_warp_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 6a2d8dbbd..c34c6b048 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -99,7 +99,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo _ = acceptedBlocks.Add(blockID) // a mapdb can newer return an error } - if !iotago.VerifyProof(proof, iotago.Identifier(acceptedBlocks.Root()), commitment.RootsID()) { + if !iotago.VerifyProof(proof, acceptedBlocks.Root(), commitment.RootsID()) { w.LogError("failed to verify blocks proof", "commitment", commitment.LogName(), "blockIDs", blockIDs, "proof", proof, "fromPeer", from) return false From 5d5dccf16a98e89ffdc5f8752fdbbdddf99c956a Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sat, 28 Oct 2023 11:37:35 +0200 Subject: [PATCH 06/14] Fix: fixed storage layout --- pkg/storage/prunable/epochstore/epoch_kv.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/storage/prunable/epochstore/epoch_kv.go b/pkg/storage/prunable/epochstore/epoch_kv.go index 014bcacc4..85d85b2f7 100644 --- a/pkg/storage/prunable/epochstore/epoch_kv.go +++ b/pkg/storage/prunable/epochstore/epoch_kv.go @@ -22,10 +22,10 @@ func NewEpochKVStore(storeRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay return &EpochKVStore{ realm: storeRealm, - kv: lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), + kv: lo.PanicOnErr(kv.WithExtendedRealm(append(storeRealm, entriesKey))), pruningDelay: pruningDelay, - lastAccessedEpoch: kvstore.NewTypedValue(kv, kvstore.Realm{lastAccessedEpochKey}, iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes), - lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(kvstore.Realm{lastPrunedEpochKey})), storeRealm), + lastAccessedEpoch: kvstore.NewTypedValue(kv, append(storeRealm, lastAccessedEpochKey), iotago.EpochIndex.Bytes, iotago.EpochIndexFromBytes), + lastPrunedEpoch: model.NewPruningIndex(lo.PanicOnErr(kv.WithExtendedRealm(storeRealm)), kvstore.Realm{lastPrunedEpochKey}), } } From d48f4b99fc27cb926b538e1234b4d3323e7009ec Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 03:02:53 +0100 Subject: [PATCH 07/14] Fix: fixed more problems --- pkg/protocol/chain.go | 12 +- .../blockdag/inmemoryblockdag/blockdag.go | 2 +- .../engine/booker/inmemorybooker/booker.go | 2 +- pkg/protocol/engine/clock/blocktime/clock.go | 4 +- .../accountsfilter/commitmentfilter.go | 2 +- .../scheduler/drr/scheduler.go | 4 +- .../scheduler/passthrough/scheduler.go | 2 +- .../totalweightslotgadget/gadget.go | 4 +- pkg/protocol/engine/engine.go | 103 +++++++++++------- .../engine/filter/blockfilter/filter.go | 2 +- pkg/protocol/engine/ledger/ledger/ledger.go | 2 +- .../notarization/slotnotarization/manager.go | 22 +++- pkg/protocol/engine/tipmanager/v1/provider.go | 4 +- .../engine/tipselection/v1/provider.go | 4 +- pkg/protocol/engine_manager.go | 8 +- pkg/protocol/protocol.go | 2 +- pkg/protocol/protocol_blocks.go | 4 +- pkg/protocol/protocol_warp_sync.go | 25 ++++- .../snapshotcreator/snapshotcreator.go | 8 +- .../sybilprotection/seatmanager/poa/poa.go | 2 +- .../sybilprotectionv1/sybilprotection.go | 4 +- pkg/retainer/retainer/retainer.go | 2 +- pkg/storage/prunable/epochstore/epoch_kv.go | 3 +- pkg/storage/prunable/epochstore/store.go | 2 +- pkg/storage/prunable/prunable.go | 4 +- pkg/tests/protocol_engine_switching_test.go | 16 +-- pkg/testsuite/mock/blockissuer.go | 2 +- pkg/testsuite/mock/node.go | 2 - 28 files changed, 148 insertions(+), 105 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 8d13709f3..c67244197 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -146,9 +146,9 @@ func NewChain(protocol *Protocol) *Chain { c.Logger = protocol.NewEntityLogger("Chain", c.IsEvicted, func(entityLogger log.Logger) { c.WarpSync.LogUpdates(entityLogger, log.LevelInfo, "WarpSync") - c.NetworkClockSlot.LogUpdates(entityLogger, log.LevelDebug, "NetworkClockSlot") - c.WarpSyncThreshold.LogUpdates(entityLogger, log.LevelDebug, "WarpSyncThreshold") - c.OutOfSyncThreshold.LogUpdates(entityLogger, log.LevelDebug, "OutOfSyncThreshold") + c.NetworkClockSlot.LogUpdates(entityLogger, log.LevelTrace, "NetworkClockSlot") + c.WarpSyncThreshold.LogUpdates(entityLogger, log.LevelTrace, "WarpSyncThreshold") + c.OutOfSyncThreshold.LogUpdates(entityLogger, log.LevelTrace, "OutOfSyncThreshold") c.ForkingPoint.LogUpdates(entityLogger, log.LevelTrace, "ForkingPoint", (*Commitment).LogName) c.ClaimedWeight.LogUpdates(entityLogger, log.LevelTrace, "ClaimedWeight") c.AttestedWeight.LogUpdates(entityLogger, log.LevelTrace, "AttestedWeight") @@ -157,6 +157,7 @@ func NewChain(protocol *Protocol) *Chain { c.LatestVerifiedCommitment.LogUpdates(entityLogger, log.LevelDebug, "LatestVerifiedCommitment", (*Commitment).LogName) c.VerifyAttestations.LogUpdates(entityLogger, log.LevelTrace, "VerifyAttestations") c.VerifyState.LogUpdates(entityLogger, log.LevelDebug, "VerifyState") + c.SpawnedEngine.LogUpdates(entityLogger, log.LevelDebug, "SpawnedEngine", (*engine.Engine).LogName) }) var unsubscribe func() @@ -168,11 +169,6 @@ func NewChain(protocol *Protocol) *Chain { go func() { if warpSync { unsubscribe = c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex) bool { - latestVerifiedNil := latestVerifiedCommitment != nil - belowWarpSync := latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < warpSyncThreshold - - c.LogError("WarpSync", "latestVerifiedNil", latestVerifiedNil, "belowWarpSync", belowWarpSync) - return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < warpSyncThreshold }, c.LatestVerifiedCommitment, c.WarpSyncThreshold)) } else { diff --git a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go index f155b2c9b..713b20c8c 100644 --- a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go +++ b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go @@ -43,7 +43,7 @@ func NewProvider(opts ...options.Option[BlockDAG]) module.Provider[*engine.Engin return module.Provide(func(e *engine.Engine) blockdag.BlockDAG { b := New(e.Workers.CreateGroup("BlockDAG"), int(e.Storage.Settings().APIProvider().CommittedAPI().ProtocolParameters().MaxCommittableAge())*2, e.EvictionState, e.BlockCache, e.ErrorHandler("blockdag"), opts...) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { wp := b.workers.CreatePool("BlockDAG.Attach", workerpool.WithWorkerCount(2)) e.Events.Filter.BlockPreAllowed.Hook(func(block *model.Block) { diff --git a/pkg/protocol/engine/booker/inmemorybooker/booker.go b/pkg/protocol/engine/booker/inmemorybooker/booker.go index 51f53c52d..90dd32e1e 100644 --- a/pkg/protocol/engine/booker/inmemorybooker/booker.go +++ b/pkg/protocol/engine/booker/inmemorybooker/booker.go @@ -38,7 +38,7 @@ type Booker struct { func NewProvider(opts ...options.Option[Booker]) module.Provider[*engine.Engine, booker.Booker] { return module.Provide(func(e *engine.Engine) booker.Booker { b := New(e, e.BlockCache, e.ErrorHandler("booker"), opts...) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { b.ledger = e.Ledger b.ledger.HookConstructed(func() { b.conflictDAG = b.ledger.ConflictDAG() diff --git a/pkg/protocol/engine/clock/blocktime/clock.go b/pkg/protocol/engine/clock/blocktime/clock.go index 3c2279d01..506387df0 100644 --- a/pkg/protocol/engine/clock/blocktime/clock.go +++ b/pkg/protocol/engine/clock/blocktime/clock.go @@ -40,7 +40,7 @@ func NewProvider(opts ...options.Option[Clock]) module.Provider[*engine.Engine, confirmedTime: NewRelativeTime(), workerPool: e.Workers.CreatePool("Clock", workerpool.WithWorkerCount(1), workerpool.WithCancelPendingTasksOnShutdown(true), workerpool.WithPanicOnSubmitAfterShutdown(true)), }, opts, func(c *Clock) { - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { latestCommitmentIndex := e.Storage.Settings().LatestCommitment().Slot() c.acceptedTime.Set(e.APIForSlot(latestCommitmentIndex).TimeProvider().SlotEndTime(latestCommitmentIndex)) @@ -72,7 +72,7 @@ func NewProvider(opts ...options.Option[Clock]) module.Provider[*engine.Engine, )) }) - e.HookStopped(c.TriggerStopped) + e.Stopped.OnTrigger(c.TriggerStopped) }, (*Clock).TriggerConstructed) }) } diff --git a/pkg/protocol/engine/commitmentfilter/accountsfilter/commitmentfilter.go b/pkg/protocol/engine/commitmentfilter/accountsfilter/commitmentfilter.go index 076caf807..f54b7a027 100644 --- a/pkg/protocol/engine/commitmentfilter/accountsfilter/commitmentfilter.go +++ b/pkg/protocol/engine/commitmentfilter/accountsfilter/commitmentfilter.go @@ -32,7 +32,7 @@ type CommitmentFilter struct { func NewProvider(opts ...options.Option[CommitmentFilter]) module.Provider[*engine.Engine, commitmentfilter.CommitmentFilter] { return module.Provide(func(e *engine.Engine) commitmentfilter.CommitmentFilter { c := New(e, opts...) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { c.commitmentFunc = e.Storage.Commitments().Load c.accountRetrieveFunc = e.Ledger.Account diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go index cbb249c57..d50622d61 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/drr/scheduler.go @@ -55,7 +55,7 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi s.errorHandler = e.ErrorHandler("scheduler") s.basicBuffer = NewBufferQueue() - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { s.latestCommittedSlot = func() iotago.SlotIndex { return e.Storage.Settings().LatestCommitment().Slot() } @@ -112,7 +112,7 @@ func NewProvider(opts ...options.Option[Scheduler]) module.Provider[*engine.Engi s.removeIssuer(accountID, ierrors.New("account destroyed")) }) - e.HookInitialized(s.Start) + e.Initialized.OnTrigger(s.Start) }) return s diff --git a/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go b/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go index 1f46884b3..99f9872e4 100644 --- a/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go +++ b/pkg/protocol/engine/congestioncontrol/scheduler/passthrough/scheduler.go @@ -17,7 +17,7 @@ type Scheduler struct { func NewProvider() module.Provider[*engine.Engine, scheduler.Scheduler] { return module.Provide(func(e *engine.Engine) scheduler.Scheduler { s := New() - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { e.Events.Scheduler.LinkTo(s.events) s.TriggerConstructed() e.Events.Booker.BlockBooked.Hook(func(block *blocks.Block) { diff --git a/pkg/protocol/engine/consensus/slotgadget/totalweightslotgadget/gadget.go b/pkg/protocol/engine/consensus/slotgadget/totalweightslotgadget/gadget.go index 75553f733..e6a518023 100644 --- a/pkg/protocol/engine/consensus/slotgadget/totalweightslotgadget/gadget.go +++ b/pkg/protocol/engine/consensus/slotgadget/totalweightslotgadget/gadget.go @@ -51,7 +51,7 @@ func NewProvider(opts ...options.Option[Gadget]) module.Provider[*engine.Engine, e.Events.SlotGadget.LinkTo(g.events) g.workers = e.Workers.CreateGroup("SlotGadget") - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { g.seatManager = e.SybilProtection.SeatManager() g.TriggerConstructed() @@ -64,7 +64,7 @@ func NewProvider(opts ...options.Option[Gadget]) module.Provider[*engine.Engine, } } - e.HookInitialized(func() { + e.Initialized.OnTrigger(func() { // Can't use setter here as it has a side effect. func() { g.mutex.Lock() diff --git a/pkg/protocol/engine/engine.go b/pkg/protocol/engine/engine.go index 7733ba618..bc800d492 100644 --- a/pkg/protocol/engine/engine.go +++ b/pkg/protocol/engine/engine.go @@ -13,6 +13,8 @@ import ( "github.com/iotaledger/hive.go/ds/reactive" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" + "github.com/iotaledger/hive.go/log" + "github.com/iotaledger/hive.go/runtime/debug" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/options" @@ -83,12 +85,12 @@ type Engine struct { optsSnapshotDepth int optsBlockRequester []options.Option[eventticker.EventTicker[iotago.SlotIndex, iotago.BlockID]] - module.Module + *module.ReactiveModule } func New( + logger log.Logger, workers *workerpool.Group, - errorHandler func(error), storageInstance *storage.Storage, filterProvider module.Provider[*Engine, filter.Filter], commitmentFilterProvider module.Provider[*Engine, commitmentfilter.CommitmentFilter], @@ -121,11 +123,16 @@ func New( RootCommitment: reactive.NewVariable[*model.Commitment](), LatestCommitment: reactive.NewVariable[*model.Commitment](), Workers: workers, - errorHandler: errorHandler, optsSnapshotPath: "snapshot.bin", optsSnapshotDepth: 5, }, opts, func(e *Engine) { + e.ReactiveModule = e.initReactiveModule(logger) + + e.errorHandler = func(err error) { + e.LogTrace("engine error", "err", err) + } + // Import the settings from the snapshot file if needed. if importSnapshot = !e.Storage.Settings().IsSnapshotImported() && e.optsSnapshotPath != ""; importSnapshot { file, fileErr = os.Open(e.optsSnapshotPath) @@ -169,16 +176,16 @@ func New( (*Engine).setupBlockRequester, (*Engine).setupPruning, (*Engine).acceptanceHandler, - (*Engine).TriggerConstructed, func(e *Engine) { + e.Constructed.Trigger() + // Make sure that we have the protocol parameters for the latest supported iota.go protocol version of the software. // If not the user needs to update the protocol parameters file. // This can only happen after a user updated the node version and the new protocol version is not yet active. if _, err := e.APIForVersion(iotago.LatestProtocolVersion()); err != nil { panic(ierrors.Wrap(err, "no protocol parameters for latest protocol version found")) } - }, - func(e *Engine) { + // Import the rest of the snapshot if needed. if importSnapshot { if err := e.ImportContents(file); err != nil { @@ -215,9 +222,11 @@ func New( panic(ierrors.Wrap(err, "failed to restore upgrade orchestrator from disk")) } } + + e.Reset() + + e.Initialized.Trigger() }, - (*Engine).Reset, - (*Engine).TriggerInitialized, ) } @@ -228,7 +237,7 @@ func (e *Engine) ProcessBlockFromPeer(block *model.Block, source peer.ID) { // Reset resets the component to a clean state as if it was created at the last commitment. func (e *Engine) Reset() { - e.Workers.WaitChildren() + e.LogDebug("resetting engine", "slot", e.Storage.Settings().LatestCommitment().Slot(), "stack-trace", debug.StackTrace(false, 0)) e.BlockRequester.Clear() e.Storage.Reset() @@ -256,32 +265,28 @@ func (e *Engine) Reset() { e.Clock.Reset(latestCommittedTime) } -func (e *Engine) Shutdown() { - if !e.WasShutdown() { - e.TriggerShutdown() - - e.BlockRequester.Shutdown() - e.Attestations.Shutdown() - e.SyncManager.Shutdown() - e.Notarization.Shutdown() - e.Booker.Shutdown() - e.Ledger.Shutdown() - e.BlockDAG.Shutdown() - e.BlockGadget.Shutdown() - e.SlotGadget.Shutdown() - e.Clock.Shutdown() - e.SybilProtection.Shutdown() - e.UpgradeOrchestrator.Shutdown() - e.TipManager.Shutdown() - e.Filter.Shutdown() - e.CommitmentFilter.Shutdown() - e.Scheduler.Shutdown() - e.Retainer.Shutdown() - e.Workers.Shutdown() - e.Storage.Shutdown() - - e.TriggerStopped() - } +func (e *Engine) shutdown() { + e.BlockRequester.Shutdown() + e.Attestations.Shutdown() + e.SyncManager.Shutdown() + e.Notarization.Shutdown() + e.Booker.Shutdown() + e.Ledger.Shutdown() + e.BlockDAG.Shutdown() + e.BlockGadget.Shutdown() + e.SlotGadget.Shutdown() + e.Clock.Shutdown() + e.SybilProtection.Shutdown() + e.UpgradeOrchestrator.Shutdown() + e.TipManager.Shutdown() + e.Filter.Shutdown() + e.CommitmentFilter.Shutdown() + e.Scheduler.Shutdown() + e.Retainer.Shutdown() + e.Workers.Shutdown() + e.Storage.Shutdown() + + e.Stopped.Trigger() } func (e *Engine) BlockFromCache(id iotago.BlockID) (*blocks.Block, bool) { @@ -555,14 +560,14 @@ func (e *Engine) initRootCommitment() { }) } - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { unsubscribe := e.Events.SlotGadget.SlotFinalized.Hook(updateRootCommitment).Unhook - e.HookInitialized(func() { + e.Initialized.OnTrigger(func() { updateRootCommitment(e.Storage.Settings().LatestFinalizedSlot()) }) - e.HookShutdown(unsubscribe) + e.Shutdown.OnTrigger(unsubscribe) }) } @@ -573,15 +578,31 @@ func (e *Engine) initLatestCommitment() { }) } - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { unsubscribe := e.Events.Notarization.LatestCommitmentUpdated.Hook(updateLatestCommitment).Unhook - e.HookInitialized(func() { + e.Initialized.OnTrigger(func() { updateLatestCommitment(e.Storage.Settings().LatestCommitment()) }) - e.HookShutdown(unsubscribe) + e.Shutdown.OnTrigger(unsubscribe) + }) +} + +func (e *Engine) initReactiveModule(logger log.Logger) (reactiveModule *module.ReactiveModule) { + stopLogging := reactive.NewEvent() + + reactiveModule = module.NewReactiveModule(logger.NewEntityLogger("Engine", stopLogging, func(engineLogger log.Logger) { + // TODO: setup reactive logging + })) + + reactiveModule.Shutdown.OnTrigger(func() { + stopLogging.Trigger() + + e.shutdown() }) + + return reactiveModule } // endregion /////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/pkg/protocol/engine/filter/blockfilter/filter.go b/pkg/protocol/engine/filter/blockfilter/filter.go index 17adf9669..351f92105 100644 --- a/pkg/protocol/engine/filter/blockfilter/filter.go +++ b/pkg/protocol/engine/filter/blockfilter/filter.go @@ -39,7 +39,7 @@ func NewProvider(opts ...options.Option[Filter]) module.Provider[*engine.Engine, f := New(e, opts...) f.TriggerConstructed() - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { e.Events.Filter.LinkTo(f.events) e.SybilProtection.HookInitialized(func() { f.committeeFunc = e.SybilProtection.SeatManager().Committee diff --git a/pkg/protocol/engine/ledger/ledger/ledger.go b/pkg/protocol/engine/ledger/ledger/ledger.go index a29a624ba..584ec3f42 100644 --- a/pkg/protocol/engine/ledger/ledger/ledger.go +++ b/pkg/protocol/engine/ledger/ledger/ledger.go @@ -61,7 +61,7 @@ func NewProvider() module.Provider[*engine.Engine, ledger.Ledger] { e.ErrorHandler("ledger"), ) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { e.Events.Ledger.LinkTo(l.events) l.conflictDAG = conflictdagv1.New[iotago.TransactionID, mempool.StateID, ledger.BlockVoteRank](l.sybilProtection.SeatManager().OnlineCommittee().Size) e.Events.ConflictDAG.LinkTo(l.conflictDAG.Events()) diff --git a/pkg/protocol/engine/notarization/slotnotarization/manager.go b/pkg/protocol/engine/notarization/slotnotarization/manager.go index ee36ae9c4..29993ce3e 100644 --- a/pkg/protocol/engine/notarization/slotnotarization/manager.go +++ b/pkg/protocol/engine/notarization/slotnotarization/manager.go @@ -4,6 +4,7 @@ import ( "time" "github.com/iotaledger/hive.go/ierrors" + "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/syncutils" @@ -42,16 +43,21 @@ type Manager struct { commitmentMutex syncutils.Mutex + log.Logger + module.Module } func NewProvider() module.Provider[*engine.Engine, notarization.Notarization] { return module.Provide(func(e *engine.Engine) notarization.Notarization { - m := NewManager(e.Workers.CreateGroup("NotarizationManager"), e.ErrorHandler("notarization")) + logger, shutdownLogger := e.NewChildLogger("NotarizationManager") + + m := NewManager(logger, e.Workers.CreateGroup("NotarizationManager"), e.ErrorHandler("notarization")) + m.HookShutdown(shutdownLogger) m.apiProvider = e - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { m.storage = e.Storage m.acceptedTimeFunc = e.Clock.Accepted().Time @@ -78,12 +84,15 @@ func NewProvider() module.Provider[*engine.Engine, notarization.Notarization] { m.TriggerConstructed() }) + e.Shutdown.OnTrigger(m.Shutdown) + return m }) } -func NewManager(workers *workerpool.Group, errorHandler func(error)) *Manager { +func NewManager(logger log.Logger, workers *workerpool.Group, errorHandler func(error)) *Manager { return &Manager{ + Logger: logger, events: notarization.NewEvents(), workers: workers, errorHandler: errorHandler, @@ -91,12 +100,15 @@ func NewManager(workers *workerpool.Group, errorHandler func(error)) *Manager { } func (m *Manager) Shutdown() { - m.TriggerStopped() + m.TriggerShutdown() + // Alternative 2 if m.acceptedBlockProcessedDetach != nil { m.acceptedBlockProcessedDetach() } m.workers.Shutdown() + + m.TriggerStopped() } // tryCommitUntil tries to create slot commitments until the new provided acceptance time. @@ -236,6 +248,8 @@ func (m *Manager) createCommitment(slot iotago.SlotIndex) (*model.Commitment, er rmc, ) + m.LogTrace("Committing", "commitment", newCommitment, "roots ", roots) + newModelCommitment, err := model.CommitmentFromCommitment(newCommitment, apiForSlot, serix.WithValidation()) if err != nil { return nil, ierrors.Wrapf(err, "failed to create model commitment for commitment %s", newCommitment.MustID()) diff --git a/pkg/protocol/engine/tipmanager/v1/provider.go b/pkg/protocol/engine/tipmanager/v1/provider.go index f44ac9d0f..eb859133d 100644 --- a/pkg/protocol/engine/tipmanager/v1/provider.go +++ b/pkg/protocol/engine/tipmanager/v1/provider.go @@ -14,7 +14,7 @@ func NewProvider() module.Provider[*engine.Engine, tipmanager.TipManager] { return module.Provide(func(e *engine.Engine) tipmanager.TipManager { t := New(e.BlockCache.Block) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { tipWorker := e.Workers.CreatePool("AddTip", workerpool.WithWorkerCount(2)) e.Events.Scheduler.BlockScheduled.Hook(lo.Void(t.AddBlock), event.WithWorkerPool(tipWorker)) e.Events.Scheduler.BlockSkipped.Hook(lo.Void(t.AddBlock), event.WithWorkerPool(tipWorker)) @@ -25,7 +25,7 @@ func NewProvider() module.Provider[*engine.Engine, tipmanager.TipManager] { t.TriggerInitialized() }) - e.HookShutdown(func() { + e.Shutdown.OnTrigger(func() { t.TriggerShutdown() t.TriggerStopped() }) diff --git a/pkg/protocol/engine/tipselection/v1/provider.go b/pkg/protocol/engine/tipselection/v1/provider.go index ad9bb1d1b..d320e4500 100644 --- a/pkg/protocol/engine/tipselection/v1/provider.go +++ b/pkg/protocol/engine/tipselection/v1/provider.go @@ -17,7 +17,7 @@ func NewProvider(opts ...options.Option[TipSelection]) module.Provider[*engine.E return module.Provide(func(e *engine.Engine) tipselection.TipSelection { t := New(opts...) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { // wait for submodules to be constructed (so all of their properties are available) module.OnAllConstructed(func() { t.Construct(e.TipManager, e.Ledger.ConflictDAG(), e.Ledger.MemPool().TransactionMetadata, e.EvictionState.LatestRootBlocks, DynamicLivenessThreshold(e.SybilProtection.SeatManager().OnlineCommittee().Size)) @@ -28,7 +28,7 @@ func NewProvider(opts ...options.Option[TipSelection]) module.Provider[*engine.E }, e.TipManager, e.Ledger, e.SybilProtection) }) - e.HookShutdown(t.Shutdown) + e.Shutdown.OnTrigger(t.Shutdown) return t }) diff --git a/pkg/protocol/engine_manager.go b/pkg/protocol/engine_manager.go index 42195dba6..2ca1d8349 100644 --- a/pkg/protocol/engine_manager.go +++ b/pkg/protocol/engine_manager.go @@ -186,11 +186,7 @@ func (e *EngineManager) loadEngineInstanceFromSnapshot(engineAlias string, snaps } func (e *EngineManager) loadEngineInstanceWithStorage(engineAlias string, storage *storage.Storage) *engine.Engine { - errorHandler := func(err error) { - e.protocol.LogError("engine error", "err", err, "name", engineAlias[0:8]) - } - - return engine.New(e.protocol.Workers.CreateGroup(engineAlias), errorHandler, storage, e.protocol.Options.FilterProvider, e.protocol.Options.CommitmentFilterProvider, e.protocol.Options.BlockDAGProvider, e.protocol.Options.BookerProvider, e.protocol.Options.ClockProvider, e.protocol.Options.BlockGadgetProvider, e.protocol.Options.SlotGadgetProvider, e.protocol.Options.SybilProtectionProvider, e.protocol.Options.NotarizationProvider, e.protocol.Options.AttestationProvider, e.protocol.Options.LedgerProvider, e.protocol.Options.SchedulerProvider, e.protocol.Options.TipManagerProvider, e.protocol.Options.TipSelectionProvider, e.protocol.Options.RetainerProvider, e.protocol.Options.UpgradeOrchestratorProvider, e.protocol.Options.SyncManagerProvider, e.protocol.Options.EngineOptions...) + return engine.New(e.protocol.Logger, e.protocol.Workers.CreateGroup(engineAlias), storage, e.protocol.Options.FilterProvider, e.protocol.Options.CommitmentFilterProvider, e.protocol.Options.BlockDAGProvider, e.protocol.Options.BookerProvider, e.protocol.Options.ClockProvider, e.protocol.Options.BlockGadgetProvider, e.protocol.Options.SlotGadgetProvider, e.protocol.Options.SybilProtectionProvider, e.protocol.Options.NotarizationProvider, e.protocol.Options.AttestationProvider, e.protocol.Options.LedgerProvider, e.protocol.Options.SchedulerProvider, e.protocol.Options.TipManagerProvider, e.protocol.Options.TipSelectionProvider, e.protocol.Options.RetainerProvider, e.protocol.Options.UpgradeOrchestratorProvider, e.protocol.Options.SyncManagerProvider, e.protocol.Options.EngineOptions...) } func (e *EngineManager) syncMainEngineFromMainChain() (unsubscribe func()) { @@ -230,7 +226,7 @@ func (e *EngineManager) injectEngineInstances() (unsubscribe func()) { }(); err != nil { e.LogError("failed to create new engine instance", "err", err) } else { - e.protocol.Network.OnShutdown(newEngine.Shutdown) + e.protocol.Network.OnShutdown(func() { newEngine.Shutdown.Trigger() }) chain.SpawnedEngine.Set(newEngine) } diff --git a/pkg/protocol/protocol.go b/pkg/protocol/protocol.go index bb6dd21e0..e0f1546c8 100644 --- a/pkg/protocol/protocol.go +++ b/pkg/protocol/protocol.go @@ -110,7 +110,7 @@ func (p *Protocol) waitEngineInitialized() { waitInitialized.Add(1) p.MainEngine.OnUpdateOnce(func(_, engine *engine.Engine) { - engine.HookInitialized(waitInitialized.Done) + engine.Initialized.OnTrigger(waitInitialized.Done) }) waitInitialized.Wait() } diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 90f11e5f5..5a50117d7 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -38,8 +38,6 @@ func NewBlocksProtocol(protocol *Protocol) *BlocksProtocol { return } - protocol.LogError("REPLAYING DROPPED BLOCKS", "slot", commitment.ID().Slot()) - for _, droppedBlock := range b.droppedBlocksBuffer.GetValues(commitment.ID()) { b.ProcessResponse(droppedBlock.A, droppedBlock.B) } @@ -51,7 +49,7 @@ func NewBlocksProtocol(protocol *Protocol) *BlocksProtocol { chain.Engine.OnUpdate(func(_, engine *engine.Engine) { unsubscribe := engine.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook - engine.HookShutdown(unsubscribe) + engine.Shutdown.OnTrigger(unsubscribe) }) }) }) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index c34c6b048..395a57fa2 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -28,7 +28,7 @@ func NewWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { c := &WarpSyncProtocol{ Logger: lo.Return1(protocol.Logger.NewChildLogger("WarpSync")), protocol: protocol, - workerPool: protocol.Workers.CreatePool("WarpSync"), + workerPool: protocol.Workers.CreatePool("WarpSync", workerpool.WithWorkerCount(1)), ticker: eventticker.New[iotago.SlotIndex, iotago.CommitmentID](), } @@ -80,6 +80,19 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } + chain := commitment.Chain.Get() + if chain == nil { + w.LogTrace("failed to get chain for response", "commitment", commitment.LogName(), "fromPeer", from) + + return + } + + if !chain.WarpSync.Get() { + w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) + + return + } + targetEngine := commitment.Engine.Get() if targetEngine == nil { w.LogDebug("failed to get target engine for response", "commitment", commitment.LogName()) @@ -118,6 +131,14 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo w.ticker.StopTicker(commitmentID) + targetEngine.Workers.WaitChildren() + + if !chain.WarpSync.Get() { + w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) + + return false + } + targetEngine.Reset() // If the engine is "dirty" we need to restore the state of the engine to the state of the chain commitment. // As we already decided to switch and sync to this chain we should make sure that processing the blocks from the commitment @@ -134,8 +155,6 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo var bookedBlocks atomic.Uint32 var notarizedBlocks atomic.Uint32 - w.LogError("TOTAL BLOCKS IN RESPONSE", "totalBlocks", totalBlocks) - forceCommitmentFunc := func() { // 3. Force commitment of the slot producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) diff --git a/pkg/protocol/snapshotcreator/snapshotcreator.go b/pkg/protocol/snapshotcreator/snapshotcreator.go index eae10d089..2976f803f 100644 --- a/pkg/protocol/snapshotcreator/snapshotcreator.go +++ b/pkg/protocol/snapshotcreator/snapshotcreator.go @@ -7,6 +7,7 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" + "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/runtime/workerpool" "github.com/iotaledger/iota-core/pkg/core/account" @@ -88,8 +89,9 @@ func CreateSnapshot(opts ...options.Option[Options]) error { } } - engineInstance := engine.New(workers.CreateGroup("Engine"), - errorHandler, + engineInstance := engine.New( + log.NewLogger("snapshot-creator"), + workers.CreateGroup("Engine"), s, blockfilter.NewProvider(), accountsfilter.NewProvider(), @@ -110,7 +112,7 @@ func CreateSnapshot(opts ...options.Option[Options]) error { trivialsyncmanager.NewProvider(), engine.WithSnapshotPath(""), // magic to disable loading snapshot ) - defer engineInstance.Shutdown() + defer engineInstance.Shutdown.Trigger() for blockID, commitmentID := range opt.RootBlocks { engineInstance.EvictionState.AddRootBlock(blockID, commitmentID) diff --git a/pkg/protocol/sybilprotection/seatmanager/poa/poa.go b/pkg/protocol/sybilprotection/seatmanager/poa/poa.go index 8d5de21dd..97633d0db 100644 --- a/pkg/protocol/sybilprotection/seatmanager/poa/poa.go +++ b/pkg/protocol/sybilprotection/seatmanager/poa/poa.go @@ -55,7 +55,7 @@ func NewProvider(opts ...options.Option[SeatManager]) module.Provider[*engine.En }, opts, func(s *SeatManager) { e.Events.SeatManager.LinkTo(s.events) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { s.clock = e.Clock s.TriggerConstructed() diff --git a/pkg/protocol/sybilprotection/sybilprotectionv1/sybilprotection.go b/pkg/protocol/sybilprotection/sybilprotectionv1/sybilprotection.go index f07da1215..d8003a0aa 100644 --- a/pkg/protocol/sybilprotection/sybilprotectionv1/sybilprotection.go +++ b/pkg/protocol/sybilprotection/sybilprotectionv1/sybilprotection.go @@ -56,7 +56,7 @@ func NewProvider(opts ...options.Option[SybilProtection]) module.Provider[*engin func(o *SybilProtection) { o.seatManager = o.optsSeatManagerProvider(e) - e.HookConstructed(func() { + e.Constructed.OnTrigger(func() { o.ledger = e.Ledger o.errHandler = e.ErrorHandler("SybilProtection") @@ -74,7 +74,7 @@ func NewProvider(opts ...options.Option[SybilProtection]) module.Provider[*engin // When the engine is triggered initialized, snapshot has been read or database has been initialized properly, // so the committee should be available in the performance manager. - e.HookInitialized(func() { + e.Initialized.OnTrigger(func() { // Make sure that the sybil protection knows about the committee of the current epoch // (according to the latest committed slot), and potentially the next selected // committee if we have one. diff --git a/pkg/retainer/retainer/retainer.go b/pkg/retainer/retainer/retainer.go index 4f244e092..8f32d6705 100644 --- a/pkg/retainer/retainer/retainer.go +++ b/pkg/retainer/retainer/retainer.go @@ -87,7 +87,7 @@ func NewProvider() module.Provider[*engine.Engine, retainer.Retainer] { r.RetainBlockFailure(b.ID(), apimodels.BlockFailureDroppedDueToCongestion) }) - e.HookInitialized(func() { + e.Initialized.OnTrigger(func() { e.Ledger.MemPool().OnSignedTransactionAttached(func(signedTransactionMetadata mempool.SignedTransactionMetadata) { attachment := signedTransactionMetadata.Attachments()[0] diff --git a/pkg/storage/prunable/epochstore/epoch_kv.go b/pkg/storage/prunable/epochstore/epoch_kv.go index 85d85b2f7..1a1dff4d5 100644 --- a/pkg/storage/prunable/epochstore/epoch_kv.go +++ b/pkg/storage/prunable/epochstore/epoch_kv.go @@ -19,7 +19,6 @@ type EpochKVStore struct { } func NewEpochKVStore(storeRealm kvstore.Realm, kv kvstore.KVStore, pruningDelay iotago.EpochIndex) *EpochKVStore { - return &EpochKVStore{ realm: storeRealm, kv: lo.PanicOnErr(kv.WithExtendedRealm(append(storeRealm, entriesKey))), @@ -110,7 +109,7 @@ func (e *EpochKVStore) RollbackEpochs(epoch iotago.EpochIndex) (lastPrunedEpoch return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed epoch") } - for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + for epochToPrune := epoch; epochToPrune <= lastAccessedEpoch; epochToPrune++ { if err = e.DeleteEpoch(epochToPrune); err != nil { return epochToPrune, ierrors.Wrapf(err, "error while deleting epoch %d", epochToPrune) } diff --git a/pkg/storage/prunable/epochstore/store.go b/pkg/storage/prunable/epochstore/store.go index a8ef66e68..fa6be918e 100644 --- a/pkg/storage/prunable/epochstore/store.go +++ b/pkg/storage/prunable/epochstore/store.go @@ -162,7 +162,7 @@ func (s *Store[V]) RollbackEpochs(epoch iotago.EpochIndex) (lastPrunedEpoch iota return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed epoch") } - for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + for epochToPrune := epoch; epochToPrune <= lastAccessedEpoch; epochToPrune++ { if err = s.DeleteEpoch(epochToPrune); err != nil { return epochToPrune, ierrors.Wrapf(err, "error while deleting epoch %d", epochToPrune) } diff --git a/pkg/storage/prunable/prunable.go b/pkg/storage/prunable/prunable.go index 87e04a3ab..edb3ccfb1 100644 --- a/pkg/storage/prunable/prunable.go +++ b/pkg/storage/prunable/prunable.go @@ -147,7 +147,7 @@ func (p *Prunable) Rollback(targetEpoch iotago.EpochIndex, pruningRange [2]iotag return ierrors.Wrapf(err, "failed to prune slots in range [%d, %d] from target epoch %d", pruningRange[0], pruningRange[1], targetEpoch) } - lastPrunedCommitteeEpoch, err := p.rollbackCommitteeEpochs(targetEpoch, pruningRange[0]-1) + lastPrunedCommitteeEpoch, err := p.rollbackCommitteeEpochs(targetEpoch+1, pruningRange[0]-1) if err != nil { return ierrors.Wrapf(err, "failed to rollback committee epochs to target epoch %d", targetEpoch) } @@ -185,7 +185,7 @@ func (p *Prunable) rollbackCommitteeEpochs(epoch iotago.EpochIndex, targetSlot i return lastAccessedEpoch, ierrors.Wrap(err, "failed to get last accessed committee epoch") } - for epochToPrune := epoch + 1; epochToPrune <= lastAccessedEpoch; epochToPrune++ { + for epochToPrune := epoch; epochToPrune <= lastAccessedEpoch; epochToPrune++ { if shouldRollback, rollbackErr := p.shouldRollbackCommittee(epochToPrune, targetSlot); rollbackErr != nil { return epochToPrune, ierrors.Wrapf(rollbackErr, "error while checking if committee for epoch %d should be rolled back", epochToPrune) } else if shouldRollback { diff --git a/pkg/tests/protocol_engine_switching_test.go b/pkg/tests/protocol_engine_switching_test.go index fc974328a..70adb8d63 100644 --- a/pkg/tests/protocol_engine_switching_test.go +++ b/pkg/tests/protocol_engine_switching_test.go @@ -108,15 +108,15 @@ func TestProtocol_EngineSwitching(t *testing.T) { ts.Run(false, nodeOptions) - node0.Protocol.SetLogLevel(log.LevelInfo) - node1.Protocol.SetLogLevel(log.LevelInfo) - node2.Protocol.SetLogLevel(log.LevelInfo) - node3.Protocol.SetLogLevel(log.LevelInfo) - node4.Protocol.SetLogLevel(log.LevelInfo) - node5.Protocol.SetLogLevel(log.LevelInfo) + node0.Protocol.SetLogLevel(log.LevelError) + node1.Protocol.SetLogLevel(log.LevelError) + node2.Protocol.SetLogLevel(log.LevelError) + node3.Protocol.SetLogLevel(log.LevelError) + node4.Protocol.SetLogLevel(log.LevelError) + node5.Protocol.SetLogLevel(log.LevelError) node6.Protocol.SetLogLevel(log.LevelDebug) - node7.Protocol.SetLogLevel(log.LevelDebug) - node8.Protocol.SetLogLevel(log.LevelDebug) + node7.Protocol.SetLogLevel(log.LevelError) + node8.Protocol.SetLogLevel(log.LevelError) expectedCommittee := []iotago.AccountID{ node0.Validator.AccountID, diff --git a/pkg/testsuite/mock/blockissuer.go b/pkg/testsuite/mock/blockissuer.go index 9e7c673da..7ed6b63d1 100644 --- a/pkg/testsuite/mock/blockissuer.go +++ b/pkg/testsuite/mock/blockissuer.go @@ -188,7 +188,7 @@ func (i *BlockIssuer) IssueValidationBlock(ctx context.Context, alias string, no validationBlock, _ := block.ValidationBlock() - fmt.Printf("Issued ValidationBlock: %s - slot %d - commitment %s %d - latest finalized slot %d - version: %d - highestSupportedVersion: %d, hash: %s\n", block.ID(), block.ID().Slot(), block.SlotCommitmentID(), block.SlotCommitmentID().Slot(), block.ProtocolBlock().LatestFinalizedSlot, block.ProtocolBlock().ProtocolVersion, validationBlock.HighestSupportedVersion, validationBlock.ProtocolParametersHash) + node.Protocol.MainEngine.Get().LogTrace("issued validation block", "block", validationBlock, "blockID", block.ID(), "slot", block.ID().Slot(), "commitment", block.SlotCommitmentID(), "latestFinalizedSlot", block.ProtocolBlock().LatestFinalizedSlot, "version", block.ProtocolBlock().ProtocolVersion, "highestSupportedVersion", validationBlock.HighestSupportedVersion, "hash", validationBlock.ProtocolParametersHash) return block } diff --git a/pkg/testsuite/mock/node.go b/pkg/testsuite/mock/node.go index d17588062..8ab489984 100644 --- a/pkg/testsuite/mock/node.go +++ b/pkg/testsuite/mock/node.go @@ -149,8 +149,6 @@ func (n *Node) Initialize(failOnBlockFiltered bool, opts ...options.Option[proto func (n *Node) hookEvents() { n.Protocol.HeaviestAttestedChain.OnUpdate(func(prevHeaviestAttestedChain, heaviestAttestedChain *protocol.Chain) { - fmt.Println(n.Name, " > HeaviestAttestedChain.OnUpdate", heaviestAttestedChain) - if prevHeaviestAttestedChain != nil { n.forkDetectedCount.Add(1) From 37c919c0aa01a274405429f17c59543e2fed1a4a Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 03:27:12 +0100 Subject: [PATCH 08/14] Feat: improved log output --- pkg/protocol/chain.go | 8 +++- pkg/protocol/engine/engine.go | 58 +++++++++++++-------------- pkg/protocol/protocol_attestations.go | 2 +- pkg/protocol/protocol_blocks.go | 2 +- pkg/protocol/protocol_commitments.go | 2 +- pkg/protocol/protocol_warp_sync.go | 2 +- pkg/testsuite/mock/blockissuer.go | 2 +- 7 files changed, 41 insertions(+), 35 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index c67244197..68c2bc8d8 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -145,7 +145,7 @@ func NewChain(protocol *Protocol) *Chain { }) c.Logger = protocol.NewEntityLogger("Chain", c.IsEvicted, func(entityLogger log.Logger) { - c.WarpSync.LogUpdates(entityLogger, log.LevelInfo, "WarpSync") + c.WarpSync.LogUpdates(entityLogger, log.LevelTrace, "WarpSync") c.NetworkClockSlot.LogUpdates(entityLogger, log.LevelTrace, "NetworkClockSlot") c.WarpSyncThreshold.LogUpdates(entityLogger, log.LevelTrace, "WarpSyncThreshold") c.OutOfSyncThreshold.LogUpdates(entityLogger, log.LevelTrace, "OutOfSyncThreshold") @@ -162,6 +162,12 @@ func NewChain(protocol *Protocol) *Chain { var unsubscribe func() c.WarpSync.OnUpdate(func(_, warpSync bool) { + if warpSync { + c.LogDebug("warp-sync enabled") + } else { + c.LogDebug("warp-sync disabled") + } + if unsubscribe != nil { unsubscribe() } diff --git a/pkg/protocol/engine/engine.go b/pkg/protocol/engine/engine.go index bc800d492..a67aa469f 100644 --- a/pkg/protocol/engine/engine.go +++ b/pkg/protocol/engine/engine.go @@ -14,7 +14,6 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" "github.com/iotaledger/hive.go/log" - "github.com/iotaledger/hive.go/runtime/debug" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/options" @@ -237,7 +236,7 @@ func (e *Engine) ProcessBlockFromPeer(block *model.Block, source peer.ID) { // Reset resets the component to a clean state as if it was created at the last commitment. func (e *Engine) Reset() { - e.LogDebug("resetting engine", "slot", e.Storage.Settings().LatestCommitment().Slot(), "stack-trace", debug.StackTrace(false, 0)) + e.LogDebug("resetting", "target-slot", e.Storage.Settings().LatestCommitment().Slot()) e.BlockRequester.Clear() e.Storage.Reset() @@ -265,30 +264,6 @@ func (e *Engine) Reset() { e.Clock.Reset(latestCommittedTime) } -func (e *Engine) shutdown() { - e.BlockRequester.Shutdown() - e.Attestations.Shutdown() - e.SyncManager.Shutdown() - e.Notarization.Shutdown() - e.Booker.Shutdown() - e.Ledger.Shutdown() - e.BlockDAG.Shutdown() - e.BlockGadget.Shutdown() - e.SlotGadget.Shutdown() - e.Clock.Shutdown() - e.SybilProtection.Shutdown() - e.UpgradeOrchestrator.Shutdown() - e.TipManager.Shutdown() - e.Filter.Shutdown() - e.CommitmentFilter.Shutdown() - e.Scheduler.Shutdown() - e.Retainer.Shutdown() - e.Workers.Shutdown() - e.Storage.Shutdown() - - e.Stopped.Trigger() -} - func (e *Engine) BlockFromCache(id iotago.BlockID) (*blocks.Block, bool) { return e.BlockCache.Block(id) } @@ -592,14 +567,39 @@ func (e *Engine) initLatestCommitment() { func (e *Engine) initReactiveModule(logger log.Logger) (reactiveModule *module.ReactiveModule) { stopLogging := reactive.NewEvent() - reactiveModule = module.NewReactiveModule(logger.NewEntityLogger("Engine", stopLogging, func(engineLogger log.Logger) { - // TODO: setup reactive logging + reactiveModule = module.NewReactiveModule(logger.NewEntityLogger("Engine", stopLogging, func(l log.Logger) { + e.RootCommitment.LogUpdates(l, log.LevelTrace, "RootCommitment") + e.LatestCommitment.LogUpdates(l, log.LevelTrace, "LatestCommitment") })) reactiveModule.Shutdown.OnTrigger(func() { + reactiveModule.LogDebug("shutting down") + stopLogging.Trigger() - e.shutdown() + e.BlockRequester.Shutdown() + e.Attestations.Shutdown() + e.SyncManager.Shutdown() + e.Notarization.Shutdown() + e.Booker.Shutdown() + e.Ledger.Shutdown() + e.BlockDAG.Shutdown() + e.BlockGadget.Shutdown() + e.SlotGadget.Shutdown() + e.Clock.Shutdown() + e.SybilProtection.Shutdown() + e.UpgradeOrchestrator.Shutdown() + e.TipManager.Shutdown() + e.Filter.Shutdown() + e.CommitmentFilter.Shutdown() + e.Scheduler.Shutdown() + e.Retainer.Shutdown() + e.Workers.Shutdown() + e.Storage.Shutdown() + + reactiveModule.LogDebug("stopped") + + e.Stopped.Trigger() }) return reactiveModule diff --git a/pkg/protocol/protocol_attestations.go b/pkg/protocol/protocol_attestations.go index 6b34fc1c9..89ac81ee3 100644 --- a/pkg/protocol/protocol_attestations.go +++ b/pkg/protocol/protocol_attestations.go @@ -207,7 +207,7 @@ func (a *AttestationsProtocol) sendRequest(commitmentID iotago.CommitmentID) { if commitment, err := a.protocol.Commitment(commitmentID, false); err == nil { a.protocol.Network.RequestAttestations(commitmentID) - a.LogDebug("sent request", "commitment", commitment.LogName()) + a.LogDebug("request", "commitment", commitment.LogName()) } else { a.LogError("failed to load commitment", "commitmentID", commitmentID, "err", err) } diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 5a50117d7..748657a18 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -73,7 +73,7 @@ func (b *BlocksProtocol) SendRequest(blockID iotago.BlockID) { b.workerPool.Submit(func() { b.protocol.Network.RequestBlock(blockID) - b.protocol.LogTrace("sent request", "blockID", blockID) + b.protocol.LogTrace("request", "blockID", blockID) }) } diff --git a/pkg/protocol/protocol_commitments.go b/pkg/protocol/protocol_commitments.go index 7c30fef72..160caeeb2 100644 --- a/pkg/protocol/protocol_commitments.go +++ b/pkg/protocol/protocol_commitments.go @@ -37,7 +37,7 @@ func (c *CommitmentsProtocol) SendRequest(commitmentID iotago.CommitmentID) { c.workerPool.Submit(func() { c.protocol.Network.RequestSlotCommitment(commitmentID) - c.LogDebug("sent request", "commitmentID", commitmentID) + c.LogDebug("request", "commitment", commitmentID) }) } diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 395a57fa2..a0a938317 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -54,7 +54,7 @@ func (w *WarpSyncProtocol) SendRequest(commitmentID iotago.CommitmentID) { if commitment, err := w.protocol.Commitment(commitmentID, false); err == nil { w.protocol.Network.SendWarpSyncRequest(commitmentID) - w.LogDebug("sent request", "commitment", commitment.LogName()) + w.LogDebug("request", "commitment", commitment.LogName()) } }) } diff --git a/pkg/testsuite/mock/blockissuer.go b/pkg/testsuite/mock/blockissuer.go index 7ed6b63d1..64c896fab 100644 --- a/pkg/testsuite/mock/blockissuer.go +++ b/pkg/testsuite/mock/blockissuer.go @@ -283,7 +283,7 @@ func (i *BlockIssuer) IssueBasicBlock(ctx context.Context, alias string, node *N require.NoErrorf(i.Testing, i.IssueBlock(block.ModelBlock(), node), "%s > failed to issue block with alias %s", i.Name, alias) - fmt.Printf("%s > Issued block: %s - slot %d - commitment %s %d - latest finalized slot %d\n", i.Name, block.ID(), block.ID().Slot(), block.SlotCommitmentID(), block.SlotCommitmentID().Slot(), block.ProtocolBlock().LatestFinalizedSlot) + node.Protocol.LogTrace("issued block", "block", block, "blockID", block.ID(), "slot", block.ID().Slot(), "commitment", block.SlotCommitmentID(), "latestFinalizedSlot", block.ProtocolBlock().LatestFinalizedSlot, "version", block.ProtocolBlock().ProtocolVersion) return block } From b8c74e3abe5323f3c90081b9cf1b846cbafe4102 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 11:01:56 +0100 Subject: [PATCH 09/14] Refactor: updated logging --- pkg/protocol/chain.go | 2 ++ pkg/protocol/chain_manager.go | 4 ++-- pkg/protocol/commitment.go | 3 --- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 68c2bc8d8..31071be9c 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -145,6 +145,8 @@ func NewChain(protocol *Protocol) *Chain { }) c.Logger = protocol.NewEntityLogger("Chain", c.IsEvicted, func(entityLogger log.Logger) { + entityLogger.LogDebug("created") + c.WarpSync.LogUpdates(entityLogger, log.LevelTrace, "WarpSync") c.NetworkClockSlot.LogUpdates(entityLogger, log.LevelTrace, "NetworkClockSlot") c.WarpSyncThreshold.LogUpdates(entityLogger, log.LevelTrace, "WarpSyncThreshold") diff --git a/pkg/protocol/chain_manager.go b/pkg/protocol/chain_manager.go index 5910232ab..5569620f7 100644 --- a/pkg/protocol/chain_manager.go +++ b/pkg/protocol/chain_manager.go @@ -89,7 +89,7 @@ func (c *ChainManager) PublishCommitment(commitment *model.Commitment) (commitme }) if published = commitmentMetadata == publishedCommitmentMetadata; published { - c.protocol.LogDebug("new commitment created", "name", commitmentMetadata.LogName(), "id", commitment.ID()) + commitmentMetadata.LogDebug("created", "id", commitment.ID()) } return commitmentMetadata, commitmentMetadata == publishedCommitmentMetadata, nil @@ -125,7 +125,7 @@ func (c *ChainManager) initMainChain() { mainChain := NewChain(c.protocol) - c.protocol.LogDebug("new chain created", "name", mainChain.LogName(), "forkingPoint", "") + //c.protocol.LogDebug("new chain created", "name", mainChain.LogName(), "forkingPoint", "") mainChain.VerifyState.Set(true) mainChain.Engine.OnUpdate(func(_, newEngine *engine.Engine) { c.protocol.Events.Engine.LinkTo(newEngine.Events) }) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 1fc50488a..84b5d451d 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -204,9 +204,6 @@ func (c *Commitment) inheritChain(parent *Commitment) func(*Commitment, *Commitm } spawnedChain = NewChain(c.protocol) - - c.protocol.LogDebug("new chain created", "name", spawnedChain.LogName(), "forkingPoint", c.LogName()) - spawnedChain.ForkingPoint.Set(c) } From 9f1f6bf3c6e24b9f6fa831d070d3663408b3d3f7 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:47:13 +0100 Subject: [PATCH 10/14] Fix: fix race with workerpool --- pkg/protocol/chain.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 31071be9c..d30b0d09d 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -9,6 +9,7 @@ import ( "github.com/iotaledger/hive.go/ds/shrinkingmap" "github.com/iotaledger/hive.go/lo" "github.com/iotaledger/hive.go/log" + "github.com/iotaledger/hive.go/runtime/workerpool" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine" iotago "github.com/iotaledger/iota.go/v4" @@ -162,29 +163,31 @@ func NewChain(protocol *Protocol) *Chain { c.SpawnedEngine.LogUpdates(entityLogger, log.LevelDebug, "SpawnedEngine", (*engine.Engine).LogName) }) + warpSyncTogglePool := workerpool.New("WarpSync toggle", workerpool.WithWorkerCount(1)) + var unsubscribe func() c.WarpSync.OnUpdate(func(_, warpSync bool) { - if warpSync { - c.LogDebug("warp-sync enabled") - } else { - c.LogDebug("warp-sync disabled") - } - if unsubscribe != nil { unsubscribe() } - go func() { - if warpSync { + if warpSync { + c.LogDebug("warp-sync enabled") + + warpSyncTogglePool.Submit(func() { unsubscribe = c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex) bool { return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < warpSyncThreshold }, c.LatestVerifiedCommitment, c.WarpSyncThreshold)) - } else { + }) + } else { + c.LogDebug("warp-sync disabled") + + warpSyncTogglePool.Submit(func() { unsubscribe = c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, outOfSyncThreshold iotago.SlotIndex) bool { return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < outOfSyncThreshold }, c.LatestVerifiedCommitment, c.OutOfSyncThreshold)) - } - }() + }) + } }) return c From e148b380dedab4581fc9a8a38bea9e90f26c1c15 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 18:40:36 +0100 Subject: [PATCH 11/14] Fix: fixed some SCHTUFF --- pkg/protocol/chain.go | 82 ++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index d30b0d09d..35b63017c 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -51,7 +51,9 @@ func NewChain(protocol *Protocol) *Chain { LatestCommitment: reactive.NewVariable[*Commitment](), LatestAttestedCommitment: reactive.NewVariable[*Commitment](), LatestVerifiedCommitment: reactive.NewVariable[*Commitment](), + ClaimedWeight: reactive.NewVariable[uint64](), AttestedWeight: reactive.NewVariable[uint64](), + VerifiedWeight: reactive.NewVariable[uint64](), WarpSync: reactive.NewVariable[bool]().Init(true), NetworkClockSlot: reactive.NewVariable[iotago.SlotIndex](), WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), @@ -65,14 +67,10 @@ func NewChain(protocol *Protocol) *Chain { SpawnedEngine: reactive.NewVariable[*engine.Engine](), } - c.LatestAttestedCommitment.OnUpdateWithContext(func(_, latestAttestedCommitment *Commitment, withinContext func(subscriptionFactory func() (unsubscribe func()))) { - withinContext(func() (unsubscribe func()) { - return latestAttestedCommitment.CumulativeAttestedWeight.OnUpdate(func(_, newValue uint64) { c.AttestedWeight.Set(newValue) }) - }) - }) - - c.ClaimedWeight = reactive.NewDerivedVariable((*Commitment).cumulativeWeight, c.LatestCommitment) - c.VerifiedWeight = reactive.NewDerivedVariable((*Commitment).cumulativeWeight, c.LatestVerifiedCommitment) + c.initClaimedWeight() + c.initAttestedWeight() + c.initVerifiedWeight() + c.initWarpSync() c.Engine = reactive.NewDerivedVariable2(func(spawnedEngine, parentEngine *engine.Engine) *engine.Engine { if spawnedEngine != nil { @@ -163,34 +161,60 @@ func NewChain(protocol *Protocol) *Chain { c.SpawnedEngine.LogUpdates(entityLogger, log.LevelDebug, "SpawnedEngine", (*engine.Engine).LogName) }) - warpSyncTogglePool := workerpool.New("WarpSync toggle", workerpool.WithWorkerCount(1)) + return c +} - var unsubscribe func() - c.WarpSync.OnUpdate(func(_, warpSync bool) { - if unsubscribe != nil { - unsubscribe() +func (c *Chain) initClaimedWeight() { + c.ClaimedWeight.InheritFrom(reactive.NewDerivedVariable(func(c *Commitment) uint64 { + if c == nil { + return 0 } - if warpSync { - c.LogDebug("warp-sync enabled") - - warpSyncTogglePool.Submit(func() { - unsubscribe = c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex) bool { - return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < warpSyncThreshold - }, c.LatestVerifiedCommitment, c.WarpSyncThreshold)) - }) - } else { - c.LogDebug("warp-sync disabled") + return c.CumulativeWeight() + }, c.LatestCommitment)) +} - warpSyncTogglePool.Submit(func() { - unsubscribe = c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, outOfSyncThreshold iotago.SlotIndex) bool { - return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < outOfSyncThreshold - }, c.LatestVerifiedCommitment, c.OutOfSyncThreshold)) - }) +func (c *Chain) initAttestedWeight() { + c.LatestAttestedCommitment.OnUpdateWithContext(func(_, latestAttestedCommitment *Commitment, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + setupInheritance := func() func() { + return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) } + + unsubscribeOnUpdate(setupInheritance) }) +} - return c +func (c *Chain) initVerifiedWeight() { + c.VerifiedWeight.InheritFrom(reactive.NewDerivedVariable(func(c *Commitment) uint64 { + if c == nil { + return 0 + } + + return c.CumulativeWeight() + }, c.LatestVerifiedCommitment)) +} + +func (c *Chain) initWarpSync() { + enableWarpSyncIfNecessary := func() (unsubscribe func()) { + return c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, outOfSyncThreshold iotago.SlotIndex) bool { + return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < outOfSyncThreshold + }, c.LatestVerifiedCommitment, c.OutOfSyncThreshold)) + } + + disableWarpSyncIfNecessary := func() (unsubscribe func()) { + return c.WarpSync.InheritFrom(reactive.NewDerivedVariable2(func(latestVerifiedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex) bool { + return latestVerifiedCommitment != nil && latestVerifiedCommitment.ID().Slot() < warpSyncThreshold + }, c.LatestVerifiedCommitment, c.WarpSyncThreshold)) + } + + warpSyncTogglePool := workerpool.New("WarpSync toggle", workerpool.WithWorkerCount(1)).Start() + c.IsEvicted.OnTrigger(func() { warpSyncTogglePool.Shutdown() }) + + c.WarpSync.OnUpdateWithContext(func(_, warpSync bool, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + if !c.IsEvicted.Get() { + warpSyncTogglePool.Submit(func() { unsubscribeOnUpdate(lo.Cond(warpSync, disableWarpSyncIfNecessary, enableWarpSyncIfNecessary)) }) + } + }) } func (c *Chain) Commitment(slot iotago.SlotIndex) (commitment *Commitment, exists bool) { From 4dd3f602003d146c0bcbf888c832a839f6f06df5 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 18:52:45 +0100 Subject: [PATCH 12/14] fix linter warning --- pkg/protocol/commitment.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 84b5d451d..1fe0b6746 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -250,11 +250,3 @@ func (c *Commitment) triggerEventIfBelowThreshold(event func(*Commitment) reacti }) }) } - -func (c *Commitment) cumulativeWeight() uint64 { - if c == nil { - return 0 - } - - return c.CumulativeWeight() -} From 2ae426810d235990171ea79a4e5376befb481b42 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 29 Oct 2023 21:41:10 +0100 Subject: [PATCH 13/14] Fix: fixeridoo --- pkg/protocol/chain.go | 44 +++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 35b63017c..499d6b0e9 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -59,6 +59,7 @@ func NewChain(protocol *Protocol) *Chain { WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), VerifyAttestations: reactive.NewVariable[bool](), + Engine: reactive.NewVariable[*engine.Engine](), IsEvicted: reactive.NewEvent(), commitments: shrinkingmap.New[iotago.SlotIndex, *Commitment](), @@ -70,16 +71,9 @@ func NewChain(protocol *Protocol) *Chain { c.initClaimedWeight() c.initAttestedWeight() c.initVerifiedWeight() + c.initEngine() c.initWarpSync() - c.Engine = reactive.NewDerivedVariable2(func(spawnedEngine, parentEngine *engine.Engine) *engine.Engine { - if spawnedEngine != nil { - return spawnedEngine - } - - return parentEngine - }, c.SpawnedEngine, c.parentEngine) - protocol.NetworkClock.OnUpdate(func(_, now time.Time) { if engineInstance := c.Engine.Get(); engineInstance != nil { c.NetworkClockSlot.Set(engineInstance.LatestAPI().TimeProvider().SlotFromTime(now)) @@ -210,13 +204,43 @@ func (c *Chain) initWarpSync() { warpSyncTogglePool := workerpool.New("WarpSync toggle", workerpool.WithWorkerCount(1)).Start() c.IsEvicted.OnTrigger(func() { warpSyncTogglePool.Shutdown() }) - c.WarpSync.OnUpdateWithContext(func(_, warpSync bool, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + var unsubscribe func() + + c.WarpSync.OnUpdate(func(_, warpSync bool) { if !c.IsEvicted.Get() { - warpSyncTogglePool.Submit(func() { unsubscribeOnUpdate(lo.Cond(warpSync, disableWarpSyncIfNecessary, enableWarpSyncIfNecessary)) }) + warpSyncTogglePool.Submit(func() { + if unsubscribe != nil { + unsubscribe() + } + + if warpSync { + unsubscribe = disableWarpSyncIfNecessary() + } else { + unsubscribe = enableWarpSyncIfNecessary() + } + }) } }) } +func (c *Chain) initEngine() { + c.ParentChain.OnUpdateWithContext(func(_, parentChain *Chain, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + unsubscribeOnUpdate(func() func() { + if parentChain == nil { + return c.Engine.InheritFrom(c.SpawnedEngine) + } + + return c.Engine.InheritFrom(reactive.NewDerivedVariable2(func(spawnedEngine, parentEngine *engine.Engine) *engine.Engine { + if spawnedEngine != nil { + return spawnedEngine + } + + return parentEngine + }, c.SpawnedEngine, parentChain.Engine)) + }) + }, true) +} + func (c *Chain) Commitment(slot iotago.SlotIndex) (commitment *Commitment, exists bool) { for currentChain := c; currentChain != nil; { switch forkingPoint := currentChain.ForkingPoint.Get(); { From f5606140358b729b4e3a1d174a4dfd6928e1ecdb Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 30 Oct 2023 01:10:49 +0100 Subject: [PATCH 14/14] Fix: fix more stuff --- pkg/protocol/chain.go | 12 +++-- pkg/protocol/commitment.go | 92 +++++++++++++++++++++------------ pkg/protocol/protocol_blocks.go | 12 ++--- 3 files changed, 71 insertions(+), 45 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 499d6b0e9..7f9edc09f 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -170,11 +170,13 @@ func (c *Chain) initClaimedWeight() { func (c *Chain) initAttestedWeight() { c.LatestAttestedCommitment.OnUpdateWithContext(func(_, latestAttestedCommitment *Commitment, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { - setupInheritance := func() func() { - return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) - } + if latestAttestedCommitment != nil { + setupInheritance := func() func() { + return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) + } - unsubscribeOnUpdate(setupInheritance) + unsubscribeOnUpdate(setupInheritance) + } }) } @@ -270,7 +272,7 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (success bool) { for _, chain := range append([]*Chain{c}, c.ChildChains.ToSlice()...) { if chain.VerifyState.Get() { - if targetEngine := chain.SpawnedEngine.Get(); targetEngine != nil && !chain.WarpSync.Get() || targetEngine.BlockRequester.HasTicker(block.ID()) { + if targetEngine := chain.Engine.Get(); targetEngine != nil && !chain.WarpSync.Get() || targetEngine.BlockRequester.HasTicker(block.ID()) { targetEngine.ProcessBlockFromPeer(block, src) success = true diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 1fe0b6746..b7e6a66ba 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -14,23 +14,25 @@ import ( type Commitment struct { *model.Commitment - Parent reactive.Variable[*Commitment] - Children reactive.Set[*Commitment] - MainChild reactive.Variable[*Commitment] - SpawnedChain reactive.Variable[*Chain] - Chain reactive.Variable[*Chain] - Engine reactive.Variable[*engine.Engine] - RequestAttestations reactive.Variable[bool] - RequestBlocks reactive.Variable[bool] - RequestedBlocksReceived reactive.Variable[bool] - Weight reactive.Variable[uint64] - AttestedWeight reactive.Variable[uint64] - CumulativeAttestedWeight reactive.Variable[uint64] - IsSolid reactive.Event - IsAttested reactive.Event - IsVerified reactive.Event - IsRoot reactive.Event - IsEvicted reactive.Event + Parent reactive.Variable[*Commitment] + Children reactive.Set[*Commitment] + MainChild reactive.Variable[*Commitment] + SpawnedChain reactive.Variable[*Chain] + Chain reactive.Variable[*Chain] + Engine reactive.Variable[*engine.Engine] + RequestAttestations reactive.Variable[bool] + RequestBlocks reactive.Variable[bool] + RequestedBlocksReceived reactive.Variable[bool] + Weight reactive.Variable[uint64] + AttestedWeight reactive.Variable[uint64] + CumulativeAttestedWeight reactive.Variable[uint64] + IsSolid reactive.Event + IsAttested reactive.Event + IsVerified reactive.Event + IsRoot reactive.Event + IsEvicted reactive.Event + IsAboveLatestVerifiedCommitment reactive.Variable[bool] + ReplayBlocks reactive.Variable[bool] protocol *Protocol isDirectlyAboveLatestAttestedCommitment reactive.Variable[bool] @@ -45,23 +47,25 @@ func NewCommitment(commitment *model.Commitment, protocol *Protocol) *Commitment c := &Commitment{ Commitment: commitment, - Parent: reactive.NewVariable[*Commitment](), - MainChild: reactive.NewVariable[*Commitment](), - Children: reactive.NewSet[*Commitment](), - SpawnedChain: reactive.NewVariable[*Chain](), - Chain: reactive.NewVariable[*Chain](), - Engine: reactive.NewVariable[*engine.Engine](), - RequestAttestations: reactive.NewVariable[bool](), - RequestBlocks: reactive.NewVariable[bool](), - RequestedBlocksReceived: reactive.NewVariable[bool](), - Weight: reactive.NewVariable[uint64](), - AttestedWeight: reactive.NewVariable[uint64](func(currentValue uint64, newValue uint64) uint64 { return max(currentValue, newValue) }), - CumulativeAttestedWeight: reactive.NewVariable[uint64](), - IsSolid: reactive.NewEvent(), - IsAttested: reactive.NewEvent(), - IsVerified: reactive.NewEvent(), - IsRoot: reactive.NewEvent(), - IsEvicted: reactive.NewEvent(), + Parent: reactive.NewVariable[*Commitment](), + MainChild: reactive.NewVariable[*Commitment](), + Children: reactive.NewSet[*Commitment](), + SpawnedChain: reactive.NewVariable[*Chain](), + Chain: reactive.NewVariable[*Chain](), + Engine: reactive.NewVariable[*engine.Engine](), + RequestAttestations: reactive.NewVariable[bool](), + RequestBlocks: reactive.NewVariable[bool](), + RequestedBlocksReceived: reactive.NewVariable[bool](), + Weight: reactive.NewVariable[uint64](), + AttestedWeight: reactive.NewVariable[uint64](func(currentValue uint64, newValue uint64) uint64 { return max(currentValue, newValue) }), + CumulativeAttestedWeight: reactive.NewVariable[uint64](), + IsSolid: reactive.NewEvent(), + IsAttested: reactive.NewEvent(), + IsVerified: reactive.NewEvent(), + IsRoot: reactive.NewEvent(), + IsEvicted: reactive.NewEvent(), + IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), + ReplayBlocks: reactive.NewVariable[bool](), protocol: protocol, isDirectlyAboveLatestAttestedCommitment: reactive.NewVariable[bool](), @@ -70,6 +74,26 @@ func NewCommitment(commitment *model.Commitment, protocol *Protocol) *Commitment isBelowWarpSyncThreshold: reactive.NewEvent(), } + c.Parent.OnUpdateWithContext(func(_, parent *Commitment, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + if parent != nil { + unsubscribeOnUpdate(func() func() { + return c.IsAboveLatestVerifiedCommitment.InheritFrom(reactive.NewDerivedVariable2(func(parentAboveLatestVerifiedCommitment, directlyAboveLatestVerifiedCommitment bool) bool { + return parentAboveLatestVerifiedCommitment || directlyAboveLatestVerifiedCommitment + }, parent.IsAboveLatestVerifiedCommitment, c.isDirectlyAboveLatestVerifiedCommitment)) + }) + } + }) + + c.Chain.OnUpdateWithContext(func(_, chain *Chain, unsubscribeOnUpdate func(subscriptionFactory func() (unsubscribe func()))) { + if chain != nil { + unsubscribeOnUpdate(func() func() { + return c.ReplayBlocks.InheritFrom(reactive.NewDerivedVariable3(func(spawnedEngine *engine.Engine, warpSyncing, isAboveLatestVerifiedCommitment bool) bool { + return spawnedEngine != nil && !warpSyncing && isAboveLatestVerifiedCommitment + }, chain.SpawnedEngine, chain.WarpSync, c.IsAboveLatestVerifiedCommitment)) + }) + } + }) + c.Parent.OnUpdateOnce(func(_, parent *Commitment) { parent.registerChild(c) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 748657a18..12a30335e 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -33,13 +33,13 @@ func NewBlocksProtocol(protocol *Protocol) *BlocksProtocol { protocol.Constructed.OnTrigger(func() { protocol.CommitmentCreated.Hook(func(commitment *Commitment) { - commitment.isDirectlyAboveLatestVerifiedCommitment.OnUpdate(func(_, isDirectlyAboveLatestVerifiedCommitment bool) { - if !isDirectlyAboveLatestVerifiedCommitment { - return - } + commitment.ReplayBlocks.OnUpdate(func(_, replayBlocks bool) { + if replayBlocks { + b.LogDebug("replaying blocks", "commitmentID", commitment.ID()) - for _, droppedBlock := range b.droppedBlocksBuffer.GetValues(commitment.ID()) { - b.ProcessResponse(droppedBlock.A, droppedBlock.B) + for _, droppedBlock := range b.droppedBlocksBuffer.GetValues(commitment.ID()) { + b.ProcessResponse(droppedBlock.A, droppedBlock.B) + } } }) })