From b6878f45ebb5447e22a465c20a93c6f239cf49e5 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:10:27 +0100 Subject: [PATCH 01/55] Fix: Fix WarpSync --- pkg/protocol/commitment.go | 15 ++++++++++++--- pkg/protocol/protocol_warp_sync.go | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 88f454b6c..34f383ae0 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -56,6 +56,11 @@ type Commitment struct { // IsAttested contains a flag indicating if we have received attestations for this Commitment. IsAttested reactive.Event + // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). + IsCommittable reactive.Event + + IsFullyBooked reactive.Event + // IsVerified contains a flag indicating if this Commitment is verified (we produced this Commitment ourselves by // booking all the contained blocks and transactions). IsVerified reactive.Event @@ -94,6 +99,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen CumulativeAttestedWeight: reactive.NewVariable[uint64](), IsRoot: reactive.NewEvent(), IsAttested: reactive.NewEvent(), + IsCommittable: reactive.NewEvent(), IsVerified: reactive.NewEvent(), IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), ReplayDroppedBlocks: reactive.NewVariable[bool](), @@ -152,6 +158,9 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { // mark commitments that are marked as verified as attested c.IsAttested.InheritFrom(c.IsVerified), + // mark commitments that are marked as verified as fully booked + c.IsFullyBooked.InheritFrom(c.IsVerified), + c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent // commitment) @@ -261,9 +270,9 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting // warp sync, and we are the directly above the latest verified Commitment. func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { - return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsVerified bool, isVerified bool) bool { - return engineInstance != nil && warpSync && parentIsVerified && !isVerified - }, chain.Engine, chain.WarpSyncMode, parent.IsVerified, c.IsVerified)) + return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { + return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked + }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 6e96536bd..bc40b06a4 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -187,7 +187,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } if totalBlocks == 0 { - forceCommitmentFunc() + commitment.IsCommittable.OnTrigger(forceCommitmentFunc) return blocksToWarpSync } @@ -218,7 +218,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - forceCommitmentFunc() + commitment.IsCommittable.OnTrigger(forceCommitmentFunc) }) } } From 0539f3468a558d413d6943e0ded647eb7a13a9fc Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:15:03 +0100 Subject: [PATCH 02/55] Feat: added more logic --- pkg/protocol/commitment.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 34f383ae0..6eab9e523 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -56,11 +56,11 @@ type Commitment struct { // IsAttested contains a flag indicating if we have received attestations for this Commitment. IsAttested reactive.Event + IsFullyBooked reactive.Event + // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). IsCommittable reactive.Event - IsFullyBooked reactive.Event - // IsVerified contains a flag indicating if this Commitment is verified (we produced this Commitment ourselves by // booking all the contained blocks and transactions). IsVerified reactive.Event @@ -99,6 +99,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen CumulativeAttestedWeight: reactive.NewVariable[uint64](), IsRoot: reactive.NewEvent(), IsAttested: reactive.NewEvent(), + IsFullyBooked: reactive.NewEvent(), IsCommittable: reactive.NewEvent(), IsVerified: reactive.NewEvent(), IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), @@ -155,11 +156,10 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { // mark commitments that are marked as root as verified c.IsVerified.InheritFrom(c.IsRoot), - // mark commitments that are marked as verified as attested + // mark commitments that are marked as verified as attested, fully booked and committable c.IsAttested.InheritFrom(c.IsVerified), - - // mark commitments that are marked as verified as fully booked c.IsFullyBooked.InheritFrom(c.IsVerified), + c.IsCommittable.InheritFrom(c.IsVerified), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent From e82a4bed26fde181983ac88f0f2150a371acbce6 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:24:01 +0100 Subject: [PATCH 03/55] Fix: fixed more code --- pkg/protocol/chain.go | 2 +- pkg/protocol/commitment.go | 21 +++++++++++---------- pkg/protocol/commitments.go | 2 +- pkg/protocol/protocol_warp_sync.go | 11 +++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 8e48e1613..40ddfe5a1 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -341,7 +341,7 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), - newCommitment.IsVerified.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), + newCommitment.IsCommitted.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), ) } diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 6eab9e523..290e03d4e 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -56,14 +56,15 @@ type Commitment struct { // IsAttested contains a flag indicating if we have received attestations for this Commitment. IsAttested reactive.Event + // IsFullyBooked contains a flag indicating if we have received all blocks for this Commitment. IsFullyBooked reactive.Event // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). IsCommittable reactive.Event - // IsVerified contains a flag indicating if this Commitment is verified (we produced this Commitment ourselves by - // booking all the contained blocks and transactions). - IsVerified reactive.Event + // IsCommitted contains a flag indicating if we Commitment produced this Commitment ourselves by replaying all the + // blocks of the Commitment. + IsCommitted reactive.Event // IsAboveLatestVerifiedCommitment contains a flag indicating if this Commitment is above the latest verified // Commitment. @@ -101,7 +102,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen IsAttested: reactive.NewEvent(), IsFullyBooked: reactive.NewEvent(), IsCommittable: reactive.NewEvent(), - IsVerified: reactive.NewEvent(), + IsCommitted: reactive.NewEvent(), IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), ReplayDroppedBlocks: reactive.NewVariable[bool](), IsEvicted: reactive.NewEvent(), @@ -142,7 +143,7 @@ func (c *Commitment) initLogger() (shutdown func()) { c.CumulativeAttestedWeight.LogUpdates(c, log.LevelTrace, "CumulativeAttestedWeight"), c.IsRoot.LogUpdates(c, log.LevelTrace, "IsRoot"), c.IsAttested.LogUpdates(c, log.LevelTrace, "IsAttested"), - c.IsVerified.LogUpdates(c, log.LevelTrace, "IsVerified"), + c.IsCommitted.LogUpdates(c, log.LevelTrace, "IsCommitted"), c.ReplayDroppedBlocks.LogUpdates(c, log.LevelTrace, "ReplayDroppedBlocks"), c.IsEvicted.LogUpdates(c, log.LevelTrace, "IsEvicted"), @@ -154,12 +155,12 @@ func (c *Commitment) initLogger() (shutdown func()) { func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( // mark commitments that are marked as root as verified - c.IsVerified.InheritFrom(c.IsRoot), + c.IsCommitted.InheritFrom(c.IsRoot), // mark commitments that are marked as verified as attested, fully booked and committable - c.IsAttested.InheritFrom(c.IsVerified), - c.IsFullyBooked.InheritFrom(c.IsVerified), - c.IsCommittable.InheritFrom(c.IsVerified), + c.IsAttested.InheritFrom(c.IsCommitted), + c.IsFullyBooked.InheritFrom(c.IsCommitted), + c.IsCommittable.InheritFrom(c.IsCommitted), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent @@ -256,7 +257,7 @@ func (c *Commitment) deriveCumulativeAttestedWeight(parent *Commitment) func() { func (c *Commitment) deriveIsAboveLatestVerifiedCommitment(parent *Commitment) func() { return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsVerified bool, isVerified bool) bool { return parentAboveLatestVerifiedCommitment || (parentIsVerified && !isVerified) - }, parent.IsAboveLatestVerifiedCommitment, parent.IsVerified, c.IsVerified)) + }, parent.IsAboveLatestVerifiedCommitment, parent.IsCommitted, c.IsCommitted)) } // deriveRequestAttestations derives the RequestAttestations flag of this Commitment which is true if our Chain is diff --git a/pkg/protocol/commitments.go b/pkg/protocol/commitments.go index 857b1c00d..4066fc01d 100644 --- a/pkg/protocol/commitments.go +++ b/pkg/protocol/commitments.go @@ -155,7 +155,7 @@ func (c *Commitments) publishEngineCommitments(chain *Chain, engine *engine.Engi // mark it as produced by ourselves and force it to be on the right chain (in case our chain produced a // different commitment than the one we erroneously expected it to be - we always trust our engine most). publishedCommitment.AttestedWeight.Set(publishedCommitment.Weight.Get()) - publishedCommitment.IsVerified.Set(true) + publishedCommitment.IsCommitted.Set(true) publishedCommitment.forceChain(chain) } }) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index bc40b06a4..812171483 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -186,8 +186,10 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } } + commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + if totalBlocks == 0 { - commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + commitment.IsFullyBooked.Set(true) return blocksToWarpSync } @@ -213,12 +215,9 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo targetEngine.BlockGadget.SetAccepted(block) block.Notarized().OnUpdate(func(_ bool, _ bool) { - // Wait for all blocks to be notarized before forcing the commitment of the slot. - if notarizedBlocks.Add(1) != totalBlocks { - return + if notarizedBlocks.Add(1) == totalBlocks { + commitment.IsFullyBooked.Set(true) } - - commitment.IsCommittable.OnTrigger(forceCommitmentFunc) }) } } From ab3fdec7a5299f22ed100728a34ba22efac0d154 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:48:25 +0100 Subject: [PATCH 04/55] Trigger Commitment's IsCommittable at - MCA + 1 --- pkg/protocol/commitment.go | 6 ++++-- pkg/protocol/protocol_warp_sync.go | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 290e03d4e..cd3fd8387 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -143,6 +143,8 @@ func (c *Commitment) initLogger() (shutdown func()) { c.CumulativeAttestedWeight.LogUpdates(c, log.LevelTrace, "CumulativeAttestedWeight"), c.IsRoot.LogUpdates(c, log.LevelTrace, "IsRoot"), c.IsAttested.LogUpdates(c, log.LevelTrace, "IsAttested"), + c.IsFullyBooked.LogUpdates(c, log.LevelTrace, "IsFullyBooked"), + c.IsCommittable.LogUpdates(c, log.LevelTrace, "IsCommittable"), c.IsCommitted.LogUpdates(c, log.LevelTrace, "IsCommitted"), c.ReplayDroppedBlocks.LogUpdates(c, log.LevelTrace, "ReplayDroppedBlocks"), c.IsEvicted.LogUpdates(c, log.LevelTrace, "IsEvicted"), @@ -255,8 +257,8 @@ func (c *Commitment) deriveCumulativeAttestedWeight(parent *Commitment) func() { // deriveIsAboveLatestVerifiedCommitment derives the IsAboveLatestVerifiedCommitment flag of this Commitment which is // true if the parent is already above the latest verified Commitment or if the parent is verified and we are not. func (c *Commitment) deriveIsAboveLatestVerifiedCommitment(parent *Commitment) func() { - return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsVerified bool, isVerified bool) bool { - return parentAboveLatestVerifiedCommitment || (parentIsVerified && !isVerified) + return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsCommitted bool, isCommitted bool) bool { + return parentAboveLatestVerifiedCommitment || (parentIsCommitted && !isCommitted) }, parent.IsAboveLatestVerifiedCommitment, parent.IsCommitted, c.IsCommitted)) } diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 812171483..005d62b2f 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -160,7 +160,8 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo if targetEngine.Workers.WaitChildren(); targetEngine.Storage.Settings().LatestCommitment().ID().Slot() > commitmentID.Slot() { return blocksToWarpSync } - targetEngine.Reset() + // TODO: reset the engine only once at the very beginning of the warp-sync process + // targetEngine.Reset() // Once all blocks are booked we // 1. Mark all transactions as accepted @@ -186,7 +187,16 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } } - commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { + minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { + committableCommitment.IsCommittable.Trigger() + } + }) + + commitment.IsCommittable.OnUpdateOnce(func(_ bool, _ bool) { + forceCommitmentFunc() + }) if totalBlocks == 0 { commitment.IsFullyBooked.Set(true) From 66807e05399d454a9e8ebfde82ebd440ac56b40c Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:49:14 +0100 Subject: [PATCH 05/55] Deadlock fix --- pkg/protocol/protocol_warp_sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 005d62b2f..b51177dae 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -190,12 +190,12 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { - committableCommitment.IsCommittable.Trigger() + committableCommitment.IsCommittable.Set(true) } }) commitment.IsCommittable.OnUpdateOnce(func(_ bool, _ bool) { - forceCommitmentFunc() + w.workerPool.Submit(forceCommitmentFunc) }) if totalBlocks == 0 { From 000355bab8e8618a98465f052234e342d1cccb01 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:09:16 +0100 Subject: [PATCH 06/55] Force accept and commit with MCA delay --- pkg/protocol/protocol_warp_sync.go | 60 ++++++++++++++---------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index b51177dae..352358148 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -167,10 +167,24 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 1. Mark all transactions as accepted // 2. Mark all blocks as accepted // 3. Force commitment of the slot - var bookedBlocks atomic.Uint32 - var notarizedBlocks atomic.Uint32 - forceCommitmentFunc := func() { + // 1. Mark all transactions as accepted + for _, transactionID := range transactionIDs { + targetEngine.Ledger.ConflictDAG().SetAccepted(transactionID) + } + + // 2. Mark all blocks as accepted + for _, blockIDs := range blockIDsBySlotCommitment { + for _, blockID := range blockIDs { + block, exists := targetEngine.BlockCache.Block(blockID) + if !exists { // this should never happen as we just booked these blocks in this slot. + continue + } + + targetEngine.BlockGadget.SetAccepted(block) + } + } + // 3. Force commitment of the slot producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) if err != nil { @@ -188,6 +202,8 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { + // Let's assume that MCA is 5: when we want to book 15, we expect to have the commitment of 10 to load + // accounts from it, hence why we make committable the slot at - MCA + 1 with respect of the current slot. minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { committableCommitment.IsCommittable.Set(true) @@ -204,35 +220,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return blocksToWarpSync } - blockBookedFunc := func(_ bool, _ bool) { - if bookedBlocks.Add(1) != totalBlocks { - return - } - - // 1. Mark all transactions as accepted - for _, transactionID := range transactionIDs { - targetEngine.Ledger.ConflictDAG().SetAccepted(transactionID) - } - - // 2. Mark all blocks as accepted - for _, blockIDs := range blockIDsBySlotCommitment { - for _, blockID := range blockIDs { - block, exists := targetEngine.BlockCache.Block(blockID) - if !exists { // this should never happen as we just booked these blocks in this slot. - continue - } - - targetEngine.BlockGadget.SetAccepted(block) - - block.Notarized().OnUpdate(func(_ bool, _ bool) { - if notarizedBlocks.Add(1) == totalBlocks { - commitment.IsFullyBooked.Set(true) - } - }) - } - } - } - + var bookedBlocks atomic.Uint32 blocksToWarpSync = ds.NewSet[iotago.BlockID]() for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { for _, blockID := range blockIDs { @@ -252,7 +240,13 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // block cache and thus if not root blocks no block in the next slot can become solid. targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) - block.Booked().OnUpdate(blockBookedFunc) + block.Booked().OnUpdate(func(_ bool, _ bool) { + if bookedBlocks.Add(1) != totalBlocks { + return + } + + commitment.IsFullyBooked.Set(true) + }) } } From 58a9972665de5a38da25c4d0edcd2fbb61dde235 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:24:40 +0100 Subject: [PATCH 07/55] Add rootblocks only before force-accepting them --- pkg/protocol/protocol_warp_sync.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 352358148..237367fc9 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -174,13 +174,18 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } // 2. Mark all blocks as accepted - for _, blockIDs := range blockIDsBySlotCommitment { + for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { for _, blockID := range blockIDs { block, exists := targetEngine.BlockCache.Block(blockID) if !exists { // this should never happen as we just booked these blocks in this slot. continue } + // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without + // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the + // block cache and thus if not root blocks no block in the next slot can become solid. + targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) + targetEngine.BlockGadget.SetAccepted(block) } } @@ -222,7 +227,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo var bookedBlocks atomic.Uint32 blocksToWarpSync = ds.NewSet[iotago.BlockID]() - for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { + for _, blockIDs := range blockIDsBySlotCommitment { for _, blockID := range blockIDs { blocksToWarpSync.Add(blockID) @@ -235,11 +240,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo continue } - // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without - // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the - // block cache and thus if not root blocks no block in the next slot can become solid. - targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) - + // TODO: this will need to be changed to WeightPropagated before marking a block as actually booked. block.Booked().OnUpdate(func(_ bool, _ bool) { if bookedBlocks.Add(1) != totalBlocks { return From 9e9697c88b21c65cf1171ea4fd2c4ba916637619 Mon Sep 17 00:00:00 2001 From: Andrea V <1577639+karimodm@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:22:24 +0100 Subject: [PATCH 08/55] Reset the Engine when WarpSyncMode turns true --- pkg/protocol/protocol_warp_sync.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 237367fc9..579858fe1 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -13,6 +13,7 @@ import ( "github.com/iotaledger/hive.go/lo" "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/workerpool" + "github.com/iotaledger/iota-core/pkg/protocol/engine" iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/merklehasher" ) @@ -44,7 +45,14 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { c.ticker.Events.Tick.Hook(c.SendRequest) protocol.Constructed.OnTrigger(func() { - c.protocol.Commitments.WithElements(func(commitment *Commitment) (shutdown func()) { + protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { + return chain.WarpSyncMode.OnUpdate(func(_ bool, warpsyncMode bool) { + if warpsyncMode { + engine.Reset() + } + }) + }) + protocol.Commitments.WithElements(func(commitment *Commitment) (shutdown func()) { return commitment.WarpSyncBlocks.OnUpdate(func(_ bool, warpSyncBlocks bool) { if warpSyncBlocks { c.ticker.StartTicker(commitment.ID()) @@ -160,8 +168,6 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo if targetEngine.Workers.WaitChildren(); targetEngine.Storage.Settings().LatestCommitment().ID().Slot() > commitmentID.Slot() { return blocksToWarpSync } - // TODO: reset the engine only once at the very beginning of the warp-sync process - // targetEngine.Reset() // Once all blocks are booked we // 1. Mark all transactions as accepted From 51431c795b2fd4580778c811fa59ea0f3c2e7da3 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:57:44 +0100 Subject: [PATCH 09/55] Fix: TestLossOfAcceptanceFromGenesis test fixed --- pkg/protocol/chain.go | 2 +- pkg/protocol/engine/blocks/block.go | 12 +-- pkg/protocol/engine/clock/blocktime/clock.go | 4 +- pkg/protocol/protocol_warp_sync.go | 81 +++++++++++++------- 4 files changed, 62 insertions(+), 37 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 40ddfe5a1..557f5d3e7 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -92,7 +92,7 @@ func newChain(chains *Chains) *Chain { ClaimedWeight: reactive.NewVariable[uint64](), AttestedWeight: reactive.NewVariable[uint64](), VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncMode: reactive.NewVariable[bool]().Init(true), + WarpSyncMode: reactive.NewVariable[bool](), WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), RequestAttestations: reactive.NewVariable[bool](), diff --git a/pkg/protocol/engine/blocks/block.go b/pkg/protocol/engine/blocks/block.go index 88ce9192e..8085d4010 100644 --- a/pkg/protocol/engine/blocks/block.go +++ b/pkg/protocol/engine/blocks/block.go @@ -48,7 +48,7 @@ type Block struct { dropped bool // Notarization - notarized reactive.Variable[bool] + notarized reactive.Event mutex syncutils.RWMutex @@ -86,7 +86,7 @@ func NewBlock(data *model.Block) *Block { invalid: reactive.NewVariable[bool](), booked: reactive.NewVariable[bool](), accepted: reactive.NewVariable[bool](), - notarized: reactive.NewVariable[bool](), + notarized: reactive.NewEvent(), workScore: data.WorkScore(), } } @@ -109,7 +109,7 @@ func NewRootBlock(blockID iotago.BlockID, commitmentID iotago.CommitmentID, issu booked: reactive.NewVariable[bool](), preAccepted: true, accepted: reactive.NewVariable[bool](), - notarized: reactive.NewVariable[bool](), + notarized: reactive.NewEvent(), scheduled: true, } @@ -135,7 +135,7 @@ func NewMissingBlock(blockID iotago.BlockID) *Block { invalid: reactive.NewVariable[bool](), booked: reactive.NewVariable[bool](), accepted: reactive.NewVariable[bool](), - notarized: reactive.NewVariable[bool](), + notarized: reactive.NewEvent(), } } @@ -605,7 +605,7 @@ func (b *Block) SetPreConfirmed() (wasUpdated bool) { return wasUpdated } -func (b *Block) Notarized() reactive.Variable[bool] { +func (b *Block) Notarized() reactive.Event { return b.notarized } @@ -614,7 +614,7 @@ func (b *Block) IsNotarized() (isBooked bool) { } func (b *Block) SetNotarized() (wasUpdated bool) { - return !b.notarized.Set(true) + return b.notarized.Trigger() } func (b *Block) String() string { diff --git a/pkg/protocol/engine/clock/blocktime/clock.go b/pkg/protocol/engine/clock/blocktime/clock.go index 506387df0..51801187b 100644 --- a/pkg/protocol/engine/clock/blocktime/clock.go +++ b/pkg/protocol/engine/clock/blocktime/clock.go @@ -55,7 +55,9 @@ func NewProvider(opts ...options.Option[Clock]) module.Provider[*engine.Engine, asyncOpt := event.WithWorkerPool(c.workerPool) c.HookStopped(lo.Batch( e.Events.BlockGadget.BlockAccepted.Hook(func(block *blocks.Block) { - c.acceptedTime.Advance(block.IssuingTime()) + block.Notarized().OnTrigger(func() { + c.acceptedTime.Advance(block.IssuingTime()) + }) }, asyncOpt).Unhook, e.Events.BlockGadget.BlockConfirmed.Hook(func(block *blocks.Block) { diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 579858fe1..e17a85d7d 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -8,6 +8,7 @@ import ( "github.com/iotaledger/hive.go/ads" "github.com/iotaledger/hive.go/core/eventticker" "github.com/iotaledger/hive.go/ds" + "github.com/iotaledger/hive.go/ds/reactive" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/kvstore/mapdb" "github.com/iotaledger/hive.go/lo" @@ -46,12 +47,14 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { protocol.Constructed.OnTrigger(func() { protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { - return chain.WarpSyncMode.OnUpdate(func(_ bool, warpsyncMode bool) { - if warpsyncMode { + return chain.WarpSyncMode.OnUpdate(func(_ bool, warpSyncMode bool) { + if warpSyncMode { + engine.Workers.WaitChildren() engine.Reset() } }) }) + protocol.Commitments.WithElements(func(commitment *Commitment) (shutdown func()) { return commitment.WarpSyncBlocks.OnUpdate(func(_ bool, warpSyncBlocks bool) { if warpSyncBlocks { @@ -165,7 +168,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } // make sure the engine is clean and requires a warp-sync before we start processing the blocks - if targetEngine.Workers.WaitChildren(); targetEngine.Storage.Settings().LatestCommitment().ID().Slot() > commitmentID.Slot() { + if targetEngine.Storage.Settings().LatestCommitment().ID().Slot() > commitmentID.Slot() { return blocksToWarpSync } @@ -174,42 +177,64 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 2. Mark all blocks as accepted // 3. Force commitment of the slot forceCommitmentFunc := func() { + if !chain.WarpSyncMode.Get() { + return + } + + // 0. Prepare data flow + var ( + notarizedBlocksCount uint64 + allBlocksNotarized = reactive.NewEvent() + ) + // 1. Mark all transactions as accepted for _, transactionID := range transactionIDs { targetEngine.Ledger.ConflictDAG().SetAccepted(transactionID) } - // 2. Mark all blocks as accepted - for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { - for _, blockID := range blockIDs { - block, exists := targetEngine.BlockCache.Block(blockID) - if !exists { // this should never happen as we just booked these blocks in this slot. - continue + // 2. Mark all blocks as accepted and wait for them to be notarized + if totalBlocks == 0 { + allBlocksNotarized.Trigger() + } else { + for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { + for _, blockID := range blockIDs { + block, exists := targetEngine.BlockCache.Block(blockID) + if !exists { // this should never happen as we just booked these blocks in this slot. + continue + } + + // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without + // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the + // block cache and thus if not root blocks no block in the next slot can become solid. + targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) + + targetEngine.BlockGadget.SetAccepted(block) + + block.Notarized().OnTrigger(func() { + if atomic.AddUint64(¬arizedBlocksCount, 1) == uint64(totalBlocks) { + allBlocksNotarized.Trigger() + } + }) } - - // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without - // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the - // block cache and thus if not root blocks no block in the next slot can become solid. - targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) - - targetEngine.BlockGadget.SetAccepted(block) } } - // 3. Force commitment of the slot - producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) - if err != nil { - w.protocol.LogError("failed to force commitment", "commitmentID", commitmentID, "err", err) + allBlocksNotarized.OnTrigger(func() { + // 3. Force commitment of the slot + producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) + if err != nil { + w.protocol.LogError("failed to force commitment", "commitmentID", commitmentID, "err", err) - return - } + return + } - // 4. Verify that the produced commitment is the same as the initially requested one - if producedCommitment.ID() != commitmentID { - w.protocol.LogError("commitment does not match", "expectedCommitmentID", commitmentID, "producedCommitmentID", producedCommitment.ID()) + // 4. Verify that the produced commitment is the same as the initially requested one + if producedCommitment.ID() != commitmentID { + w.protocol.LogError("commitment does not match", "expectedCommitmentID", commitmentID, "producedCommitmentID", producedCommitment.ID()) - return - } + return + } + }) } commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { @@ -237,8 +262,6 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo for _, blockID := range blockIDs { blocksToWarpSync.Add(blockID) - w.LogError("requesting block", "blockID", blockID) - block, _ := targetEngine.BlockDAG.GetOrRequestBlock(blockID) if block == nil { w.protocol.LogError("failed to request block", "blockID", blockID) From b4d6346265ed7f23aa216576b3f36794887377ee Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Wed, 29 Nov 2023 01:55:21 +0100 Subject: [PATCH 10/55] Feat: increased timeout --- pkg/tests/loss_of_acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index 9ab862f80..acf2aa39c 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -32,7 +32,7 @@ func TestLossOfAcceptanceFromGenesis(t *testing.T) { 5, ), ), - testsuite.WithWaitFor(15*time.Second), + testsuite.WithWaitFor(30*time.Second), ) defer ts.Shutdown() From e1bfcdfc32c312bba572e1691078191d1f911ed4 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Wed, 29 Nov 2023 03:09:03 +0100 Subject: [PATCH 11/55] Feat: extend waiting time? --- pkg/tests/loss_of_acceptance_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index acf2aa39c..eac280eef 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -173,6 +173,8 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { { ts.IssueBlocksAtSlots("", []iotago.SlotIndex{21, 22}, 2, "block0", ts.Nodes("node0-restarted"), true, nil) + time.Sleep(10 * time.Second) + ts.AssertEqualStoredCommitmentAtIndex(20, ts.Nodes()...) ts.AssertLatestCommitmentSlotIndex(20, ts.Nodes()...) } From 1e4b2f26633bed120ed1da300cb5fa6eab9f020b Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:59:21 +0100 Subject: [PATCH 12/55] Feat: Added PR for bundled changes while debugging --- pkg/tests/loss_of_acceptance_test.go | 2 +- pkg/testsuite/mock/node.go | 101 ++++++++++++++------------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index eac280eef..3929165cc 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -127,7 +127,7 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { node2 := ts.AddNode("node2") ts.Run(true, nil) - node2.Protocol.SetLogLevel(log.LevelTrace) + node2.Protocol.SetLogLevel(log.LevelDebug) // Issue up to slot 10, committing slot 8. { diff --git a/pkg/testsuite/mock/node.go b/pkg/testsuite/mock/node.go index a3b6b8c56..5d6531796 100644 --- a/pkg/testsuite/mock/node.go +++ b/pkg/testsuite/mock/node.go @@ -191,11 +191,11 @@ func (n *Node) hookLogging(failOnBlockFiltered bool) { }) } -func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engine.Engine, engineName string) { +func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engine.Engine) { events := instance.Events events.BlockDAG.BlockAttached.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockAttached: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockAttached", "block", block.ID()) n.mutex.Lock() defer n.mutex.Unlock() @@ -203,78 +203,80 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi }) events.BlockDAG.BlockSolid.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockSolid: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockSolid", "block", block.ID()) }) events.BlockDAG.BlockInvalid.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] BlockDAG.BlockInvalid: %s - %s\n", n.Name, engineName, block.ID(), err) + instance.LogTrace("BlockDAG.BlockInvalid", "block", block.ID(), "err", err) }) events.BlockDAG.BlockMissing.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockMissing: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockMissing", "block", block.ID()) }) events.BlockDAG.MissingBlockAttached.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.MissingBlockAttached: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.MissingBlockAttached", "block", block.ID()) }) events.SeatManager.BlockProcessed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] SybilProtection.BlockProcessed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("SeatManager.BlockProcessed", "block", block.ID()) }) events.Booker.BlockBooked.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Booker.BlockBooked: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Booker.BlockBooked", "block", block.ID()) }) events.Booker.BlockInvalid.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] Booker.BlockInvalid: %s - %s\n", n.Name, engineName, block.ID(), err.Error()) + instance.LogTrace("Booker.BlockInvalid", "block", block.ID(), "err", err) }) events.Booker.TransactionInvalid.Hook(func(metadata mempool.TransactionMetadata, err error) { - fmt.Printf("%s > [%s] Booker.TransactionInvalid: %s - %s\n", n.Name, engineName, metadata.ID(), err.Error()) + instance.LogTrace("Booker.TransactionInvalid", "tx", metadata.ID(), "err", err) }) events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockScheduled: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockScheduled", "block", block.ID()) }) events.Scheduler.BlockEnqueued.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockEnqueued: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockEnqueued", "block", block.ID()) }) events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockSkipped: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockSkipped", "block", block.ID()) }) events.Scheduler.BlockDropped.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] Scheduler.BlockDropped: %s - %s\n", n.Name, engineName, block.ID(), err.Error()) + instance.LogTrace("Scheduler.BlockDropped", "block", block.ID(), "err", err) }) events.Clock.AcceptedTimeUpdated.Hook(func(newTime time.Time) { - fmt.Printf("%s > [%s] Clock.AcceptedTimeUpdated: %s [Slot %d]\n", n.Name, engineName, newTime, instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) + instance.LogTrace("Clock.AcceptedTimeUpdated", "time", newTime, "slot", instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) }) events.Clock.ConfirmedTimeUpdated.Hook(func(newTime time.Time) { - fmt.Printf("%s > [%s] Clock.ConfirmedTimeUpdated: %s [Slot %d]\n", n.Name, engineName, newTime, instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) + instance.LogTrace("Clock.ConfirmedTimeUpdated", "time", newTime, "slot", instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) }) events.PreSolidFilter.BlockPreAllowed.Hook(func(block *model.Block) { - fmt.Printf("%s > [%s] PreSolidFilter.BlockPreAllowed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("PreSolidFilter.BlockPreAllowed", "block", block.ID()) }) events.PreSolidFilter.BlockPreFiltered.Hook(func(event *presolidfilter.BlockPreFilteredEvent) { - fmt.Printf("%s > [%s] PreSolidFilter.BlockPreFiltered: %s - %s\n", n.Name, engineName, event.Block.ID(), event.Reason.Error()) + instance.LogTrace("PreSolidFilter.BlockPreFiltered", "block", event.Block.ID(), "err", event.Reason) + if failOnBlockFiltered { n.Testing.Fatal("no blocks should be prefiltered") } }) events.PostSolidFilter.BlockAllowed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] PostSolidFilter.BlockAllowed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("PostSolidFilter.BlockAllowed", "block", block.ID()) }) events.PostSolidFilter.BlockFiltered.Hook(func(event *postsolidfilter.BlockFilteredEvent) { - fmt.Printf("%s > [%s] PostSolidFilter.BlockFiltered: %s - %s\n", n.Name, engineName, event.Block.ID(), event.Reason.Error()) + instance.LogTrace("PostSolidFilter.BlockFiltered", "block", event.Block.ID(), "err", event.Reason) + if failOnBlockFiltered { n.Testing.Fatal("no blocks should be filtered") } @@ -285,11 +287,11 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi }) events.BlockRequester.Tick.Hook(func(blockID iotago.BlockID) { - fmt.Printf("%s > [%s] BlockRequester.Tick: %s\n", n.Name, engineName, blockID) + instance.LogTrace("BlockRequester.Tick", "block", blockID) }) events.BlockProcessed.Hook(func(blockID iotago.BlockID) { - fmt.Printf("%s > [%s] Engine.BlockProcessed: %s\n", n.Name, engineName, blockID) + instance.LogTrace("BlockProcessed", "block", blockID) }) events.Notarization.SlotCommitted.Hook(func(details *notarization.SlotCommittedDetails) { @@ -316,109 +318,108 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi require.NoError(n.Testing, err) } - fmt.Printf("%s > [%s] NotarizationManager.SlotCommitted: %s %s Accepted Blocks: %s\n %s\n Attestations: %s\n", n.Name, engineName, details.Commitment.ID(), details.Commitment, acceptedBlocks, roots, attestationBlockIDs) + instance.LogTrace("NotarizationManager.SlotCommitted", "commitment", details.Commitment.ID(), "acceptedBlocks", acceptedBlocks, "roots", roots, "attestations", attestationBlockIDs) }) events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { - fmt.Printf("%s > [%s] NotarizationManager.LatestCommitmentUpdated: %s\n", n.Name, engineName, commitment.ID()) + instance.LogTrace("NotarizationManager.LatestCommitmentUpdated", "commitment", commitment.ID()) }) events.BlockGadget.BlockPreAccepted.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockPreAccepted: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockPreAccepted", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockAccepted.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockAccepted: %s @ slot %s committing to %s\n", n.Name, engineName, block.ID(), block.ID().Slot(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockAccepted", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockPreConfirmed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockPreConfirmed: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockPreConfirmed", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockConfirmed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockConfirmed: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockConfirmed", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.SlotGadget.SlotFinalized.Hook(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] Consensus.SlotGadget.SlotFinalized: %s\n", n.Name, engineName, slot) + instance.LogTrace("SlotGadget.SlotFinalized", "slot", slot) }) events.SeatManager.OnlineCommitteeSeatAdded.Hook(func(seat account.SeatIndex, accountID iotago.AccountID) { - fmt.Printf("%s > [%s] SybilProtection.OnlineCommitteeSeatAdded: %d - %s\n", n.Name, engineName, seat, accountID) + instance.LogTrace("SybilProtection.OnlineCommitteeSeatAdded", "seat", seat, "accountID", accountID) }) events.SeatManager.OnlineCommitteeSeatRemoved.Hook(func(seat account.SeatIndex) { - fmt.Printf("%s > [%s] SybilProtection.OnlineCommitteeSeatRemoved: %d\n", n.Name, engineName, seat) + instance.LogTrace("SybilProtection.OnlineCommitteeSeatRemoved", "seat", seat) }) events.SybilProtection.CommitteeSelected.Hook(func(committee *account.Accounts, epoch iotago.EpochIndex) { - fmt.Printf("%s > [%s] SybilProtection.CommitteeSelected: epoch %d - %s\n", n.Name, engineName, epoch, committee.IDs()) + instance.LogTrace("SybilProtection.CommitteeSelected", "epoch", epoch, "committee", committee.IDs()) }) events.ConflictDAG.ConflictCreated.Hook(func(conflictID iotago.TransactionID) { - fmt.Printf("%s > [%s] ConflictDAG.ConflictCreated: %s\n", n.Name, engineName, conflictID) + instance.LogTrace("ConflictDAG.ConflictCreated", "conflictID", conflictID) }) events.ConflictDAG.ConflictEvicted.Hook(func(conflictID iotago.TransactionID) { - fmt.Printf("%s > [%s] ConflictDAG.ConflictEvicted: %s\n", n.Name, engineName, conflictID) + instance.LogTrace("ConflictDAG.ConflictEvicted", "conflictID", conflictID) }) + events.ConflictDAG.ConflictRejected.Hook(func(conflictID iotago.TransactionID) { - fmt.Printf("%s > [%s] ConflictDAG.ConflictRejected: %s\n", n.Name, engineName, conflictID) + instance.LogTrace("ConflictDAG.ConflictRejected", "conflictID", conflictID) }) events.ConflictDAG.ConflictAccepted.Hook(func(conflictID iotago.TransactionID) { - fmt.Printf("%s > [%s] ConflictDAG.ConflictAccepted: %s\n", n.Name, engineName, conflictID) + instance.LogTrace("ConflictDAG.ConflictAccepted", "conflictID", conflictID) }) instance.Ledger.OnTransactionAttached(func(transactionMetadata mempool.TransactionMetadata) { - fmt.Printf("%s > [%s] Ledger.TransactionAttached: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("Ledger.TransactionAttached", "tx", transactionMetadata.ID()) transactionMetadata.OnSolid(func() { - fmt.Printf("%s > [%s] MemPool.TransactionSolid: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionSolid", "tx", transactionMetadata.ID()) }) transactionMetadata.OnExecuted(func() { - fmt.Printf("%s > [%s] MemPool.TransactionExecuted: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionExecuted", "tx", transactionMetadata.ID()) }) transactionMetadata.OnBooked(func() { - fmt.Printf("%s > [%s] MemPool.TransactionBooked: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionBooked", "tx", transactionMetadata.ID()) }) transactionMetadata.OnConflicting(func() { - fmt.Printf("%s > [%s] MemPool.TransactionConflicting: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionConflicting", "tx", transactionMetadata.ID()) }) transactionMetadata.OnAccepted(func() { - fmt.Printf("%s > [%s] MemPool.TransactionAccepted: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionAccepted", "tx", transactionMetadata.ID()) }) transactionMetadata.OnRejected(func() { - fmt.Printf("%s > [%s] MemPool.TransactionRejected: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionRejected", "tx", transactionMetadata.ID()) }) transactionMetadata.OnInvalid(func(err error) { - fmt.Printf("%s > [%s] MemPool.TransactionInvalid(%s): %s\n", n.Name, engineName, err, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionInvalid", "tx", transactionMetadata.ID(), "err", err) }) transactionMetadata.OnOrphanedSlotUpdated(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] MemPool.TransactionOrphanedSlotUpdated in slot %d: %s\n", n.Name, engineName, slot, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionOrphanedSlotUpdated", "tx", transactionMetadata.ID(), "slot", slot) }) transactionMetadata.OnCommittedSlotUpdated(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] MemPool.TransactionCommittedSlotUpdated in slot %d: %s\n", n.Name, engineName, slot, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionCommittedSlotUpdated", "tx", transactionMetadata.ID(), "slot", slot) }) transactionMetadata.OnPending(func() { - fmt.Printf("%s > [%s] MemPool.TransactionPending: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionPending", "tx", transactionMetadata.ID()) }) }) } func (n *Node) attachEngineLogs(failOnBlockFiltered bool, instance *engine.Engine) { - engineName := fmt.Sprintf("%s - %s", lo.Cond(n.Protocol.Engines.Main.Get() != instance, "Candidate", "Main"), instance.Name()[:8]) - - n.attachEngineLogsWithName(failOnBlockFiltered, instance, engineName) + n.attachEngineLogsWithName(failOnBlockFiltered, instance) } func (n *Node) Wait() { From a467a260c8004cf7425b873a4e596c40904c039f Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Wed, 29 Nov 2023 11:20:02 +0100 Subject: [PATCH 13/55] Feat: added more logs --- pkg/protocol/protocol_blocks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index b18983f3b..4550f9818 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -108,7 +108,7 @@ func (b *BlocksProtocol) ProcessResponse(block *model.Block, from peer.ID) { if !b.droppedBlocksBuffer.Add(block.ProtocolBlock().Header.SlotCommitmentID, types.NewTuple(block, from)) { b.LogError("failed to add dropped block referencing unsolid commitment to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) } else { - b.LogTrace("dropped block referencing unsolid commitment added to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) + b.LogDebug("dropped block referencing unsolid commitment added to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) } return From ed3888792cbee5123a8e61606ceb697510adad76 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 00:09:37 +0100 Subject: [PATCH 14/55] Feat: added log output --- pkg/protocol/protocol_blocks.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 4550f9818..41a675a9f 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -112,9 +112,11 @@ func (b *BlocksProtocol) ProcessResponse(block *model.Block, from peer.ID) { } return + } else if block.ProtocolBlock().Header.SlotCommitmentID.Slot() >= 19 { + b.LogError("received block", "blockID", block.ID(), "commitment", commitment.LogName()) + } else { + b.LogTrace("received block", "blockID", block.ID(), "commitment", commitment.LogName()) } - - b.LogTrace("received block", "blockID", block.ID(), "commitment", commitment.LogName()) }) } From 0db2bf006e7cc359b74adb4abfda4e0aa528d603 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 09:38:55 +0100 Subject: [PATCH 15/55] Feat: changed some code --- pkg/protocol/protocol_blocks.go | 37 +++++++++++++++++----------- pkg/tests/loss_of_acceptance_test.go | 2 -- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 41a675a9f..1a2e948c6 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -53,22 +53,29 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { }) }) - protocol.Chains.WithElements(func(chain *Chain) func() { - return chain.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return engineInstance.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook - }) - }) - - protocol.Chains.Main.Get().Engine.OnUpdateWithContext(func(_ *engine.Engine, engine *engine.Engine, unsubscribeOnEngineChange func(subscriptionFactory func() (unsubscribe func()))) { - if engine != nil { - unsubscribeOnEngineChange(func() (unsubscribe func()) { - return lo.Batch( - engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - ) - }) - } + //protocol.Chains.WithElements(func(chain *Chain) func() { + // return chain.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { + // return engineInstance.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook + // }) + //}) + + protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { + return lo.Batch( + engine.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook, + engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + ) }) + //protocol.Chains.Main.Get().Engine.OnUpdateWithContext(func(_ *engine.Engine, engine *engine.Engine, unsubscribeOnEngineChange func(subscriptionFactory func() (unsubscribe func()))) { + // if engine != nil { + // unsubscribeOnEngineChange(func() (unsubscribe func()) { + // return lo.Batch( + // engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + // engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + // ) + // }) + // } + //}) }) return b diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index 3929165cc..c0d1d2420 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -173,8 +173,6 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { { ts.IssueBlocksAtSlots("", []iotago.SlotIndex{21, 22}, 2, "block0", ts.Nodes("node0-restarted"), true, nil) - time.Sleep(10 * time.Second) - ts.AssertEqualStoredCommitmentAtIndex(20, ts.Nodes()...) ts.AssertLatestCommitmentSlotIndex(20, ts.Nodes()...) } From 13cd73a53e62b61424b2a6428f2250b2ad622c37 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:48:30 +0100 Subject: [PATCH 16/55] Feat: added log --- pkg/tests/loss_of_acceptance_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index c0d1d2420..d5afd8a66 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -171,6 +171,8 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { // Need to issue to slot 22 so that all other nodes can warp sync up to slot 19 and then commit slot 20 themselves. { + fmt.Println("ISSUE BLOCKS SO THAT NODES CAN WARP SYNC UP TO SLOT 19") + ts.IssueBlocksAtSlots("", []iotago.SlotIndex{21, 22}, 2, "block0", ts.Nodes("node0-restarted"), true, nil) ts.AssertEqualStoredCommitmentAtIndex(20, ts.Nodes()...) From 8f60e3de6b65295314e5abf7ecdbc05f48988088 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:52:06 +0100 Subject: [PATCH 17/55] Feat: add logging --- pkg/protocol/protocol_blocks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 1a2e948c6..4833486cd 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -45,7 +45,7 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { return commitment.ReplayDroppedBlocks.OnUpdate(func(_ bool, replayBlocks bool) { if replayBlocks { for _, droppedBlock := range b.droppedBlocksBuffer.GetValues(commitment.ID()) { - b.LogTrace("replaying dropped block", "commitmentID", commitment.ID(), "blockID", droppedBlock.A.ID()) + b.LogError("replaying dropped block", "commitmentID", commitment.ID(), "blockID", droppedBlock.A.ID()) b.ProcessResponse(droppedBlock.A, droppedBlock.B) } From a52498764e14c006cf87628100b38045f5f90a53 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 13:15:30 +0100 Subject: [PATCH 18/55] Feat: added logging --- pkg/tests/loss_of_acceptance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index d5afd8a66..96609bcb6 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -127,7 +127,7 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { node2 := ts.AddNode("node2") ts.Run(true, nil) - node2.Protocol.SetLogLevel(log.LevelDebug) + node2.Protocol.SetLogLevel(log.LevelTrace) // Issue up to slot 10, committing slot 8. { From ff87a2e64754bab1758949513df16385fc05e271 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 15:54:04 +0100 Subject: [PATCH 19/55] Feat: added solidity stuff to the logging - let's see --- .../engine/blockdag/inmemoryblockdag/blockdag.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go index 6a9c3cdf4..d7fb579c9 100644 --- a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go +++ b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go @@ -1,8 +1,10 @@ package inmemoryblockdag import ( + "fmt" "sync/atomic" + "github.com/iotaledger/hive.go/ds" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" @@ -74,6 +76,8 @@ func (b *BlockDAG) setupBlock(block *blocks.Block) { var unsolidParentsCount atomic.Int32 unsolidParentsCount.Store(int32(len(block.Parents()))) + unsolidParents := ds.NewSet[iotago.BlockID]() + block.ForEachParent(func(parent iotago.Parent) { parentBlock, exists := b.blockCache.Block(parent.ID) if !exists { @@ -82,7 +86,13 @@ func (b *BlockDAG) setupBlock(block *blocks.Block) { return } + unsolidParents.Add(parent.ID) + fmt.Println("unsolid Parents of ", block.ID(), unsolidParents) + parentBlock.Solid().OnUpdateOnce(func(_ bool, _ bool) { + unsolidParents.Delete(parent.ID) + fmt.Println("unsolid Parents of ", block.ID(), unsolidParents) + if unsolidParentsCount.Add(-1) == 0 { if block.SetSolid() { b.events.BlockSolid.Trigger(block) From 8b97187593daf890a630706e290f601d2e53142e Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Thu, 30 Nov 2023 16:52:05 +0100 Subject: [PATCH 20/55] Feat: add logging --- pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go index d7fb579c9..7a2c1828f 100644 --- a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go +++ b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go @@ -93,8 +93,10 @@ func (b *BlockDAG) setupBlock(block *blocks.Block) { unsolidParents.Delete(parent.ID) fmt.Println("unsolid Parents of ", block.ID(), unsolidParents) - if unsolidParentsCount.Add(-1) == 0 { + if counter := unsolidParentsCount.Add(-1); counter == 0 { + fmt.Println("unsolid counter", counter) if block.SetSolid() { + fmt.Println("unsolid Trigger", counter) b.events.BlockSolid.Trigger(block) } } From aa6642ed8e03136b1bd1c2f1665d2997e9eaf949 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:51:22 +0800 Subject: [PATCH 21/55] Change warp sync to wait for WeightPropagated instead of Booked of blocks before commitment is triggered --- pkg/protocol/protocol_warp_sync.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 38e6b2244..844881d89 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -196,18 +196,13 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo if totalBlocks == 0 { allBlocksNotarized.Trigger() } else { - for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { + for _, blockIDs := range blockIDsBySlotCommitment { for _, blockID := range blockIDs { block, exists := targetEngine.BlockCache.Block(blockID) if !exists { // this should never happen as we just booked these blocks in this slot. continue } - // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without - // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the - // block cache and thus if not root blocks no block in the next slot can become solid. - targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) - targetEngine.BlockGadget.SetAccepted(block) block.Notarized().OnTrigger(func() { @@ -234,6 +229,21 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } + + // TODO: maybe we don't need to set root blocks as we leave a window for warp sync now. + for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { + for _, blockID := range blockIDs { + block, exists := targetEngine.BlockCache.Block(blockID) + if !exists { // this should never happen as we just booked these blocks in this slot. + continue + } + + // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without + // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the + // block cache and thus if not root blocks no block in the next slot can become solid. + targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) + } + } }) } @@ -241,7 +251,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // Let's assume that MCA is 5: when we want to book 15, we expect to have the commitment of 10 to load // accounts from it, hence why we make committable the slot at - MCA + 1 with respect of the current slot. minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() - if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge); exists { committableCommitment.IsCommittable.Set(true) } }) @@ -269,8 +279,9 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo continue } - // TODO: this will need to be changed to WeightPropagated before marking a block as actually booked. - block.Booked().OnUpdate(func(_ bool, _ bool) { + // We need to make sure that all blocks are fully booked and their weight propagated before we can + // move the window forward. This is in order to ensure that confirmation and finalization is correctly propagated. + block.WeightPropagated().OnUpdate(func(_ bool, _ bool) { if bookedBlocks.Add(1) != totalBlocks { return } From bb338aff99922a3fde7aa79398eaa06110b3b7d0 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:52:54 +0800 Subject: [PATCH 22/55] Adjust TestProtocol_EngineSwitching_CommitteeRotation to wait for the correct slot after node3 switched the engine --- pkg/tests/protocol_engine_switching_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/tests/protocol_engine_switching_test.go b/pkg/tests/protocol_engine_switching_test.go index a6ab346f3..046ffbd99 100644 --- a/pkg/tests/protocol_engine_switching_test.go +++ b/pkg/tests/protocol_engine_switching_test.go @@ -377,7 +377,7 @@ func TestProtocol_EngineSwitching(t *testing.T) { func TestProtocol_EngineSwitching_CommitteeRotation(t *testing.T) { ts := testsuite.NewTestSuite(t, - testsuite.WithWaitFor(30*time.Second), + testsuite.WithWaitFor(15*time.Second), testsuite.WithProtocolParametersOptions( iotago.WithTimeProviderOptions( @@ -402,7 +402,7 @@ func TestProtocol_EngineSwitching_CommitteeRotation(t *testing.T) { node2 := ts.AddValidatorNode("node2") node3 := ts.AddValidatorNode("node3") - const expectedCommittedSlotAfterPartitionMerge = 19 + const expectedCommittedSlotAfterPartitionMerge = 18 nodesP1 := []*mock.Node{node0, node1, node2} nodesP2 := []*mock.Node{node3} @@ -605,6 +605,9 @@ func TestProtocol_EngineSwitching_CommitteeRotation(t *testing.T) { // Here we need to let enough time pass for the nodes to sync up the candidate engines and switch them ts.AssertMainEngineSwitchedCount(1, nodesP2...) + // Make sure that enough activity messages are issued so that a block in slot 21 gets accepted and triggers commitment of slot 18. + time.Sleep(3 * time.Second) + ctxP1Cancel() wg.Wait() } @@ -613,12 +616,10 @@ func TestProtocol_EngineSwitching_CommitteeRotation(t *testing.T) { // Those nodes should also have all the blocks from the target fork P1 and should not have blocks from P2. // This is to make sure that the storage was copied correctly during engine switching. ts.AssertBlocksExist(ts.BlocksWithPrefix("P0"), true, ts.Nodes()...) - ts.AssertBlocksExist(ts.BlocksWithPrefix("P1"), true, ts.Nodes()...) // not all blocks of slot 19 are available on node3 (buffer issue?) + ts.AssertBlocksExist(ts.BlocksWithPrefix("P1"), true, ts.Nodes()...) ts.AssertBlocksExist(ts.BlocksWithPrefix("P2"), false, ts.Nodes()...) - ts.AssertNodeState(ts.Nodes(), - testsuite.WithEqualStoredCommitmentAtIndex(expectedCommittedSlotAfterPartitionMerge), - ) + ts.AssertEqualStoredCommitmentAtIndex(expectedCommittedSlotAfterPartitionMerge, ts.Nodes()...) // Assert committee in epoch 1. ts.AssertSybilProtectionCandidates(0, ts.AccountsOfNodes("node1", "node2"), ts.Nodes()...) From aeb92580e279a9ec166da6da3ec86f99e919a5c7 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:59:13 +0800 Subject: [PATCH 23/55] Fix calculation of WarpSyncThreshold --- pkg/protocol/chain.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 557f5d3e7..01c88f98b 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -325,7 +325,12 @@ func (c *Chain) deriveOutOfSyncThreshold(latestSeenSlot reactive.ReadableVariabl // committable age or 0 if this would cause an overflow to the negative numbers). func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { return c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - if warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); warpSyncOffset < latestSeenSlot { + // The offset needs to be always at least minCommittableAge as we measure whether to turn off WarpSyncMode + // against the LatestProducedCommitment. As such, we always need to warp sync at least a window of minCommittableAge slots, + // so that we can commit. + // For example assuming minCommittable=3: if we warpsync until slot 17, the highest commitment we can produce is 14. + warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() + engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge() + if warpSyncOffset < latestSeenSlot { return latestSeenSlot - warpSyncOffset } From 62a9cb494857ec771f993b112f34aff1fd6988a2 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:59:43 +0800 Subject: [PATCH 24/55] WarpSyncMode should be enabled if latestProducedCommitment == nil --- pkg/protocol/chain.go | 5 ++++- pkg/protocol/engines.go | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 01c88f98b..620903d3b 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -232,9 +232,12 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { // deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. func (c *Chain) deriveWarpSyncMode() func() { return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + c.Log("warp sync mode", log.LevelTrace, "warpSyncMode", warpSyncMode, "latestProducedCommitment", latestProducedCommitment, "warpSyncThreshold", warpSyncThreshold, "outOfSyncThreshold", outOfSyncThreshold) + + // TODO: we don't set a latestProducedCommitment when forking, and therefore it is always nil after an engine is forked // if we have no latest produced commitment, then the engine is not yet initialized and warp sync is disabled if latestProducedCommitment == nil { - return false + return true } // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold diff --git a/pkg/protocol/engines.go b/pkg/protocol/engines.go index 781dc0d32..e30cfe19f 100644 --- a/pkg/protocol/engines.go +++ b/pkg/protocol/engines.go @@ -9,6 +9,7 @@ import ( "github.com/iotaledger/hive.go/ds/reactive" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" + "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/ioutils" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/workerpool" @@ -158,6 +159,8 @@ func (e *Engines) ForkAtSlot(slot iotago.SlotIndex) (*engine.Engine, error) { return nil, ierrors.Wrap(err, "error while rolling back attestations storage on candidate engine") } + e.Log("forked engine", log.LevelTrace, "name", newEngineAlias[0:8], "slot", slot) + return candidateEngine, nil } From 7243706c4cbec3ae6a77f4ccaa641c7b812d3fd5 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:49:49 +0100 Subject: [PATCH 25/55] Feat: fixed some stuff --- pkg/protocol/chain.go | 14 ++++---------- pkg/protocol/chains.go | 4 ++-- pkg/protocol/commitment.go | 1 + pkg/protocol/protocol_attestations.go | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 620903d3b..8c26912dd 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -232,12 +232,10 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { // deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. func (c *Chain) deriveWarpSyncMode() func() { return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - c.Log("warp sync mode", log.LevelTrace, "warpSyncMode", warpSyncMode, "latestProducedCommitment", latestProducedCommitment, "warpSyncThreshold", warpSyncThreshold, "outOfSyncThreshold", outOfSyncThreshold) - - // TODO: we don't set a latestProducedCommitment when forking, and therefore it is always nil after an engine is forked - // if we have no latest produced commitment, then the engine is not yet initialized and warp sync is disabled + // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during + // startup) if latestProducedCommitment == nil { - return true + return warpSyncMode } // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold @@ -328,11 +326,7 @@ func (c *Chain) deriveOutOfSyncThreshold(latestSeenSlot reactive.ReadableVariabl // committable age or 0 if this would cause an overflow to the negative numbers). func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { return c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - // The offset needs to be always at least minCommittableAge as we measure whether to turn off WarpSyncMode - // against the LatestProducedCommitment. As such, we always need to warp sync at least a window of minCommittableAge slots, - // so that we can commit. - // For example assuming minCommittable=3: if we warpsync until slot 17, the highest commitment we can produce is 14. - warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() + engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge() + warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge() if warpSyncOffset < latestSeenSlot { return latestSeenSlot - warpSyncOffset } diff --git a/pkg/protocol/chains.go b/pkg/protocol/chains.go index 2f0051148..d41c4ebab 100644 --- a/pkg/protocol/chains.go +++ b/pkg/protocol/chains.go @@ -31,7 +31,7 @@ type Chains struct { // HeaviestVerifiedCandidate contains the candidate chain with the heaviest verified weight. HeaviestVerifiedCandidate reactive.Variable[*Chain] - // LatestSeenSlot contains the latest slot that was seen by any of the chains. + // LatestSeenSlot contains the slot of the latest commitment of any received block. LatestSeenSlot reactive.Variable[iotago.SlotIndex] // protocol contains a reference to the Protocol instance that this component belongs to. @@ -158,7 +158,7 @@ func (c *Chains) deriveLatestSeenSlot(protocol *Protocol) func() { }), protocol.Network.OnBlockReceived(func(block *model.Block, src peer.ID) { - c.LatestSeenSlot.Set(mainEngine.LatestAPI().TimeProvider().SlotFromTime(block.ProtocolBlock().Header.IssuingTime)) + c.LatestSeenSlot.Set(block.ProtocolBlock().Header.SlotCommitmentID.Slot()) }), ) }) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index cd3fd8387..ca06a32ab 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -230,6 +230,7 @@ func (c *Commitment) deriveChain(parent *Commitment) func() { if currentChain == nil { currentChain = c.commitments.protocol.Chains.newChain() currentChain.ForkingPoint.Set(c) + currentChain.LatestProducedCommitment.Set(parent) } return currentChain diff --git a/pkg/protocol/protocol_attestations.go b/pkg/protocol/protocol_attestations.go index 819813758..f2c4c3153 100644 --- a/pkg/protocol/protocol_attestations.go +++ b/pkg/protocol/protocol_attestations.go @@ -30,7 +30,7 @@ type AttestationsProtocol struct { // commitmentVerifiers contains the commitment verifiers that are used to verify received attestations. commitmentVerifiers *shrinkingmap.ShrinkingMap[iotago.CommitmentID, *CommitmentVerifier] - // Logger embeds a logger that can be used to log messages emitted by this chain. + // Logger embeds a logger that can be used to log messages emitted by this component. log.Logger } From 32ebf8b507af13f68a0b802a0048bb5522349643 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:22:15 +0100 Subject: [PATCH 26/55] Fix: fixed tests --- pkg/core/buffer/unsolid_commitment_buffer.go | 2 +- pkg/protocol/chain.go | 6 +- pkg/protocol/commitment.go | 1 + .../blockdag/inmemoryblockdag/blockdag.go | 14 +---- pkg/protocol/protocol_warp_sync.go | 2 +- pkg/testsuite/mock/node.go | 60 +++++++++++++++++-- 6 files changed, 62 insertions(+), 23 deletions(-) diff --git a/pkg/core/buffer/unsolid_commitment_buffer.go b/pkg/core/buffer/unsolid_commitment_buffer.go index 27df0dd51..a6417a4d7 100644 --- a/pkg/core/buffer/unsolid_commitment_buffer.go +++ b/pkg/core/buffer/unsolid_commitment_buffer.go @@ -51,7 +51,7 @@ func (u *UnsolidCommitmentBuffer[V]) Add(commitmentID iotago.CommitmentID, value u.mutex.RLock() defer u.mutex.RUnlock() - if commitmentID.Slot() <= u.lastEvictedSlot { + if u.lastEvictedSlot != 0 && commitmentID.Slot() <= u.lastEvictedSlot { return false } diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 8c26912dd..aeaa16c02 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -92,7 +92,7 @@ func newChain(chains *Chains) *Chain { ClaimedWeight: reactive.NewVariable[uint64](), AttestedWeight: reactive.NewVariable[uint64](), VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncMode: reactive.NewVariable[bool](), + WarpSyncMode: reactive.NewVariable[bool]().Init(true), WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), RequestAttestations: reactive.NewVariable[bool](), @@ -326,9 +326,9 @@ func (c *Chain) deriveOutOfSyncThreshold(latestSeenSlot reactive.ReadableVariabl // committable age or 0 if this would cause an overflow to the negative numbers). func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { return c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge() + warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() if warpSyncOffset < latestSeenSlot { - return latestSeenSlot - warpSyncOffset + return latestSeenSlot - warpSyncOffset + 1 } return 0 diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index ca06a32ab..d739bc43f 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -158,6 +158,7 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( // mark commitments that are marked as root as verified c.IsCommitted.InheritFrom(c.IsRoot), + c.IsAboveLatestVerifiedCommitment.InheritFrom(c.IsRoot), // mark commitments that are marked as verified as attested, fully booked and committable c.IsAttested.InheritFrom(c.IsCommitted), diff --git a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go index 7a2c1828f..6a9c3cdf4 100644 --- a/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go +++ b/pkg/protocol/engine/blockdag/inmemoryblockdag/blockdag.go @@ -1,10 +1,8 @@ package inmemoryblockdag import ( - "fmt" "sync/atomic" - "github.com/iotaledger/hive.go/ds" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" @@ -76,8 +74,6 @@ func (b *BlockDAG) setupBlock(block *blocks.Block) { var unsolidParentsCount atomic.Int32 unsolidParentsCount.Store(int32(len(block.Parents()))) - unsolidParents := ds.NewSet[iotago.BlockID]() - block.ForEachParent(func(parent iotago.Parent) { parentBlock, exists := b.blockCache.Block(parent.ID) if !exists { @@ -86,17 +82,9 @@ func (b *BlockDAG) setupBlock(block *blocks.Block) { return } - unsolidParents.Add(parent.ID) - fmt.Println("unsolid Parents of ", block.ID(), unsolidParents) - parentBlock.Solid().OnUpdateOnce(func(_ bool, _ bool) { - unsolidParents.Delete(parent.ID) - fmt.Println("unsolid Parents of ", block.ID(), unsolidParents) - - if counter := unsolidParentsCount.Add(-1); counter == 0 { - fmt.Println("unsolid counter", counter) + if unsolidParentsCount.Add(-1) == 0 { if block.SetSolid() { - fmt.Println("unsolid Trigger", counter) b.events.BlockSolid.Trigger(block) } } diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 844881d89..fb6a3c111 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -251,7 +251,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // Let's assume that MCA is 5: when we want to book 15, we expect to have the commitment of 10 to load // accounts from it, hence why we make committable the slot at - MCA + 1 with respect of the current slot. minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() - if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge); exists { + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { committableCommitment.IsCommittable.Set(true) } }) diff --git a/pkg/testsuite/mock/node.go b/pkg/testsuite/mock/node.go index 2d1ca30fc..b9031ee2e 100644 --- a/pkg/testsuite/mock/node.go +++ b/pkg/testsuite/mock/node.go @@ -45,6 +45,11 @@ func UnregisterIDAliases() { idAliases = make(map[peer.ID]string) } +type InvalidSignedTransactionEvent struct { + Metadata mempool.SignedTransactionMetadata + Error error +} + type Node struct { Testing *testing.T @@ -72,10 +77,11 @@ type Node struct { logHandler slog.Handler enableEngineLogging bool - mutex syncutils.RWMutex - attachedBlocks []*blocks.Block - currentSlot iotago.SlotIndex - filteredBlockEvents []*postsolidfilter.BlockFilteredEvent + mutex syncutils.RWMutex + attachedBlocks []*blocks.Block + currentSlot iotago.SlotIndex + filteredBlockEvents []*postsolidfilter.BlockFilteredEvent + invalidTransactionEvents map[iotago.SignedTransactionID]InvalidSignedTransactionEvent } func NewNode(t *testing.T, net *Network, partition string, name string, validator bool, logHandler slog.Handler) *Node { @@ -113,7 +119,8 @@ func NewNode(t *testing.T, net *Network, partition string, name string, validato logHandler: logHandler, enableEngineLogging: true, - attachedBlocks: make([]*blocks.Block, 0), + attachedBlocks: make([]*blocks.Block, 0), + invalidTransactionEvents: make(map[iotago.SignedTransactionID]InvalidSignedTransactionEvent), } } @@ -179,6 +186,33 @@ func (n *Node) hookEvents() { n.filteredBlockEvents = append(n.filteredBlockEvents, event) }) + + n.Protocol.Engines.Main.Get().Ledger.MemPool().OnSignedTransactionAttached( + func(signedTransactionMetadata mempool.SignedTransactionMetadata) { + signedTxID := signedTransactionMetadata.ID() + + signedTransactionMetadata.OnSignaturesInvalid(func(err error) { + n.mutex.Lock() + defer n.mutex.Unlock() + + n.invalidTransactionEvents[signedTxID] = InvalidSignedTransactionEvent{ + Metadata: signedTransactionMetadata, + Error: err, + } + }) + + transactionMetadata := signedTransactionMetadata.TransactionMetadata() + + transactionMetadata.OnInvalid(func(err error) { + n.mutex.Lock() + defer n.mutex.Unlock() + + n.invalidTransactionEvents[signedTxID] = InvalidSignedTransactionEvent{ + Metadata: signedTransactionMetadata, + Error: err, + } + }) + }) } func (n *Node) hookLogging(failOnBlockFiltered bool) { @@ -373,6 +407,14 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi instance.LogTrace("ConflictDAG.SpendAccepted", "conflictID", conflictID) }) + instance.Ledger.MemPool().OnSignedTransactionAttached( + func(signedTransactionMetadata mempool.SignedTransactionMetadata) { + signedTransactionMetadata.OnSignaturesInvalid(func(err error) { + instance.LogTrace("MemPool.SignedTransactionSignaturesInvalid", "tx", signedTransactionMetadata.ID(), "err", err) + }) + }, + ) + instance.Ledger.OnTransactionAttached(func(transactionMetadata mempool.TransactionMetadata) { instance.LogTrace("Ledger.TransactionAttached", "tx", transactionMetadata.ID()) @@ -487,6 +529,14 @@ func (n *Node) MainEngineSwitchedCount() int { return int(n.mainEngineSwitchedCount.Load()) } +func (n *Node) TransactionFailure(txID iotago.SignedTransactionID) (InvalidSignedTransactionEvent, bool) { + n.mutex.RLock() + defer n.mutex.RUnlock() + event, exists := n.invalidTransactionEvents[txID] + + return event, exists +} + func (n *Node) AttachedBlocks() []*blocks.Block { n.mutex.RLock() defer n.mutex.RUnlock() From b3049b5f74f58fa254968ddb8febf6328b42df2c Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:25:36 +0100 Subject: [PATCH 27/55] Refactor: reverted changes --- pkg/protocol/protocol_blocks.go | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index 4833486cd..3afd74807 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -45,7 +45,7 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { return commitment.ReplayDroppedBlocks.OnUpdate(func(_ bool, replayBlocks bool) { if replayBlocks { for _, droppedBlock := range b.droppedBlocksBuffer.GetValues(commitment.ID()) { - b.LogError("replaying dropped block", "commitmentID", commitment.ID(), "blockID", droppedBlock.A.ID()) + b.LogTrace("replaying dropped block", "commitmentID", commitment.ID(), "blockID", droppedBlock.A.ID()) b.ProcessResponse(droppedBlock.A, droppedBlock.B) } @@ -53,12 +53,6 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { }) }) - //protocol.Chains.WithElements(func(chain *Chain) func() { - // return chain.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - // return engineInstance.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook - // }) - //}) - protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { return lo.Batch( engine.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook, @@ -66,16 +60,6 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, ) }) - //protocol.Chains.Main.Get().Engine.OnUpdateWithContext(func(_ *engine.Engine, engine *engine.Engine, unsubscribeOnEngineChange func(subscriptionFactory func() (unsubscribe func()))) { - // if engine != nil { - // unsubscribeOnEngineChange(func() (unsubscribe func()) { - // return lo.Batch( - // engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - // engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - // ) - // }) - // } - //}) }) return b @@ -115,15 +99,13 @@ func (b *BlocksProtocol) ProcessResponse(block *model.Block, from peer.ID) { if !b.droppedBlocksBuffer.Add(block.ProtocolBlock().Header.SlotCommitmentID, types.NewTuple(block, from)) { b.LogError("failed to add dropped block referencing unsolid commitment to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) } else { - b.LogDebug("dropped block referencing unsolid commitment added to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) + b.LogTrace("dropped block referencing unsolid commitment added to dropped blocks buffer", "commitmentID", block.ProtocolBlock().Header.SlotCommitmentID, "blockID", block.ID()) } return - } else if block.ProtocolBlock().Header.SlotCommitmentID.Slot() >= 19 { - b.LogError("received block", "blockID", block.ID(), "commitment", commitment.LogName()) - } else { - b.LogTrace("received block", "blockID", block.ID(), "commitment", commitment.LogName()) } + + b.LogTrace("received block", "blockID", block.ID(), "commitment", commitment.LogName()) }) } From 9c845ec9d8b03983b10ec37f5ee25ca24aa1f23f Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:26:42 +0100 Subject: [PATCH 28/55] Refactor: reverted more changes --- pkg/tests/loss_of_acceptance_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index 62cf680d8..93ec48b57 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -171,8 +171,6 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { // Need to issue to slot 22 so that all other nodes can warp sync up to slot 19 and then commit slot 20 themselves. { - fmt.Println("ISSUE BLOCKS SO THAT NODES CAN WARP SYNC UP TO SLOT 19") - ts.IssueBlocksAtSlots("", []iotago.SlotIndex{21, 22}, 2, "block0", ts.Nodes("node0-restarted"), true, false) ts.AssertEqualStoredCommitmentAtIndex(20, ts.Nodes()...) From 1e8e9ddac14752b5882a73722e0d31efbaba4089 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:51:48 +0100 Subject: [PATCH 29/55] Refactor: reverted more changes --- pkg/protocol/chain.go | 10 ++++++++++ pkg/protocol/commitment.go | 2 +- pkg/testsuite/mock/node.go | 16 ++++++++-------- pkg/testsuite/storage_settings.go | 15 +++++++++++++-- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index aeaa16c02..90378abe0 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -157,6 +157,8 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (dispatched bool) func (c *Chain) Commitment(slot iotago.SlotIndex) (commitment *Commitment, exists bool) { for currentChain := c; currentChain != nil; { switch forkingPoint := currentChain.ForkingPoint.Get(); { + case forkingPoint == nil: + return nil, false case forkingPoint.Slot() == slot: return forkingPoint, true case slot > forkingPoint.Slot(): @@ -335,6 +337,14 @@ func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable }, latestSeenSlot)) } +func warpSyncThreshold(latestSeenSlot iotago.SlotIndex, minCommittableAge iotago.SlotIndex) iotago.SlotIndex { + if minCommittableAge < latestSeenSlot { + return latestSeenSlot - minCommittableAge + 1 + } + + return 0 +} + // addCommitment adds the given commitment to this chain. func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { c.commitments.Set(newCommitment.Slot(), newCommitment) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index d739bc43f..8a04de88a 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -158,7 +158,7 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( // mark commitments that are marked as root as verified c.IsCommitted.InheritFrom(c.IsRoot), - c.IsAboveLatestVerifiedCommitment.InheritFrom(c.IsRoot), + c.ReplayDroppedBlocks.InheritFrom(c.IsRoot), // mark commitments that are marked as verified as attested, fully booked and committable c.IsAttested.InheritFrom(c.IsCommitted), diff --git a/pkg/testsuite/mock/node.go b/pkg/testsuite/mock/node.go index b9031ee2e..32394616a 100644 --- a/pkg/testsuite/mock/node.go +++ b/pkg/testsuite/mock/node.go @@ -392,19 +392,19 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi }) events.SpendDAG.SpenderCreated.Hook(func(conflictID iotago.TransactionID) { - instance.LogTrace("ConflictDAG.SpendCreated", "conflictID", conflictID) + instance.LogTrace("SpendDAG.SpenderCreated", "conflictID", conflictID) }) events.SpendDAG.SpenderEvicted.Hook(func(conflictID iotago.TransactionID) { - instance.LogTrace("ConflictDAG.SpendEvicted", "conflictID", conflictID) + instance.LogTrace("SpendDAG.SpenderEvicted", "conflictID", conflictID) }) events.SpendDAG.SpenderRejected.Hook(func(conflictID iotago.TransactionID) { - instance.LogTrace("ConflictDAG.SpendRejected", "conflictID", conflictID) + instance.LogTrace("SpendDAG.SpenderRejected", "conflictID", conflictID) }) events.SpendDAG.SpenderAccepted.Hook(func(conflictID iotago.TransactionID) { - instance.LogTrace("ConflictDAG.SpendAccepted", "conflictID", conflictID) + instance.LogTrace("SpendDAG.SpenderAccepted", "conflictID", conflictID) }) instance.Ledger.MemPool().OnSignedTransactionAttached( @@ -525,10 +525,6 @@ func (n *Node) FilteredBlocks() []*postsolidfilter.BlockFilteredEvent { return n.filteredBlockEvents } -func (n *Node) MainEngineSwitchedCount() int { - return int(n.mainEngineSwitchedCount.Load()) -} - func (n *Node) TransactionFailure(txID iotago.SignedTransactionID) (InvalidSignedTransactionEvent, bool) { n.mutex.RLock() defer n.mutex.RUnlock() @@ -537,6 +533,10 @@ func (n *Node) TransactionFailure(txID iotago.SignedTransactionID) (InvalidSigne return event, exists } +func (n *Node) MainEngineSwitchedCount() int { + return int(n.mainEngineSwitchedCount.Load()) +} + func (n *Node) AttachedBlocks() []*blocks.Block { n.mutex.RLock() defer n.mutex.RUnlock() diff --git a/pkg/testsuite/storage_settings.go b/pkg/testsuite/storage_settings.go index 956797be7..04409b25b 100644 --- a/pkg/testsuite/storage_settings.go +++ b/pkg/testsuite/storage_settings.go @@ -68,8 +68,15 @@ func (t *TestSuite) AssertCommitmentSlotIndexExists(slot iotago.SlotIndex, nodes return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found", node.Name, slot) } + // Make sure the main chain exists + mainChain := node.Protocol.Chains.Main.Get() + if mainChain == nil { + return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: main chain not found when checking for commitment at index %v", node.Name, slot) + } + // Make sure the commitment is also available in the ChainManager. - if node.Protocol.Chains.Main.Get().LatestCommitment.Get().ID().Slot() < slot { + latestCommitment := mainChain.LatestCommitment.Get() + if latestCommitment == nil || latestCommitment.ID().Slot() < slot { return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found in ChainManager", node.Name, slot) } @@ -126,8 +133,12 @@ func (t *TestSuite) AssertChainID(expectedChainID iotago.CommitmentID, nodes ... for _, node := range nodes { t.Eventually(func() error { - actualChainID := node.Protocol.Chains.Main.Get().ForkingPoint.Get().ID() + mainChain := node.Protocol.Chains.Main.Get() + if mainChain == nil { + return ierrors.Errorf("AssertChainID: %s: main chain not found", node.Name) + } + actualChainID := mainChain.ForkingPoint.Get().ID() if expectedChainID != actualChainID { fmt.Println(expectedChainID, actualChainID) From 52694bef06d4504eca82f04c65c9f48acf308509 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Fri, 1 Dec 2023 15:07:31 +0100 Subject: [PATCH 30/55] Fix: fixed bugs and refactored code --- pkg/protocol/chain.go | 162 ++++++++++++----------------- pkg/protocol/commitment.go | 21 ++-- pkg/protocol/protocol_warp_sync.go | 13 +-- 3 files changed, 86 insertions(+), 110 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 90378abe0..c7d4eadc6 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -45,8 +45,8 @@ type Chain struct { // latest verified commitment. VerifiedWeight reactive.Variable[uint64] - // WarpSyncMode contains a flag that indicates whether this chain is in warp sync mode. - WarpSyncMode reactive.Variable[bool] + // WarpSyncModeEnabled contains a flag that indicates whether this chain is in warp sync mode. + WarpSyncModeEnabled reactive.Variable[bool] // WarpSyncThreshold contains the slot at which the chain will exit warp sync mode which is derived from the latest // network slot minus the max committable age. @@ -92,7 +92,7 @@ func newChain(chains *Chains) *Chain { ClaimedWeight: reactive.NewVariable[uint64](), AttestedWeight: reactive.NewVariable[uint64](), VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncMode: reactive.NewVariable[bool]().Init(true), + WarpSyncModeEnabled: reactive.NewVariable[bool]().Init(true), WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), RequestAttestations: reactive.NewVariable[bool](), @@ -188,7 +188,7 @@ func (c *Chain) initLogger() (shutdown func()) { c.Logger, shutdown = c.chains.NewEntityLogger("") return lo.Batch( - c.WarpSyncMode.LogUpdates(c, log.LevelTrace, "WarpSyncMode"), + c.WarpSyncModeEnabled.LogUpdates(c, log.LevelTrace, "WarpSyncModeEnabled"), c.WarpSyncThreshold.LogUpdates(c, log.LevelTrace, "WarpSyncThreshold"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), @@ -209,10 +209,21 @@ func (c *Chain) initLogger() (shutdown func()) { // initDerivedProperties initializes the behavior of this chain by setting up the relations between its properties. func (c *Chain) initDerivedProperties() (shutdown func()) { return lo.Batch( - c.deriveClaimedWeight(), - c.deriveVerifiedWeight(), - c.deriveLatestAttestedWeight(), - c.deriveWarpSyncMode(), + c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { + return latestCommitment.cumulativeWeight() + }, c.LatestCommitment)), + + c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { + return latestProducedCommitment.cumulativeWeight() + }, c.LatestProducedCommitment)), + + c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(warpSyncMode, latestProducedCommitment, warpSyncThreshold, outOfSyncThreshold) + }, c.LatestProducedCommitment, c.WarpSyncThreshold, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), + + c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { + return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) + }), c.ForkingPoint.WithValue(func(forkingPoint *Commitment) (shutdown func()) { return c.deriveParentChain(forkingPoint) @@ -224,65 +235,18 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { c.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { return lo.Batch( - c.deriveWarpSyncThreshold(c.chains.LatestSeenSlot, engineInstance), - c.deriveOutOfSyncThreshold(c.chains.LatestSeenSlot, engineInstance), + c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + return warpSyncThreshold(engineInstance, latestSeenSlot) + }, c.chains.LatestSeenSlot)), + + c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + return outOfSyncThreshold(engineInstance, latestSeenSlot) + }, c.chains.LatestSeenSlot)), ) }), ) } -// deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. -func (c *Chain) deriveWarpSyncMode() func() { - return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during - // startup) - if latestProducedCommitment == nil { - return warpSyncMode - } - - // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold - if warpSyncMode { - return latestProducedCommitment.ID().Slot() < warpSyncThreshold - } - - // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestProducedCommitment.ID().Slot() < outOfSyncThreshold - }, c.LatestProducedCommitment, c.WarpSyncThreshold, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) -} - -// deriveClaimedWeight defines how a chain determines its claimed weight (by setting the cumulative weight of the -// latest commitment). -func (c *Chain) deriveClaimedWeight() (shutdown func()) { - return c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { - if latestCommitment == nil { - return 0 - } - - return latestCommitment.CumulativeWeight() - }, c.LatestCommitment)) -} - -// deriveLatestAttestedWeight defines how a chain determines its attested weight (by inheriting the cumulative attested -// weight of the latest attested commitment). It uses inheritance instead of simply setting the value as the cumulative -// attested weight can change over time depending on the attestations that are received. -func (c *Chain) deriveLatestAttestedWeight() func() { - return c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { - return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) - }) -} - -// deriveVerifiedWeight defines how a chain determines its verified weight (by setting the cumulative weight of the -// latest produced commitment). -func (c *Chain) deriveVerifiedWeight() func() { - return c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { - if latestProducedCommitment == nil { - return 0 - } - - return latestProducedCommitment.CumulativeWeight() - }, c.LatestProducedCommitment)) -} - // deriveChildChains defines how a chain determines its ChildChains (by adding each child to the set). func (c *Chain) deriveChildChains(child *Chain) func() { c.ChildChains.Add(child) @@ -312,39 +276,6 @@ func (c *Chain) deriveParentChain(forkingPoint *Commitment) (shutdown func()) { return nil } -// deriveOutOfSyncThreshold defines how a chain determines its "out of sync" threshold (the latest seen slot minus 2 -// times the max committable age or 0 if this would cause an overflow to the negative numbers). -func (c *Chain) deriveOutOfSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { - return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { - return latestSeenSlot - outOfSyncOffset - } - - return 0 - }, latestSeenSlot)) -} - -// deriveWarpSyncThreshold defines how a chain determines its warp sync threshold (the latest seen slot minus the max -// committable age or 0 if this would cause an overflow to the negative numbers). -func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { - return c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() - if warpSyncOffset < latestSeenSlot { - return latestSeenSlot - warpSyncOffset + 1 - } - - return 0 - }, latestSeenSlot)) -} - -func warpSyncThreshold(latestSeenSlot iotago.SlotIndex, minCommittableAge iotago.SlotIndex) iotago.SlotIndex { - if minCommittableAge < latestSeenSlot { - return latestSeenSlot - minCommittableAge + 1 - } - - return 0 -} - // addCommitment adds the given commitment to this chain. func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { c.commitments.Set(newCommitment.Slot(), newCommitment) @@ -373,7 +304,7 @@ func (c *Chain) dispatchBlockToSpawnedEngine(block *model.Block, src peer.ID) (d } // perform additional checks if we are in warp sync mode (only let blocks pass that we requested) - if c.WarpSyncMode.Get() { + if c.WarpSyncModeEnabled.Get() { // abort if the target commitment does not exist targetCommitment, targetCommitmentExists := c.Commitment(targetSlot) if !targetCommitmentExists { @@ -410,3 +341,42 @@ func (c *Chain) verifiedWeight() reactive.Variable[uint64] { func (c *Chain) attestedWeight() reactive.Variable[uint64] { return c.AttestedWeight } + +// warpSyncThreshold returns the slot index at which the warp sync should stop. +func warpSyncThreshold(engineInstance *engine.Engine, latestSlot iotago.SlotIndex) iotago.SlotIndex { + // TODO: explain why we do - 1 here + warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() - 1 + + // prevent overflow to negative numbers + if warpSyncOffset >= latestSlot { + return 0 + } + + return latestSlot - warpSyncOffset +} + +// outOfSyncThreshold returns the slot index at which the node is considered out of sync. +func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { + return latestSeenSlot - outOfSyncOffset + } + + return 0 +} + +// warpSyncModeEnabled determines whether warp sync mode should be enabled or not. +func warpSyncModeEnabled(enabled bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during + // startup) + if latestProducedCommitment == nil { + return enabled + } + + // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold + if enabled { + return latestProducedCommitment.ID().Slot() < warpSyncThreshold + } + + // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold + return latestProducedCommitment.ID().Slot() < outOfSyncThreshold +} diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 8a04de88a..920799b29 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -275,17 +275,17 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting // warp sync, and we are the directly above the latest verified Commitment. func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { - return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { - return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked - }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) + return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, parentIsFullyBooked bool, isFullyBooked bool) bool { + return engineInstance != nil && warpSyncModeEnabled && parentIsFullyBooked && !isFullyBooked + }, chain.Engine, chain.WarpSyncModeEnabled, parent.IsFullyBooked, c.IsFullyBooked)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an // engine, is no longer requesting warp sync, and we are above the latest verified Commitment. func (c *Commitment) deriveReplayDroppedBlocks(chain *Chain) func() { - return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncing bool, isAboveLatestVerifiedCommitment bool) bool { - return engineInstance != nil && !warpSyncing && isAboveLatestVerifiedCommitment - }, chain.Engine, chain.WarpSyncMode, c.IsAboveLatestVerifiedCommitment)) + return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, isAboveLatestVerifiedCommitment bool) bool { + return engineInstance != nil && !warpSyncModeEnabled && isAboveLatestVerifiedCommitment + }, chain.Engine, chain.WarpSyncModeEnabled, c.IsAboveLatestVerifiedCommitment)) } // forceChain forces the Chain of this Commitment to the given Chain by promoting it to the main child of its parent if @@ -297,3 +297,12 @@ func (c *Commitment) forceChain(targetChain *Chain) { } } } + +// cumulativeWeight returns the cumulative weight of this Commitment while gracefully handling nil receivers. +func (c *Commitment) cumulativeWeight() uint64 { + if c == nil { + return 0 + } + + return c.CumulativeWeight() +} diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index fb6a3c111..1e898c90c 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -47,7 +47,7 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { protocol.Constructed.OnTrigger(func() { protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { - return chain.WarpSyncMode.OnUpdate(func(_ bool, warpSyncMode bool) { + return chain.WarpSyncModeEnabled.OnUpdate(func(_ bool, warpSyncMode bool) { if warpSyncMode { engine.Workers.WaitChildren() engine.Reset() @@ -110,7 +110,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return @@ -161,7 +161,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo targetEngine.Workers.WaitChildren() - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return blocksToWarpSync @@ -177,7 +177,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 2. Mark all blocks as accepted // 3. Force commitment of the slot forceCommitmentFunc := func() { - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { return } @@ -248,10 +248,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { - // Let's assume that MCA is 5: when we want to book 15, we expect to have the commitment of 10 to load - // accounts from it, hence why we make committable the slot at - MCA + 1 with respect of the current slot. - minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() - if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge + 1); exists { + if committableCommitment, exists := chain.Commitment(warpSyncThreshold(targetEngine, commitmentID.Slot())); exists { committableCommitment.IsCommittable.Set(true) } }) From 8ad929b0b05b429c22c10b04e337af39062ac334 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 01:57:17 +0100 Subject: [PATCH 31/55] Refactor: refactored warpsync to be able to handle loss of acceptance --- pkg/protocol/chain.go | 78 ++++++++++++------------------ pkg/protocol/commitment.go | 3 +- pkg/protocol/protocol_warp_sync.go | 16 ++++-- 3 files changed, 43 insertions(+), 54 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index c7d4eadc6..b03b7a2d0 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -29,6 +29,9 @@ type Chain struct { // LatestAttestedCommitment contains the latest commitment of this chain for which attestations were received. LatestAttestedCommitment reactive.Variable[*Commitment] + // LatestFullyBookedCommitment contains the latest commitment of this chain for which all blocks were booked. + LatestFullyBookedCommitment reactive.Variable[*Commitment] + // LatestProducedCommitment contains the latest commitment of this chain that we produced ourselves by booking the // corresponding blocks in the Engine. LatestProducedCommitment reactive.Variable[*Commitment] @@ -48,10 +51,6 @@ type Chain struct { // WarpSyncModeEnabled contains a flag that indicates whether this chain is in warp sync mode. WarpSyncModeEnabled reactive.Variable[bool] - // WarpSyncThreshold contains the slot at which the chain will exit warp sync mode which is derived from the latest - // network slot minus the max committable age. - WarpSyncThreshold reactive.Variable[iotago.SlotIndex] - // OutOfSyncThreshold contains the slot at which the chain will consider itself to be out of sync and switch to warp // sync mode. It is derived from the latest network slot minus two times the max committable age. OutOfSyncThreshold reactive.Variable[iotago.SlotIndex] @@ -83,22 +82,22 @@ type Chain struct { // newChain creates a new chain instance. func newChain(chains *Chains) *Chain { c := &Chain{ - ForkingPoint: reactive.NewVariable[*Commitment](), - ParentChain: reactive.NewVariable[*Chain](), - ChildChains: reactive.NewSet[*Chain](), - LatestCommitment: reactive.NewVariable[*Commitment](), - LatestAttestedCommitment: reactive.NewVariable[*Commitment](), - LatestProducedCommitment: reactive.NewVariable[*Commitment](), - ClaimedWeight: reactive.NewVariable[uint64](), - AttestedWeight: reactive.NewVariable[uint64](), - VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncModeEnabled: reactive.NewVariable[bool]().Init(true), - WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), - OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), - RequestAttestations: reactive.NewVariable[bool](), - StartEngine: reactive.NewVariable[bool](), - Engine: reactive.NewVariable[*engine.Engine](), - IsEvicted: reactive.NewEvent(), + ForkingPoint: reactive.NewVariable[*Commitment](), + ParentChain: reactive.NewVariable[*Chain](), + ChildChains: reactive.NewSet[*Chain](), + LatestCommitment: reactive.NewVariable[*Commitment](), + LatestAttestedCommitment: reactive.NewVariable[*Commitment](), + LatestFullyBookedCommitment: reactive.NewVariable[*Commitment](), + LatestProducedCommitment: reactive.NewVariable[*Commitment](), + ClaimedWeight: reactive.NewVariable[uint64](), + AttestedWeight: reactive.NewVariable[uint64](), + VerifiedWeight: reactive.NewVariable[uint64](), + WarpSyncModeEnabled: reactive.NewVariable[bool]().Init(true), + OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), + RequestAttestations: reactive.NewVariable[bool](), + StartEngine: reactive.NewVariable[bool](), + Engine: reactive.NewVariable[*engine.Engine](), + IsEvicted: reactive.NewEvent(), chains: chains, commitments: shrinkingmap.New[iotago.SlotIndex, *Commitment](), @@ -189,13 +188,14 @@ func (c *Chain) initLogger() (shutdown func()) { return lo.Batch( c.WarpSyncModeEnabled.LogUpdates(c, log.LevelTrace, "WarpSyncModeEnabled"), - c.WarpSyncThreshold.LogUpdates(c, log.LevelTrace, "WarpSyncThreshold"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), c.ClaimedWeight.LogUpdates(c, log.LevelTrace, "ClaimedWeight"), c.AttestedWeight.LogUpdates(c, log.LevelTrace, "AttestedWeight"), c.VerifiedWeight.LogUpdates(c, log.LevelTrace, "VerifiedWeight"), c.LatestCommitment.LogUpdates(c, log.LevelTrace, "LatestCommitment", (*Commitment).LogName), + c.LatestAttestedCommitment.LogUpdates(c, log.LevelTrace, "LatestAttestedCommitment", (*Commitment).LogName), + c.LatestFullyBookedCommitment.LogUpdates(c, log.LevelTrace, "LatestFullyBookedCommitment", (*Commitment).LogName), c.LatestProducedCommitment.LogUpdates(c, log.LevelDebug, "LatestProducedCommitment", (*Commitment).LogName), c.RequestAttestations.LogUpdates(c, log.LevelTrace, "RequestAttestations"), c.StartEngine.LogUpdates(c, log.LevelDebug, "StartEngine"), @@ -217,9 +217,9 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { return latestProducedCommitment.cumulativeWeight() }, c.LatestProducedCommitment)), - c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - return warpSyncModeEnabled(warpSyncMode, latestProducedCommitment, warpSyncThreshold, outOfSyncThreshold) - }, c.LatestProducedCommitment, c.WarpSyncThreshold, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), + c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(warpSyncMode, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) + }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) @@ -234,15 +234,9 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { }), c.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return lo.Batch( - c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - return warpSyncThreshold(engineInstance, latestSeenSlot) - }, c.chains.LatestSeenSlot)), - - c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - return outOfSyncThreshold(engineInstance, latestSeenSlot) - }, c.chains.LatestSeenSlot)), - ) + return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + return outOfSyncThreshold(engineInstance, latestSeenSlot) + }, c.chains.LatestSeenSlot)) }), ) } @@ -284,6 +278,7 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), + newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedCommitment.Set(newCommitment) }), newCommitment.IsCommitted.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), ) } @@ -342,19 +337,6 @@ func (c *Chain) attestedWeight() reactive.Variable[uint64] { return c.AttestedWeight } -// warpSyncThreshold returns the slot index at which the warp sync should stop. -func warpSyncThreshold(engineInstance *engine.Engine, latestSlot iotago.SlotIndex) iotago.SlotIndex { - // TODO: explain why we do - 1 here - warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MinCommittableAge() - 1 - - // prevent overflow to negative numbers - if warpSyncOffset >= latestSlot { - return 0 - } - - return latestSlot - warpSyncOffset -} - // outOfSyncThreshold returns the slot index at which the node is considered out of sync. func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { @@ -365,7 +347,7 @@ func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.Slo } // warpSyncModeEnabled determines whether warp sync mode should be enabled or not. -func warpSyncModeEnabled(enabled bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { +func warpSyncModeEnabled(enabled bool, latestProducedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during // startup) if latestProducedCommitment == nil { @@ -374,7 +356,7 @@ func warpSyncModeEnabled(enabled bool, latestProducedCommitment *Commitment, war // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold if enabled { - return latestProducedCommitment.ID().Slot() < warpSyncThreshold + return latestProducedCommitment.ID().Slot() < latestSeenSlot } // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 920799b29..1b956deba 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -160,10 +160,9 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { c.IsCommitted.InheritFrom(c.IsRoot), c.ReplayDroppedBlocks.InheritFrom(c.IsRoot), - // mark commitments that are marked as verified as attested, fully booked and committable + // mark commitments that are marked as verified as attested and fully booked c.IsAttested.InheritFrom(c.IsCommitted), c.IsFullyBooked.InheritFrom(c.IsCommitted), - c.IsCommittable.InheritFrom(c.IsCommitted), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 1e898c90c..0b23ea6e3 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -247,17 +247,25 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo }) } + // Once all blocks are fully booked we can mark the commitment that is minCommittableAge older as this + // commitment to be committable. commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { - if committableCommitment, exists := chain.Commitment(warpSyncThreshold(targetEngine, commitmentID.Slot())); exists { - committableCommitment.IsCommittable.Set(true) + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - targetEngine.LatestAPI().ProtocolParameters().MinCommittableAge()); exists { + w.workerPool.Submit(func() { + committableCommitment.IsCommittable.Set(true) + }) } }) - commitment.IsCommittable.OnUpdateOnce(func(_ bool, _ bool) { - w.workerPool.Submit(forceCommitmentFunc) + // force commit one by one and wait for the parent to be committed before we can commit the next one + commitment.Parent.WithNonEmptyValue(func(parent *Commitment) (teardown func()) { + return parent.IsCommitted.WithNonEmptyValue(func(_ bool) (teardown func()) { + return commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + }) }) if totalBlocks == 0 { + commitment.IsCommittable.Set(true) commitment.IsFullyBooked.Set(true) return blocksToWarpSync From 89f0213796298dc9c54b9c1ef3f50e96edbc0188 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 02:14:32 +0100 Subject: [PATCH 32/55] Refactor: fixed typo + race condition --- pkg/protocol/chain.go | 12 ++++++------ pkg/protocol/commitment_verifier.go | 9 +++++++++ pkg/protocol/protocol_warp_sync.go | 4 ++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index b03b7a2d0..86b1d1d30 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -217,8 +217,8 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { return latestProducedCommitment.cumulativeWeight() }, c.LatestProducedCommitment)), - c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - return warpSyncModeEnabled(warpSyncMode, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) + c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(enabled, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { @@ -347,18 +347,18 @@ func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.Slo } // warpSyncModeEnabled determines whether warp sync mode should be enabled or not. -func warpSyncModeEnabled(enabled bool, latestProducedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { +func warpSyncModeEnabled(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during // startup) - if latestProducedCommitment == nil { + if latestFullyBookedCommitment == nil { return enabled } // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold if enabled { - return latestProducedCommitment.ID().Slot() < latestSeenSlot + return latestFullyBookedCommitment.ID().Slot() < latestSeenSlot } // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestProducedCommitment.ID().Slot() < outOfSyncThreshold + return latestFullyBookedCommitment.ID().Slot() < outOfSyncThreshold } diff --git a/pkg/protocol/commitment_verifier.go b/pkg/protocol/commitment_verifier.go index ab600fbc9..3ee2b4cc8 100644 --- a/pkg/protocol/commitment_verifier.go +++ b/pkg/protocol/commitment_verifier.go @@ -5,6 +5,7 @@ import ( "github.com/iotaledger/hive.go/ds" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/kvstore/mapdb" + "github.com/iotaledger/hive.go/runtime/syncutils" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/accounts" @@ -22,6 +23,9 @@ type CommitmentVerifier struct { // validatorAccountsData is the accounts data of the validators for the current epoch as known at lastCommonSlotBeforeFork. // Initially, it is set to the accounts data of the validators for the epoch of the last common commitment before the fork. validatorAccountsData map[iotago.AccountID]*accounts.AccountData + + // mutex is used to synchronize access to validatorAccountsData and epoch. + mutex syncutils.RWMutex } func newCommitmentVerifier(mainEngine *engine.Engine, lastCommonCommitmentBeforeFork *model.Commitment) (*CommitmentVerifier, error) { @@ -76,6 +80,7 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio // This is necessary because the committee might have rotated at the epoch boundary and different validators might be part of it. // In case anything goes wrong we keep using previously known accounts data (initially set to the accounts data // of the validators for the epoch of the last common commitment before the fork). + c.mutex.Lock() apiForSlot := c.engine.APIForSlot(commitment.Slot()) commitmentEpoch := apiForSlot.TimeProvider().EpochFromSlot(commitment.Slot()) if commitmentEpoch > c.epoch { @@ -92,6 +97,7 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio } } } + c.mutex.Unlock() // 3. Verify attestations. blockIDs, seatCount, err := c.verifyAttestations(attestations) @@ -107,6 +113,9 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio } func (c *CommitmentVerifier) verifyAttestations(attestations []*iotago.Attestation) (iotago.BlockIDs, uint64, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + visitedIdentities := ds.NewSet[iotago.AccountID]() var blockIDs iotago.BlockIDs var seatCount uint64 diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 0b23ea6e3..374aaa3c6 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -47,8 +47,8 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { protocol.Constructed.OnTrigger(func() { protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { - return chain.WarpSyncModeEnabled.OnUpdate(func(_ bool, warpSyncMode bool) { - if warpSyncMode { + return chain.WarpSyncModeEnabled.OnUpdate(func(_ bool, warpSyncModeEnabled bool) { + if warpSyncModeEnabled { engine.Workers.WaitChildren() engine.Reset() } From af324c3775aef5742683440391cd168bd76e3d62 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 02:29:47 +0100 Subject: [PATCH 33/55] Fix bugs in WarpSync logic (#577) fixed bugs in warpsync logic and optimized logging during debug --- pkg/core/buffer/unsolid_commitment_buffer.go | 2 +- pkg/protocol/chain.go | 182 ++++++++----------- pkg/protocol/commitment.go | 25 ++- pkg/protocol/commitment_verifier.go | 9 + pkg/protocol/protocol_blocks.go | 21 +-- pkg/protocol/protocol_warp_sync.go | 29 +-- pkg/tests/loss_of_acceptance_test.go | 2 - pkg/testsuite/mock/node.go | 111 +++++------ pkg/testsuite/storage_settings.go | 15 +- 9 files changed, 191 insertions(+), 205 deletions(-) diff --git a/pkg/core/buffer/unsolid_commitment_buffer.go b/pkg/core/buffer/unsolid_commitment_buffer.go index 27df0dd51..a6417a4d7 100644 --- a/pkg/core/buffer/unsolid_commitment_buffer.go +++ b/pkg/core/buffer/unsolid_commitment_buffer.go @@ -51,7 +51,7 @@ func (u *UnsolidCommitmentBuffer[V]) Add(commitmentID iotago.CommitmentID, value u.mutex.RLock() defer u.mutex.RUnlock() - if commitmentID.Slot() <= u.lastEvictedSlot { + if u.lastEvictedSlot != 0 && commitmentID.Slot() <= u.lastEvictedSlot { return false } diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 8c26912dd..86b1d1d30 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -29,6 +29,9 @@ type Chain struct { // LatestAttestedCommitment contains the latest commitment of this chain for which attestations were received. LatestAttestedCommitment reactive.Variable[*Commitment] + // LatestFullyBookedCommitment contains the latest commitment of this chain for which all blocks were booked. + LatestFullyBookedCommitment reactive.Variable[*Commitment] + // LatestProducedCommitment contains the latest commitment of this chain that we produced ourselves by booking the // corresponding blocks in the Engine. LatestProducedCommitment reactive.Variable[*Commitment] @@ -45,12 +48,8 @@ type Chain struct { // latest verified commitment. VerifiedWeight reactive.Variable[uint64] - // WarpSyncMode contains a flag that indicates whether this chain is in warp sync mode. - WarpSyncMode reactive.Variable[bool] - - // WarpSyncThreshold contains the slot at which the chain will exit warp sync mode which is derived from the latest - // network slot minus the max committable age. - WarpSyncThreshold reactive.Variable[iotago.SlotIndex] + // WarpSyncModeEnabled contains a flag that indicates whether this chain is in warp sync mode. + WarpSyncModeEnabled reactive.Variable[bool] // OutOfSyncThreshold contains the slot at which the chain will consider itself to be out of sync and switch to warp // sync mode. It is derived from the latest network slot minus two times the max committable age. @@ -83,22 +82,22 @@ type Chain struct { // newChain creates a new chain instance. func newChain(chains *Chains) *Chain { c := &Chain{ - ForkingPoint: reactive.NewVariable[*Commitment](), - ParentChain: reactive.NewVariable[*Chain](), - ChildChains: reactive.NewSet[*Chain](), - LatestCommitment: reactive.NewVariable[*Commitment](), - LatestAttestedCommitment: reactive.NewVariable[*Commitment](), - LatestProducedCommitment: reactive.NewVariable[*Commitment](), - ClaimedWeight: reactive.NewVariable[uint64](), - AttestedWeight: reactive.NewVariable[uint64](), - VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncMode: reactive.NewVariable[bool](), - WarpSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), - OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), - RequestAttestations: reactive.NewVariable[bool](), - StartEngine: reactive.NewVariable[bool](), - Engine: reactive.NewVariable[*engine.Engine](), - IsEvicted: reactive.NewEvent(), + ForkingPoint: reactive.NewVariable[*Commitment](), + ParentChain: reactive.NewVariable[*Chain](), + ChildChains: reactive.NewSet[*Chain](), + LatestCommitment: reactive.NewVariable[*Commitment](), + LatestAttestedCommitment: reactive.NewVariable[*Commitment](), + LatestFullyBookedCommitment: reactive.NewVariable[*Commitment](), + LatestProducedCommitment: reactive.NewVariable[*Commitment](), + ClaimedWeight: reactive.NewVariable[uint64](), + AttestedWeight: reactive.NewVariable[uint64](), + VerifiedWeight: reactive.NewVariable[uint64](), + WarpSyncModeEnabled: reactive.NewVariable[bool]().Init(true), + OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), + RequestAttestations: reactive.NewVariable[bool](), + StartEngine: reactive.NewVariable[bool](), + Engine: reactive.NewVariable[*engine.Engine](), + IsEvicted: reactive.NewEvent(), chains: chains, commitments: shrinkingmap.New[iotago.SlotIndex, *Commitment](), @@ -157,6 +156,8 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (dispatched bool) func (c *Chain) Commitment(slot iotago.SlotIndex) (commitment *Commitment, exists bool) { for currentChain := c; currentChain != nil; { switch forkingPoint := currentChain.ForkingPoint.Get(); { + case forkingPoint == nil: + return nil, false case forkingPoint.Slot() == slot: return forkingPoint, true case slot > forkingPoint.Slot(): @@ -186,14 +187,15 @@ func (c *Chain) initLogger() (shutdown func()) { c.Logger, shutdown = c.chains.NewEntityLogger("") return lo.Batch( - c.WarpSyncMode.LogUpdates(c, log.LevelTrace, "WarpSyncMode"), - c.WarpSyncThreshold.LogUpdates(c, log.LevelTrace, "WarpSyncThreshold"), + c.WarpSyncModeEnabled.LogUpdates(c, log.LevelTrace, "WarpSyncModeEnabled"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), c.ClaimedWeight.LogUpdates(c, log.LevelTrace, "ClaimedWeight"), c.AttestedWeight.LogUpdates(c, log.LevelTrace, "AttestedWeight"), c.VerifiedWeight.LogUpdates(c, log.LevelTrace, "VerifiedWeight"), c.LatestCommitment.LogUpdates(c, log.LevelTrace, "LatestCommitment", (*Commitment).LogName), + c.LatestAttestedCommitment.LogUpdates(c, log.LevelTrace, "LatestAttestedCommitment", (*Commitment).LogName), + c.LatestFullyBookedCommitment.LogUpdates(c, log.LevelTrace, "LatestFullyBookedCommitment", (*Commitment).LogName), c.LatestProducedCommitment.LogUpdates(c, log.LevelDebug, "LatestProducedCommitment", (*Commitment).LogName), c.RequestAttestations.LogUpdates(c, log.LevelTrace, "RequestAttestations"), c.StartEngine.LogUpdates(c, log.LevelDebug, "StartEngine"), @@ -207,10 +209,21 @@ func (c *Chain) initLogger() (shutdown func()) { // initDerivedProperties initializes the behavior of this chain by setting up the relations between its properties. func (c *Chain) initDerivedProperties() (shutdown func()) { return lo.Batch( - c.deriveClaimedWeight(), - c.deriveVerifiedWeight(), - c.deriveLatestAttestedWeight(), - c.deriveWarpSyncMode(), + c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { + return latestCommitment.cumulativeWeight() + }, c.LatestCommitment)), + + c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { + return latestProducedCommitment.cumulativeWeight() + }, c.LatestProducedCommitment)), + + c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(enabled, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) + }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), + + c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { + return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) + }), c.ForkingPoint.WithValue(func(forkingPoint *Commitment) (shutdown func()) { return c.deriveParentChain(forkingPoint) @@ -221,66 +234,13 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { }), c.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return lo.Batch( - c.deriveWarpSyncThreshold(c.chains.LatestSeenSlot, engineInstance), - c.deriveOutOfSyncThreshold(c.chains.LatestSeenSlot, engineInstance), - ) + return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + return outOfSyncThreshold(engineInstance, latestSeenSlot) + }, c.chains.LatestSeenSlot)) }), ) } -// deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. -func (c *Chain) deriveWarpSyncMode() func() { - return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestProducedCommitment *Commitment, warpSyncThreshold iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during - // startup) - if latestProducedCommitment == nil { - return warpSyncMode - } - - // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold - if warpSyncMode { - return latestProducedCommitment.ID().Slot() < warpSyncThreshold - } - - // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestProducedCommitment.ID().Slot() < outOfSyncThreshold - }, c.LatestProducedCommitment, c.WarpSyncThreshold, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) -} - -// deriveClaimedWeight defines how a chain determines its claimed weight (by setting the cumulative weight of the -// latest commitment). -func (c *Chain) deriveClaimedWeight() (shutdown func()) { - return c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { - if latestCommitment == nil { - return 0 - } - - return latestCommitment.CumulativeWeight() - }, c.LatestCommitment)) -} - -// deriveLatestAttestedWeight defines how a chain determines its attested weight (by inheriting the cumulative attested -// weight of the latest attested commitment). It uses inheritance instead of simply setting the value as the cumulative -// attested weight can change over time depending on the attestations that are received. -func (c *Chain) deriveLatestAttestedWeight() func() { - return c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { - return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) - }) -} - -// deriveVerifiedWeight defines how a chain determines its verified weight (by setting the cumulative weight of the -// latest produced commitment). -func (c *Chain) deriveVerifiedWeight() func() { - return c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { - if latestProducedCommitment == nil { - return 0 - } - - return latestProducedCommitment.CumulativeWeight() - }, c.LatestProducedCommitment)) -} - // deriveChildChains defines how a chain determines its ChildChains (by adding each child to the set). func (c *Chain) deriveChildChains(child *Chain) func() { c.ChildChains.Add(child) @@ -310,31 +270,6 @@ func (c *Chain) deriveParentChain(forkingPoint *Commitment) (shutdown func()) { return nil } -// deriveOutOfSyncThreshold defines how a chain determines its "out of sync" threshold (the latest seen slot minus 2 -// times the max committable age or 0 if this would cause an overflow to the negative numbers). -func (c *Chain) deriveOutOfSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { - return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { - return latestSeenSlot - outOfSyncOffset - } - - return 0 - }, latestSeenSlot)) -} - -// deriveWarpSyncThreshold defines how a chain determines its warp sync threshold (the latest seen slot minus the max -// committable age or 0 if this would cause an overflow to the negative numbers). -func (c *Chain) deriveWarpSyncThreshold(latestSeenSlot reactive.ReadableVariable[iotago.SlotIndex], engineInstance *engine.Engine) func() { - return c.WarpSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - warpSyncOffset := engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge() - if warpSyncOffset < latestSeenSlot { - return latestSeenSlot - warpSyncOffset - } - - return 0 - }, latestSeenSlot)) -} - // addCommitment adds the given commitment to this chain. func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { c.commitments.Set(newCommitment.Slot(), newCommitment) @@ -343,6 +278,7 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), + newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedCommitment.Set(newCommitment) }), newCommitment.IsCommitted.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), ) } @@ -363,7 +299,7 @@ func (c *Chain) dispatchBlockToSpawnedEngine(block *model.Block, src peer.ID) (d } // perform additional checks if we are in warp sync mode (only let blocks pass that we requested) - if c.WarpSyncMode.Get() { + if c.WarpSyncModeEnabled.Get() { // abort if the target commitment does not exist targetCommitment, targetCommitmentExists := c.Commitment(targetSlot) if !targetCommitmentExists { @@ -400,3 +336,29 @@ func (c *Chain) verifiedWeight() reactive.Variable[uint64] { func (c *Chain) attestedWeight() reactive.Variable[uint64] { return c.AttestedWeight } + +// outOfSyncThreshold returns the slot index at which the node is considered out of sync. +func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { + return latestSeenSlot - outOfSyncOffset + } + + return 0 +} + +// warpSyncModeEnabled determines whether warp sync mode should be enabled or not. +func warpSyncModeEnabled(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during + // startup) + if latestFullyBookedCommitment == nil { + return enabled + } + + // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold + if enabled { + return latestFullyBookedCommitment.ID().Slot() < latestSeenSlot + } + + // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold + return latestFullyBookedCommitment.ID().Slot() < outOfSyncThreshold +} diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index ca06a32ab..1b956deba 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -158,11 +158,11 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( // mark commitments that are marked as root as verified c.IsCommitted.InheritFrom(c.IsRoot), + c.ReplayDroppedBlocks.InheritFrom(c.IsRoot), - // mark commitments that are marked as verified as attested, fully booked and committable + // mark commitments that are marked as verified as attested and fully booked c.IsAttested.InheritFrom(c.IsCommitted), c.IsFullyBooked.InheritFrom(c.IsCommitted), - c.IsCommittable.InheritFrom(c.IsCommitted), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent @@ -274,17 +274,17 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting // warp sync, and we are the directly above the latest verified Commitment. func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { - return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { - return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked - }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) + return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, parentIsFullyBooked bool, isFullyBooked bool) bool { + return engineInstance != nil && warpSyncModeEnabled && parentIsFullyBooked && !isFullyBooked + }, chain.Engine, chain.WarpSyncModeEnabled, parent.IsFullyBooked, c.IsFullyBooked)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an // engine, is no longer requesting warp sync, and we are above the latest verified Commitment. func (c *Commitment) deriveReplayDroppedBlocks(chain *Chain) func() { - return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncing bool, isAboveLatestVerifiedCommitment bool) bool { - return engineInstance != nil && !warpSyncing && isAboveLatestVerifiedCommitment - }, chain.Engine, chain.WarpSyncMode, c.IsAboveLatestVerifiedCommitment)) + return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, isAboveLatestVerifiedCommitment bool) bool { + return engineInstance != nil && !warpSyncModeEnabled && isAboveLatestVerifiedCommitment + }, chain.Engine, chain.WarpSyncModeEnabled, c.IsAboveLatestVerifiedCommitment)) } // forceChain forces the Chain of this Commitment to the given Chain by promoting it to the main child of its parent if @@ -296,3 +296,12 @@ func (c *Commitment) forceChain(targetChain *Chain) { } } } + +// cumulativeWeight returns the cumulative weight of this Commitment while gracefully handling nil receivers. +func (c *Commitment) cumulativeWeight() uint64 { + if c == nil { + return 0 + } + + return c.CumulativeWeight() +} diff --git a/pkg/protocol/commitment_verifier.go b/pkg/protocol/commitment_verifier.go index ab600fbc9..3ee2b4cc8 100644 --- a/pkg/protocol/commitment_verifier.go +++ b/pkg/protocol/commitment_verifier.go @@ -5,6 +5,7 @@ import ( "github.com/iotaledger/hive.go/ds" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/kvstore/mapdb" + "github.com/iotaledger/hive.go/runtime/syncutils" "github.com/iotaledger/iota-core/pkg/model" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/accounts" @@ -22,6 +23,9 @@ type CommitmentVerifier struct { // validatorAccountsData is the accounts data of the validators for the current epoch as known at lastCommonSlotBeforeFork. // Initially, it is set to the accounts data of the validators for the epoch of the last common commitment before the fork. validatorAccountsData map[iotago.AccountID]*accounts.AccountData + + // mutex is used to synchronize access to validatorAccountsData and epoch. + mutex syncutils.RWMutex } func newCommitmentVerifier(mainEngine *engine.Engine, lastCommonCommitmentBeforeFork *model.Commitment) (*CommitmentVerifier, error) { @@ -76,6 +80,7 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio // This is necessary because the committee might have rotated at the epoch boundary and different validators might be part of it. // In case anything goes wrong we keep using previously known accounts data (initially set to the accounts data // of the validators for the epoch of the last common commitment before the fork). + c.mutex.Lock() apiForSlot := c.engine.APIForSlot(commitment.Slot()) commitmentEpoch := apiForSlot.TimeProvider().EpochFromSlot(commitment.Slot()) if commitmentEpoch > c.epoch { @@ -92,6 +97,7 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio } } } + c.mutex.Unlock() // 3. Verify attestations. blockIDs, seatCount, err := c.verifyAttestations(attestations) @@ -107,6 +113,9 @@ func (c *CommitmentVerifier) verifyCommitment(commitment *Commitment, attestatio } func (c *CommitmentVerifier) verifyAttestations(attestations []*iotago.Attestation) (iotago.BlockIDs, uint64, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + visitedIdentities := ds.NewSet[iotago.AccountID]() var blockIDs iotago.BlockIDs var seatCount uint64 diff --git a/pkg/protocol/protocol_blocks.go b/pkg/protocol/protocol_blocks.go index b18983f3b..3afd74807 100644 --- a/pkg/protocol/protocol_blocks.go +++ b/pkg/protocol/protocol_blocks.go @@ -53,21 +53,12 @@ func newBlocksProtocol(protocol *Protocol) *BlocksProtocol { }) }) - protocol.Chains.WithElements(func(chain *Chain) func() { - return chain.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return engineInstance.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook - }) - }) - - protocol.Chains.Main.Get().Engine.OnUpdateWithContext(func(_ *engine.Engine, engine *engine.Engine, unsubscribeOnEngineChange func(subscriptionFactory func() (unsubscribe func()))) { - if engine != nil { - unsubscribeOnEngineChange(func() (unsubscribe func()) { - return lo.Batch( - engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, - ) - }) - } + protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { + return lo.Batch( + engine.Events.BlockRequester.Tick.Hook(b.SendRequest).Unhook, + engine.Events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + engine.Events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { b.SendResponse(block.ModelBlock()) }).Unhook, + ) }) }) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 844881d89..374aaa3c6 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -47,8 +47,8 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { protocol.Constructed.OnTrigger(func() { protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { - return chain.WarpSyncMode.OnUpdate(func(_ bool, warpSyncMode bool) { - if warpSyncMode { + return chain.WarpSyncModeEnabled.OnUpdate(func(_ bool, warpSyncModeEnabled bool) { + if warpSyncModeEnabled { engine.Workers.WaitChildren() engine.Reset() } @@ -110,7 +110,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return @@ -161,7 +161,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo targetEngine.Workers.WaitChildren() - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return blocksToWarpSync @@ -177,7 +177,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 2. Mark all blocks as accepted // 3. Force commitment of the slot forceCommitmentFunc := func() { - if !chain.WarpSyncMode.Get() { + if !chain.WarpSyncModeEnabled.Get() { return } @@ -247,20 +247,25 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo }) } + // Once all blocks are fully booked we can mark the commitment that is minCommittableAge older as this + // commitment to be committable. commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { - // Let's assume that MCA is 5: when we want to book 15, we expect to have the commitment of 10 to load - // accounts from it, hence why we make committable the slot at - MCA + 1 with respect of the current slot. - minimumCommittableAge := w.protocol.APIForSlot(commitmentID.Slot()).ProtocolParameters().MinCommittableAge() - if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - minimumCommittableAge); exists { - committableCommitment.IsCommittable.Set(true) + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - targetEngine.LatestAPI().ProtocolParameters().MinCommittableAge()); exists { + w.workerPool.Submit(func() { + committableCommitment.IsCommittable.Set(true) + }) } }) - commitment.IsCommittable.OnUpdateOnce(func(_ bool, _ bool) { - w.workerPool.Submit(forceCommitmentFunc) + // force commit one by one and wait for the parent to be committed before we can commit the next one + commitment.Parent.WithNonEmptyValue(func(parent *Commitment) (teardown func()) { + return parent.IsCommitted.WithNonEmptyValue(func(_ bool) (teardown func()) { + return commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + }) }) if totalBlocks == 0 { + commitment.IsCommittable.Set(true) commitment.IsFullyBooked.Set(true) return blocksToWarpSync diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index f47ea4e40..93ec48b57 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -173,8 +173,6 @@ func TestLossOfAcceptanceFromSnapshot(t *testing.T) { { ts.IssueBlocksAtSlots("", []iotago.SlotIndex{21, 22}, 2, "block0", ts.Nodes("node0-restarted"), true, false) - time.Sleep(10 * time.Second) - ts.AssertEqualStoredCommitmentAtIndex(20, ts.Nodes()...) ts.AssertLatestCommitmentSlotIndex(20, ts.Nodes()...) } diff --git a/pkg/testsuite/mock/node.go b/pkg/testsuite/mock/node.go index ab8d94abd..32394616a 100644 --- a/pkg/testsuite/mock/node.go +++ b/pkg/testsuite/mock/node.go @@ -225,11 +225,11 @@ func (n *Node) hookLogging(failOnBlockFiltered bool) { }) } -func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engine.Engine, engineName string) { +func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engine.Engine) { events := instance.Events events.BlockDAG.BlockAttached.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockAttached: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockAttached", "block", block.ID()) n.mutex.Lock() defer n.mutex.Unlock() @@ -237,78 +237,80 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi }) events.BlockDAG.BlockSolid.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockSolid: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockSolid", "block", block.ID()) }) events.BlockDAG.BlockInvalid.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] BlockDAG.BlockInvalid: %s - %s\n", n.Name, engineName, block.ID(), err) + instance.LogTrace("BlockDAG.BlockInvalid", "block", block.ID(), "err", err) }) events.BlockDAG.BlockMissing.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.BlockMissing: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.BlockMissing", "block", block.ID()) }) events.BlockDAG.MissingBlockAttached.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] BlockDAG.MissingBlockAttached: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("BlockDAG.MissingBlockAttached", "block", block.ID()) }) events.SeatManager.BlockProcessed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] SybilProtection.BlockProcessed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("SeatManager.BlockProcessed", "block", block.ID()) }) events.Booker.BlockBooked.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Booker.BlockBooked: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Booker.BlockBooked", "block", block.ID()) }) events.Booker.BlockInvalid.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] Booker.BlockInvalid: %s - %s\n", n.Name, engineName, block.ID(), err.Error()) + instance.LogTrace("Booker.BlockInvalid", "block", block.ID(), "err", err) }) events.Booker.TransactionInvalid.Hook(func(metadata mempool.TransactionMetadata, err error) { - fmt.Printf("%s > [%s] Booker.TransactionInvalid: %s - %s\n", n.Name, engineName, metadata.ID(), err.Error()) + instance.LogTrace("Booker.TransactionInvalid", "tx", metadata.ID(), "err", err) }) events.Scheduler.BlockScheduled.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockScheduled: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockScheduled", "block", block.ID()) }) events.Scheduler.BlockEnqueued.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockEnqueued: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockEnqueued", "block", block.ID()) }) events.Scheduler.BlockSkipped.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Scheduler.BlockSkipped: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("Scheduler.BlockSkipped", "block", block.ID()) }) events.Scheduler.BlockDropped.Hook(func(block *blocks.Block, err error) { - fmt.Printf("%s > [%s] Scheduler.BlockDropped: %s - %s\n", n.Name, engineName, block.ID(), err.Error()) + instance.LogTrace("Scheduler.BlockDropped", "block", block.ID(), "err", err) }) events.Clock.AcceptedTimeUpdated.Hook(func(newTime time.Time) { - fmt.Printf("%s > [%s] Clock.AcceptedTimeUpdated: %s [Slot %d]\n", n.Name, engineName, newTime, instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) + instance.LogTrace("Clock.AcceptedTimeUpdated", "time", newTime, "slot", instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) }) events.Clock.ConfirmedTimeUpdated.Hook(func(newTime time.Time) { - fmt.Printf("%s > [%s] Clock.ConfirmedTimeUpdated: %s [Slot %d]\n", n.Name, engineName, newTime, instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) + instance.LogTrace("Clock.ConfirmedTimeUpdated", "time", newTime, "slot", instance.LatestAPI().TimeProvider().SlotFromTime(newTime)) }) events.PreSolidFilter.BlockPreAllowed.Hook(func(block *model.Block) { - fmt.Printf("%s > [%s] PreSolidFilter.BlockPreAllowed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("PreSolidFilter.BlockPreAllowed", "block", block.ID()) }) events.PreSolidFilter.BlockPreFiltered.Hook(func(event *presolidfilter.BlockPreFilteredEvent) { - fmt.Printf("%s > [%s] PreSolidFilter.BlockPreFiltered: %s - %s\n", n.Name, engineName, event.Block.ID(), event.Reason.Error()) + instance.LogTrace("PreSolidFilter.BlockPreFiltered", "block", event.Block.ID(), "err", event.Reason) + if failOnBlockFiltered { n.Testing.Fatal("no blocks should be prefiltered") } }) events.PostSolidFilter.BlockAllowed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] PostSolidFilter.BlockAllowed: %s\n", n.Name, engineName, block.ID()) + instance.LogTrace("PostSolidFilter.BlockAllowed", "block", block.ID()) }) events.PostSolidFilter.BlockFiltered.Hook(func(event *postsolidfilter.BlockFilteredEvent) { - fmt.Printf("%s > [%s] PostSolidFilter.BlockFiltered: %s - %s\n", n.Name, engineName, event.Block.ID(), event.Reason.Error()) + instance.LogTrace("PostSolidFilter.BlockFiltered", "block", event.Block.ID(), "err", event.Reason) + if failOnBlockFiltered { n.Testing.Fatal("no blocks should be filtered") } @@ -319,11 +321,11 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi }) events.BlockRequester.Tick.Hook(func(blockID iotago.BlockID) { - fmt.Printf("%s > [%s] BlockRequester.Tick: %s\n", n.Name, engineName, blockID) + instance.LogTrace("BlockRequester.Tick", "block", blockID) }) events.BlockProcessed.Hook(func(blockID iotago.BlockID) { - fmt.Printf("%s > [%s] Engine.BlockProcessed: %s\n", n.Name, engineName, blockID) + instance.LogTrace("BlockProcessed", "block", blockID) }) events.Notarization.SlotCommitted.Hook(func(details *notarization.SlotCommittedDetails) { @@ -350,117 +352,116 @@ func (n *Node) attachEngineLogsWithName(failOnBlockFiltered bool, instance *engi require.NoError(n.Testing, err) } - fmt.Printf("%s > [%s] NotarizationManager.SlotCommitted: %s %s Accepted Blocks: %s\n %s\n Attestations: %s\n", n.Name, engineName, details.Commitment.ID(), details.Commitment, acceptedBlocks, roots, attestationBlockIDs) + instance.LogTrace("NotarizationManager.SlotCommitted", "commitment", details.Commitment.ID(), "acceptedBlocks", acceptedBlocks, "roots", roots, "attestations", attestationBlockIDs) }) events.Notarization.LatestCommitmentUpdated.Hook(func(commitment *model.Commitment) { - fmt.Printf("%s > [%s] NotarizationManager.LatestCommitmentUpdated: %s\n", n.Name, engineName, commitment.ID()) + instance.LogTrace("NotarizationManager.LatestCommitmentUpdated", "commitment", commitment.ID()) }) events.BlockGadget.BlockPreAccepted.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockPreAccepted: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockPreAccepted", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockAccepted.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockAccepted: %s @ slot %s committing to %s\n", n.Name, engineName, block.ID(), block.ID().Slot(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockAccepted", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockPreConfirmed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockPreConfirmed: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockPreConfirmed", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.BlockGadget.BlockConfirmed.Hook(func(block *blocks.Block) { - fmt.Printf("%s > [%s] Consensus.BlockGadget.BlockConfirmed: %s %s\n", n.Name, engineName, block.ID(), block.ProtocolBlock().Header.SlotCommitmentID) + instance.LogTrace("BlockGadget.BlockConfirmed", "block", block.ID(), "slotCommitmentID", block.ProtocolBlock().Header.SlotCommitmentID) }) events.SlotGadget.SlotFinalized.Hook(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] Consensus.SlotGadget.SlotFinalized: %s\n", n.Name, engineName, slot) + instance.LogTrace("SlotGadget.SlotFinalized", "slot", slot) }) events.SeatManager.OnlineCommitteeSeatAdded.Hook(func(seat account.SeatIndex, accountID iotago.AccountID) { - fmt.Printf("%s > [%s] SybilProtection.OnlineCommitteeSeatAdded: %d - %s\n", n.Name, engineName, seat, accountID) + instance.LogTrace("SybilProtection.OnlineCommitteeSeatAdded", "seat", seat, "accountID", accountID) }) events.SeatManager.OnlineCommitteeSeatRemoved.Hook(func(seat account.SeatIndex) { - fmt.Printf("%s > [%s] SybilProtection.OnlineCommitteeSeatRemoved: %d\n", n.Name, engineName, seat) + instance.LogTrace("SybilProtection.OnlineCommitteeSeatRemoved", "seat", seat) }) events.SybilProtection.CommitteeSelected.Hook(func(committee *account.Accounts, epoch iotago.EpochIndex) { - fmt.Printf("%s > [%s] SybilProtection.CommitteeSelected: epoch %d - %s\n", n.Name, engineName, epoch, committee.IDs()) + instance.LogTrace("SybilProtection.CommitteeSelected", "epoch", epoch, "committee", committee.IDs()) }) - events.SpendDAG.SpenderCreated.Hook(func(spenderID iotago.TransactionID) { - fmt.Printf("%s > [%s] SpendDAG.SpendCreated: %s\n", n.Name, engineName, spenderID) + events.SpendDAG.SpenderCreated.Hook(func(conflictID iotago.TransactionID) { + instance.LogTrace("SpendDAG.SpenderCreated", "conflictID", conflictID) }) - events.SpendDAG.SpenderEvicted.Hook(func(spenderID iotago.TransactionID) { - fmt.Printf("%s > [%s] SpendDAG.SpendEvicted: %s\n", n.Name, engineName, spenderID) + events.SpendDAG.SpenderEvicted.Hook(func(conflictID iotago.TransactionID) { + instance.LogTrace("SpendDAG.SpenderEvicted", "conflictID", conflictID) }) - events.SpendDAG.SpenderRejected.Hook(func(spenderID iotago.TransactionID) { - fmt.Printf("%s > [%s] SpendDAG.SpendRejected: %s\n", n.Name, engineName, spenderID) + + events.SpendDAG.SpenderRejected.Hook(func(conflictID iotago.TransactionID) { + instance.LogTrace("SpendDAG.SpenderRejected", "conflictID", conflictID) }) - events.SpendDAG.SpenderAccepted.Hook(func(spenderID iotago.TransactionID) { - fmt.Printf("%s > [%s] SpendDAG.SpendAccepted: %s\n", n.Name, engineName, spenderID) + events.SpendDAG.SpenderAccepted.Hook(func(conflictID iotago.TransactionID) { + instance.LogTrace("SpendDAG.SpenderAccepted", "conflictID", conflictID) }) instance.Ledger.MemPool().OnSignedTransactionAttached( func(signedTransactionMetadata mempool.SignedTransactionMetadata) { signedTransactionMetadata.OnSignaturesInvalid(func(err error) { - fmt.Printf("%s > [%s] MemPool.SignedTransactionSignaturesInvalid(%s): %s\n", n.Name, engineName, err, signedTransactionMetadata.ID()) + instance.LogTrace("MemPool.SignedTransactionSignaturesInvalid", "tx", signedTransactionMetadata.ID(), "err", err) }) }, ) instance.Ledger.OnTransactionAttached(func(transactionMetadata mempool.TransactionMetadata) { - fmt.Printf("%s > [%s] Ledger.TransactionAttached: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("Ledger.TransactionAttached", "tx", transactionMetadata.ID()) transactionMetadata.OnSolid(func() { - fmt.Printf("%s > [%s] MemPool.TransactionSolid: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionSolid", "tx", transactionMetadata.ID()) }) transactionMetadata.OnExecuted(func() { - fmt.Printf("%s > [%s] MemPool.TransactionExecuted: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionExecuted", "tx", transactionMetadata.ID()) }) transactionMetadata.OnBooked(func() { - fmt.Printf("%s > [%s] MemPool.TransactionBooked: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionBooked", "tx", transactionMetadata.ID()) }) transactionMetadata.OnConflicting(func() { - fmt.Printf("%s > [%s] MemPool.TransactionConflicting: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionConflicting", "tx", transactionMetadata.ID()) }) transactionMetadata.OnAccepted(func() { - fmt.Printf("%s > [%s] MemPool.TransactionAccepted: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionAccepted", "tx", transactionMetadata.ID()) }) transactionMetadata.OnRejected(func() { - fmt.Printf("%s > [%s] MemPool.TransactionRejected: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionRejected", "tx", transactionMetadata.ID()) }) transactionMetadata.OnInvalid(func(err error) { - fmt.Printf("%s > [%s] MemPool.TransactionInvalid(%s): %s\n", n.Name, engineName, err, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionInvalid", "tx", transactionMetadata.ID(), "err", err) }) transactionMetadata.OnOrphanedSlotUpdated(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] MemPool.TransactionOrphanedSlotUpdated in slot %d: %s\n", n.Name, engineName, slot, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionOrphanedSlotUpdated", "tx", transactionMetadata.ID(), "slot", slot) }) transactionMetadata.OnCommittedSlotUpdated(func(slot iotago.SlotIndex) { - fmt.Printf("%s > [%s] MemPool.TransactionCommittedSlotUpdated in slot %d: %s\n", n.Name, engineName, slot, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionCommittedSlotUpdated", "tx", transactionMetadata.ID(), "slot", slot) }) transactionMetadata.OnPending(func() { - fmt.Printf("%s > [%s] MemPool.TransactionPending: %s\n", n.Name, engineName, transactionMetadata.ID()) + instance.LogTrace("MemPool.TransactionPending", "tx", transactionMetadata.ID()) }) }) } func (n *Node) attachEngineLogs(failOnBlockFiltered bool, instance *engine.Engine) { - engineName := fmt.Sprintf("%s - %s", lo.Cond(n.Protocol.Engines.Main.Get() != instance, "Candidate", "Main"), instance.Name()[:8]) - - n.attachEngineLogsWithName(failOnBlockFiltered, instance, engineName) + n.attachEngineLogsWithName(failOnBlockFiltered, instance) } func (n *Node) Wait() { diff --git a/pkg/testsuite/storage_settings.go b/pkg/testsuite/storage_settings.go index 956797be7..04409b25b 100644 --- a/pkg/testsuite/storage_settings.go +++ b/pkg/testsuite/storage_settings.go @@ -68,8 +68,15 @@ func (t *TestSuite) AssertCommitmentSlotIndexExists(slot iotago.SlotIndex, nodes return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found", node.Name, slot) } + // Make sure the main chain exists + mainChain := node.Protocol.Chains.Main.Get() + if mainChain == nil { + return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: main chain not found when checking for commitment at index %v", node.Name, slot) + } + // Make sure the commitment is also available in the ChainManager. - if node.Protocol.Chains.Main.Get().LatestCommitment.Get().ID().Slot() < slot { + latestCommitment := mainChain.LatestCommitment.Get() + if latestCommitment == nil || latestCommitment.ID().Slot() < slot { return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found in ChainManager", node.Name, slot) } @@ -126,8 +133,12 @@ func (t *TestSuite) AssertChainID(expectedChainID iotago.CommitmentID, nodes ... for _, node := range nodes { t.Eventually(func() error { - actualChainID := node.Protocol.Chains.Main.Get().ForkingPoint.Get().ID() + mainChain := node.Protocol.Chains.Main.Get() + if mainChain == nil { + return ierrors.Errorf("AssertChainID: %s: main chain not found", node.Name) + } + actualChainID := mainChain.ForkingPoint.Get().ID() if expectedChainID != actualChainID { fmt.Println(expectedChainID, actualChainID) From 6eae1b4fd03198bf5ceae3c449768375e4b2e65d Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 22:04:14 +0100 Subject: [PATCH 34/55] Refactor: started reverting unnecessary changes --- pkg/core/buffer/unsolid_commitment_buffer.go | 2 +- pkg/protocol/protocol.go | 12 ++++++------ pkg/testsuite/storage_settings.go | 16 ++-------------- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/pkg/core/buffer/unsolid_commitment_buffer.go b/pkg/core/buffer/unsolid_commitment_buffer.go index a6417a4d7..27df0dd51 100644 --- a/pkg/core/buffer/unsolid_commitment_buffer.go +++ b/pkg/core/buffer/unsolid_commitment_buffer.go @@ -51,7 +51,7 @@ func (u *UnsolidCommitmentBuffer[V]) Add(commitmentID iotago.CommitmentID, value u.mutex.RLock() defer u.mutex.RUnlock() - if u.lastEvictedSlot != 0 && commitmentID.Slot() <= u.lastEvictedSlot { + if commitmentID.Slot() <= u.lastEvictedSlot { return false } diff --git a/pkg/protocol/protocol.go b/pkg/protocol/protocol.go index 7bef7c575..fbc08d562 100644 --- a/pkg/protocol/protocol.go +++ b/pkg/protocol/protocol.go @@ -89,7 +89,7 @@ func New(logger log.Logger, workers *workerpool.Group, networkEndpoint network.E p.Constructed.Trigger() - p.waitMainEngineInitialized() + p.waitInitialized() }) } @@ -204,14 +204,14 @@ func (p *Protocol) initNetwork() (shutdown func()) { ) } -// waitMainEngineInitialized waits until the main engine is initialized. -func (p *Protocol) waitMainEngineInitialized() { +// waitInitialized waits until the main engine is initialized (published its root commitment). +func (p *Protocol) waitInitialized() { var waitInitialized sync.WaitGroup waitInitialized.Add(1) - p.Engines.Main.OnUpdateOnce(func(_ *engine.Engine, engine *engine.Engine) { - engine.Initialized.OnTrigger(waitInitialized.Done) - }) + p.Commitments.Root.OnUpdateOnce(func(_, rootCommitment *Commitment) { + waitInitialized.Done() + }, func(_ *Commitment, rootCommitment *Commitment) bool { return rootCommitment != nil }) waitInitialized.Wait() } diff --git a/pkg/testsuite/storage_settings.go b/pkg/testsuite/storage_settings.go index 04409b25b..23dc9bfbd 100644 --- a/pkg/testsuite/storage_settings.go +++ b/pkg/testsuite/storage_settings.go @@ -68,15 +68,8 @@ func (t *TestSuite) AssertCommitmentSlotIndexExists(slot iotago.SlotIndex, nodes return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found", node.Name, slot) } - // Make sure the main chain exists - mainChain := node.Protocol.Chains.Main.Get() - if mainChain == nil { - return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: main chain not found when checking for commitment at index %v", node.Name, slot) - } - // Make sure the commitment is also available in the ChainManager. - latestCommitment := mainChain.LatestCommitment.Get() - if latestCommitment == nil || latestCommitment.ID().Slot() < slot { + if node.Protocol.Chains.Main.Get().LatestCommitment.Get().ID().Slot() < slot { return ierrors.Errorf("AssertCommitmentSlotIndexExists: %s: commitment at index %v not found in ChainManager", node.Name, slot) } @@ -133,12 +126,7 @@ func (t *TestSuite) AssertChainID(expectedChainID iotago.CommitmentID, nodes ... for _, node := range nodes { t.Eventually(func() error { - mainChain := node.Protocol.Chains.Main.Get() - if mainChain == nil { - return ierrors.Errorf("AssertChainID: %s: main chain not found", node.Name) - } - - actualChainID := mainChain.ForkingPoint.Get().ID() + actualChainID := node.Protocol.Chains.Main.Get().ForkingPoint.Get().ID() if expectedChainID != actualChainID { fmt.Println(expectedChainID, actualChainID) From 7675908672be3625a38c60628ba7060ef027ba5b Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 22:07:24 +0100 Subject: [PATCH 35/55] Refactor: reverted changes --- pkg/testsuite/storage_settings.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/testsuite/storage_settings.go b/pkg/testsuite/storage_settings.go index 23dc9bfbd..956797be7 100644 --- a/pkg/testsuite/storage_settings.go +++ b/pkg/testsuite/storage_settings.go @@ -127,6 +127,7 @@ func (t *TestSuite) AssertChainID(expectedChainID iotago.CommitmentID, nodes ... for _, node := range nodes { t.Eventually(func() error { actualChainID := node.Protocol.Chains.Main.Get().ForkingPoint.Get().ID() + if expectedChainID != actualChainID { fmt.Println(expectedChainID, actualChainID) From 8a2a19ee55152920af7ae3a2dd97b223f60eb3ce Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 22:08:37 +0100 Subject: [PATCH 36/55] Refactor: reverted rename --- pkg/protocol/chain.go | 14 +++++++------- pkg/protocol/commitment.go | 4 ++-- pkg/protocol/protocol_warp_sync.go | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 86b1d1d30..2cf78e72b 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -48,8 +48,8 @@ type Chain struct { // latest verified commitment. VerifiedWeight reactive.Variable[uint64] - // WarpSyncModeEnabled contains a flag that indicates whether this chain is in warp sync mode. - WarpSyncModeEnabled reactive.Variable[bool] + // WarpSyncMode contains a flag that indicates whether this chain is in warp sync mode. + WarpSyncMode reactive.Variable[bool] // OutOfSyncThreshold contains the slot at which the chain will consider itself to be out of sync and switch to warp // sync mode. It is derived from the latest network slot minus two times the max committable age. @@ -92,7 +92,7 @@ func newChain(chains *Chains) *Chain { ClaimedWeight: reactive.NewVariable[uint64](), AttestedWeight: reactive.NewVariable[uint64](), VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncModeEnabled: reactive.NewVariable[bool]().Init(true), + WarpSyncMode: reactive.NewVariable[bool]().Init(true), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), RequestAttestations: reactive.NewVariable[bool](), StartEngine: reactive.NewVariable[bool](), @@ -187,7 +187,7 @@ func (c *Chain) initLogger() (shutdown func()) { c.Logger, shutdown = c.chains.NewEntityLogger("") return lo.Batch( - c.WarpSyncModeEnabled.LogUpdates(c, log.LevelTrace, "WarpSyncModeEnabled"), + c.WarpSyncMode.LogUpdates(c, log.LevelTrace, "WarpSyncMode"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), c.ClaimedWeight.LogUpdates(c, log.LevelTrace, "ClaimedWeight"), @@ -217,9 +217,9 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { return latestProducedCommitment.cumulativeWeight() }, c.LatestProducedCommitment)), - c.WarpSyncModeEnabled.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { return warpSyncModeEnabled(enabled, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) - }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncModeEnabled.Get())), + }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())), c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) @@ -299,7 +299,7 @@ func (c *Chain) dispatchBlockToSpawnedEngine(block *model.Block, src peer.ID) (d } // perform additional checks if we are in warp sync mode (only let blocks pass that we requested) - if c.WarpSyncModeEnabled.Get() { + if c.WarpSyncMode.Get() { // abort if the target commitment does not exist targetCommitment, targetCommitmentExists := c.Commitment(targetSlot) if !targetCommitmentExists { diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 1b956deba..8f2b7b5ee 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -276,7 +276,7 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, parentIsFullyBooked bool, isFullyBooked bool) bool { return engineInstance != nil && warpSyncModeEnabled && parentIsFullyBooked && !isFullyBooked - }, chain.Engine, chain.WarpSyncModeEnabled, parent.IsFullyBooked, c.IsFullyBooked)) + }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an @@ -284,7 +284,7 @@ func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func func (c *Commitment) deriveReplayDroppedBlocks(chain *Chain) func() { return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, isAboveLatestVerifiedCommitment bool) bool { return engineInstance != nil && !warpSyncModeEnabled && isAboveLatestVerifiedCommitment - }, chain.Engine, chain.WarpSyncModeEnabled, c.IsAboveLatestVerifiedCommitment)) + }, chain.Engine, chain.WarpSyncMode, c.IsAboveLatestVerifiedCommitment)) } // forceChain forces the Chain of this Commitment to the given Chain by promoting it to the main child of its parent if diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 374aaa3c6..ae2cb7576 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -47,7 +47,7 @@ func newWarpSyncProtocol(protocol *Protocol) *WarpSyncProtocol { protocol.Constructed.OnTrigger(func() { protocol.Chains.WithInitializedEngines(func(chain *Chain, engine *engine.Engine) (shutdown func()) { - return chain.WarpSyncModeEnabled.OnUpdate(func(_ bool, warpSyncModeEnabled bool) { + return chain.WarpSyncMode.OnUpdate(func(_ bool, warpSyncModeEnabled bool) { if warpSyncModeEnabled { engine.Workers.WaitChildren() engine.Reset() @@ -110,7 +110,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - if !chain.WarpSyncModeEnabled.Get() { + if !chain.WarpSyncMode.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return @@ -161,7 +161,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo targetEngine.Workers.WaitChildren() - if !chain.WarpSyncModeEnabled.Get() { + if !chain.WarpSyncMode.Get() { w.LogTrace("response for chain without warp-sync", "chain", chain.LogName(), "fromPeer", from) return blocksToWarpSync @@ -177,7 +177,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 2. Mark all blocks as accepted // 3. Force commitment of the slot forceCommitmentFunc := func() { - if !chain.WarpSyncModeEnabled.Get() { + if !chain.WarpSyncMode.Get() { return } From aa420c9902821d7a0ac3bb97c70cdf131bf8b509 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 22:11:38 +0100 Subject: [PATCH 37/55] Refactor: addressed linter issues --- pkg/protocol/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/protocol.go b/pkg/protocol/protocol.go index fbc08d562..436ef0dcd 100644 --- a/pkg/protocol/protocol.go +++ b/pkg/protocol/protocol.go @@ -209,7 +209,7 @@ func (p *Protocol) waitInitialized() { var waitInitialized sync.WaitGroup waitInitialized.Add(1) - p.Commitments.Root.OnUpdateOnce(func(_, rootCommitment *Commitment) { + p.Commitments.Root.OnUpdateOnce(func(_ *Commitment, _ *Commitment) { waitInitialized.Done() }, func(_ *Commitment, rootCommitment *Commitment) bool { return rootCommitment != nil }) From dd8771a86ea75447f69f575efc45c9bddb303aeb Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 22:37:42 +0100 Subject: [PATCH 38/55] Refactor: reduced changes --- pkg/protocol/chain.go | 60 +++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 2cf78e72b..08091dc1d 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -29,9 +29,6 @@ type Chain struct { // LatestAttestedCommitment contains the latest commitment of this chain for which attestations were received. LatestAttestedCommitment reactive.Variable[*Commitment] - // LatestFullyBookedCommitment contains the latest commitment of this chain for which all blocks were booked. - LatestFullyBookedCommitment reactive.Variable[*Commitment] - // LatestProducedCommitment contains the latest commitment of this chain that we produced ourselves by booking the // corresponding blocks in the Engine. LatestProducedCommitment reactive.Variable[*Commitment] @@ -51,6 +48,9 @@ type Chain struct { // WarpSyncMode contains a flag that indicates whether this chain is in warp sync mode. WarpSyncMode reactive.Variable[bool] + // LatestFullyBookedSlot contains the latest commitment of this chain for which all blocks were booked. + LatestFullyBookedSlot reactive.Variable[iotago.SlotIndex] + // OutOfSyncThreshold contains the slot at which the chain will consider itself to be out of sync and switch to warp // sync mode. It is derived from the latest network slot minus two times the max committable age. OutOfSyncThreshold reactive.Variable[iotago.SlotIndex] @@ -82,22 +82,22 @@ type Chain struct { // newChain creates a new chain instance. func newChain(chains *Chains) *Chain { c := &Chain{ - ForkingPoint: reactive.NewVariable[*Commitment](), - ParentChain: reactive.NewVariable[*Chain](), - ChildChains: reactive.NewSet[*Chain](), - LatestCommitment: reactive.NewVariable[*Commitment](), - LatestAttestedCommitment: reactive.NewVariable[*Commitment](), - LatestFullyBookedCommitment: reactive.NewVariable[*Commitment](), - LatestProducedCommitment: reactive.NewVariable[*Commitment](), - ClaimedWeight: reactive.NewVariable[uint64](), - AttestedWeight: reactive.NewVariable[uint64](), - VerifiedWeight: reactive.NewVariable[uint64](), - WarpSyncMode: reactive.NewVariable[bool]().Init(true), - OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), - RequestAttestations: reactive.NewVariable[bool](), - StartEngine: reactive.NewVariable[bool](), - Engine: reactive.NewVariable[*engine.Engine](), - IsEvicted: reactive.NewEvent(), + ForkingPoint: reactive.NewVariable[*Commitment](), + ParentChain: reactive.NewVariable[*Chain](), + ChildChains: reactive.NewSet[*Chain](), + LatestCommitment: reactive.NewVariable[*Commitment](), + LatestAttestedCommitment: reactive.NewVariable[*Commitment](), + LatestProducedCommitment: reactive.NewVariable[*Commitment](), + ClaimedWeight: reactive.NewVariable[uint64](), + AttestedWeight: reactive.NewVariable[uint64](), + VerifiedWeight: reactive.NewVariable[uint64](), + WarpSyncMode: reactive.NewVariable[bool]().Init(true), + LatestFullyBookedSlot: reactive.NewVariable[iotago.SlotIndex](), + OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), + RequestAttestations: reactive.NewVariable[bool](), + StartEngine: reactive.NewVariable[bool](), + Engine: reactive.NewVariable[*engine.Engine](), + IsEvicted: reactive.NewEvent(), chains: chains, commitments: shrinkingmap.New[iotago.SlotIndex, *Commitment](), @@ -188,6 +188,7 @@ func (c *Chain) initLogger() (shutdown func()) { return lo.Batch( c.WarpSyncMode.LogUpdates(c, log.LevelTrace, "WarpSyncMode"), + c.LatestFullyBookedSlot.LogUpdates(c, log.LevelTrace, "LatestFullyBookedSlot"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), c.ClaimedWeight.LogUpdates(c, log.LevelTrace, "ClaimedWeight"), @@ -195,7 +196,6 @@ func (c *Chain) initLogger() (shutdown func()) { c.VerifiedWeight.LogUpdates(c, log.LevelTrace, "VerifiedWeight"), c.LatestCommitment.LogUpdates(c, log.LevelTrace, "LatestCommitment", (*Commitment).LogName), c.LatestAttestedCommitment.LogUpdates(c, log.LevelTrace, "LatestAttestedCommitment", (*Commitment).LogName), - c.LatestFullyBookedCommitment.LogUpdates(c, log.LevelTrace, "LatestFullyBookedCommitment", (*Commitment).LogName), c.LatestProducedCommitment.LogUpdates(c, log.LevelDebug, "LatestProducedCommitment", (*Commitment).LogName), c.RequestAttestations.LogUpdates(c, log.LevelTrace, "RequestAttestations"), c.StartEngine.LogUpdates(c, log.LevelDebug, "StartEngine"), @@ -217,9 +217,9 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { return latestProducedCommitment.cumulativeWeight() }, c.LatestProducedCommitment)), - c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - return warpSyncModeEnabled(enabled, latestFullyBookedCommitment, latestSeenSlot, outOfSyncThreshold) - }, c.LatestFullyBookedCommitment, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())), + c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(enabled, latestFullyBookedSlot, latestSeenSlot, outOfSyncThreshold) + }, c.LatestFullyBookedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())), c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) @@ -278,8 +278,8 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), - newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedCommitment.Set(newCommitment) }), newCommitment.IsCommitted.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), + newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedSlot.Set(newCommitment.Slot()) }), ) } @@ -347,18 +347,12 @@ func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.Slo } // warpSyncModeEnabled determines whether warp sync mode should be enabled or not. -func warpSyncModeEnabled(enabled bool, latestFullyBookedCommitment *Commitment, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - // latest produced commitment is nil if we have not produced any commitment yet (intermediary state during - // startup) - if latestFullyBookedCommitment == nil { - return enabled - } - +func warpSyncModeEnabled(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold if enabled { - return latestFullyBookedCommitment.ID().Slot() < latestSeenSlot + return latestFullyBookedSlot < latestSeenSlot } // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestFullyBookedCommitment.ID().Slot() < outOfSyncThreshold + return latestFullyBookedSlot < outOfSyncThreshold } From 044b797e31f9120ada5af21edcbd1abe27636bd8 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:04:29 +0100 Subject: [PATCH 39/55] Refactor: reverted unnecessary changes --- pkg/protocol/commitment.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 8f2b7b5ee..ff4c7bb69 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -156,11 +156,10 @@ func (c *Commitment) initLogger() (shutdown func()) { // initDerivedProperties initializes the behavior of this Commitment by setting up the relations between its properties. func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( - // mark commitments that are marked as root as verified + // mark commitments that are marked as root as committed c.IsCommitted.InheritFrom(c.IsRoot), - c.ReplayDroppedBlocks.InheritFrom(c.IsRoot), - // mark commitments that are marked as verified as attested and fully booked + // mark commitments that are marked as committed as attested and fully booked c.IsAttested.InheritFrom(c.IsCommitted), c.IsFullyBooked.InheritFrom(c.IsCommitted), @@ -230,7 +229,6 @@ func (c *Commitment) deriveChain(parent *Commitment) func() { if currentChain == nil { currentChain = c.commitments.protocol.Chains.newChain() currentChain.ForkingPoint.Set(c) - currentChain.LatestProducedCommitment.Set(parent) } return currentChain From 010e1038786f140aec754691bc9198a363dcae63 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:18:11 +0100 Subject: [PATCH 40/55] Refactor: reverted more changes --- pkg/protocol/chain.go | 51 ++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 08091dc1d..fc9ac9c01 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -209,21 +209,10 @@ func (c *Chain) initLogger() (shutdown func()) { // initDerivedProperties initializes the behavior of this chain by setting up the relations between its properties. func (c *Chain) initDerivedProperties() (shutdown func()) { return lo.Batch( - c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { - return latestCommitment.cumulativeWeight() - }, c.LatestCommitment)), - - c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { - return latestProducedCommitment.cumulativeWeight() - }, c.LatestProducedCommitment)), - - c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - return warpSyncModeEnabled(enabled, latestFullyBookedSlot, latestSeenSlot, outOfSyncThreshold) - }, c.LatestFullyBookedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())), - - c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { - return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) - }), + c.deriveClaimedWeight(), + c.deriveVerifiedWeight(), + c.deriveLatestAttestedWeight(), + c.deriveWarpSyncMode(), c.ForkingPoint.WithValue(func(forkingPoint *Commitment) (shutdown func()) { return c.deriveParentChain(forkingPoint) @@ -241,6 +230,38 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { ) } +// deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. +func (c *Chain) deriveWarpSyncMode() func() { + return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + return warpSyncModeEnabled(enabled, latestFullyBookedSlot, latestSeenSlot, outOfSyncThreshold) + }, c.LatestFullyBookedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) +} + +// deriveClaimedWeight defines how a chain determines its claimed weight (by setting the cumulative weight of the +// latest commitment). +func (c *Chain) deriveClaimedWeight() func() { + return c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { + return latestCommitment.cumulativeWeight() + }, c.LatestCommitment)) +} + +// deriveLatestAttestedWeight defines how a chain determines its attested weight (by inheriting the cumulative attested +// weight of the latest attested commitment). It uses inheritance instead of simply setting the value as the cumulative +// attested weight can change over time depending on the attestations that are received. +func (c *Chain) deriveLatestAttestedWeight() func() { + return c.LatestAttestedCommitment.WithNonEmptyValue(func(latestAttestedCommitment *Commitment) (shutdown func()) { + return c.AttestedWeight.InheritFrom(latestAttestedCommitment.CumulativeAttestedWeight) + }) +} + +// deriveVerifiedWeight defines how a chain determines its verified weight (by setting the cumulative weight of the +// latest produced commitment). +func (c *Chain) deriveVerifiedWeight() func() { + return c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { + return latestProducedCommitment.cumulativeWeight() + }, c.LatestProducedCommitment)) +} + // deriveChildChains defines how a chain determines its ChildChains (by adding each child to the set). func (c *Chain) deriveChildChains(child *Chain) func() { c.ChildChains.Add(child) From a3a5c72665c5cf051bd25cd6f9153f8c1162fb04 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:20:56 +0100 Subject: [PATCH 41/55] Refactor: minimizing more changes --- pkg/protocol/chain.go | 33 ++++++++++++++++++--------------- pkg/protocol/commitment.go | 9 --------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index fc9ac9c01..ef84c38b5 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -232,8 +232,14 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { // deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. func (c *Chain) deriveWarpSyncMode() func() { - return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - return warpSyncModeEnabled(enabled, latestFullyBookedSlot, latestSeenSlot, outOfSyncThreshold) + return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + // if warp sync mode is enabled, keep it enabled until we have fully booked all slots + if warpSyncMode { + return latestFullyBookedSlot < latestSeenSlot + } + + // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold + return latestFullyBookedSlot < outOfSyncThreshold }, c.LatestFullyBookedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) } @@ -241,7 +247,11 @@ func (c *Chain) deriveWarpSyncMode() func() { // latest commitment). func (c *Chain) deriveClaimedWeight() func() { return c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { - return latestCommitment.cumulativeWeight() + if latestCommitment == nil { + return 0 + } + + return latestCommitment.CumulativeWeight() }, c.LatestCommitment)) } @@ -258,7 +268,11 @@ func (c *Chain) deriveLatestAttestedWeight() func() { // latest produced commitment). func (c *Chain) deriveVerifiedWeight() func() { return c.VerifiedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestProducedCommitment *Commitment) uint64 { - return latestProducedCommitment.cumulativeWeight() + if latestProducedCommitment == nil { + return 0 + } + + return latestProducedCommitment.CumulativeWeight() }, c.LatestProducedCommitment)) } @@ -366,14 +380,3 @@ func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.Slo return 0 } - -// warpSyncModeEnabled determines whether warp sync mode should be enabled or not. -func warpSyncModeEnabled(enabled bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - // if warp sync mode is enabled, keep it enabled until we are no longer below the warp sync threshold - if enabled { - return latestFullyBookedSlot < latestSeenSlot - } - - // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestFullyBookedSlot < outOfSyncThreshold -} diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index ff4c7bb69..4ce7b79da 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -294,12 +294,3 @@ func (c *Commitment) forceChain(targetChain *Chain) { } } } - -// cumulativeWeight returns the cumulative weight of this Commitment while gracefully handling nil receivers. -func (c *Commitment) cumulativeWeight() uint64 { - if c == nil { - return 0 - } - - return c.CumulativeWeight() -} From 0ce46e7268eb8d3eb0a9f5e2c81c6b793a8d779f Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:27:48 +0100 Subject: [PATCH 42/55] Refactor: reverted more code --- pkg/protocol/chain.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index ef84c38b5..31ee3dad0 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -223,9 +223,7 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { }), c.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - return outOfSyncThreshold(engineInstance, latestSeenSlot) - }, c.chains.LatestSeenSlot)) + return c.deriveOutOfSyncThreshold(engineInstance) }), ) } @@ -245,7 +243,7 @@ func (c *Chain) deriveWarpSyncMode() func() { // deriveClaimedWeight defines how a chain determines its claimed weight (by setting the cumulative weight of the // latest commitment). -func (c *Chain) deriveClaimedWeight() func() { +func (c *Chain) deriveClaimedWeight() (shutdown func()) { return c.ClaimedWeight.DeriveValueFrom(reactive.NewDerivedVariable(func(_ uint64, latestCommitment *Commitment) uint64 { if latestCommitment == nil { return 0 @@ -305,6 +303,14 @@ func (c *Chain) deriveParentChain(forkingPoint *Commitment) (shutdown func()) { return nil } +// deriveOutOfSyncThreshold defines how a chain determines its "out of sync" threshold (the latest seen slot minus 2 +// times the max committable age or 0 if this would cause an overflow to the negative numbers). +func (c *Chain) deriveOutOfSyncThreshold(engineInstance *engine.Engine) func() { + return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { + return outOfSyncThreshold(engineInstance, latestSeenSlot) + }, c.chains.LatestSeenSlot)) +} + // addCommitment adds the given commitment to this chain. func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { c.commitments.Set(newCommitment.Slot(), newCommitment) From b6b430fe0bbebedc68fd2a5bf2e42b03c71aff6d Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:32:19 +0100 Subject: [PATCH 43/55] Refactor: revert --- pkg/protocol/chain.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 31ee3dad0..43ef38211 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -307,7 +307,11 @@ func (c *Chain) deriveParentChain(forkingPoint *Commitment) (shutdown func()) { // times the max committable age or 0 if this would cause an overflow to the negative numbers). func (c *Chain) deriveOutOfSyncThreshold(engineInstance *engine.Engine) func() { return c.OutOfSyncThreshold.DeriveValueFrom(reactive.NewDerivedVariable(func(_ iotago.SlotIndex, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - return outOfSyncThreshold(engineInstance, latestSeenSlot) + if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { + return latestSeenSlot - outOfSyncOffset + } + + return 0 }, c.chains.LatestSeenSlot)) } @@ -377,12 +381,3 @@ func (c *Chain) verifiedWeight() reactive.Variable[uint64] { func (c *Chain) attestedWeight() reactive.Variable[uint64] { return c.AttestedWeight } - -// outOfSyncThreshold returns the slot index at which the node is considered out of sync. -func outOfSyncThreshold(engineInstance *engine.Engine, latestSeenSlot iotago.SlotIndex) iotago.SlotIndex { - if outOfSyncOffset := 2 * engineInstance.LatestAPI().ProtocolParameters().MaxCommittableAge(); outOfSyncOffset < latestSeenSlot { - return latestSeenSlot - outOfSyncOffset - } - - return 0 -} From 5beda14f877c893f1cb08570b23d3f34b803b8ca Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Sun, 3 Dec 2023 23:53:25 +0100 Subject: [PATCH 44/55] Refactor: revert --- pkg/protocol/chain.go | 16 +++------------- pkg/protocol/commitment.go | 8 ++++---- pkg/protocol/engine/clock/blocktime/clock.go | 4 +--- pkg/protocol/engines.go | 3 --- pkg/tests/loss_of_acceptance_test.go | 2 +- 5 files changed, 9 insertions(+), 24 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 43ef38211..a2f2671bd 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -156,8 +156,6 @@ func (c *Chain) DispatchBlock(block *model.Block, src peer.ID) (dispatched bool) func (c *Chain) Commitment(slot iotago.SlotIndex) (commitment *Commitment, exists bool) { for currentChain := c; currentChain != nil; { switch forkingPoint := currentChain.ForkingPoint.Get(); { - case forkingPoint == nil: - return nil, false case forkingPoint.Slot() == slot: return forkingPoint, true case slot > forkingPoint.Slot(): @@ -214,17 +212,9 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { c.deriveLatestAttestedWeight(), c.deriveWarpSyncMode(), - c.ForkingPoint.WithValue(func(forkingPoint *Commitment) (shutdown func()) { - return c.deriveParentChain(forkingPoint) - }), - - c.ParentChain.WithNonEmptyValue(func(parentChain *Chain) (shutdown func()) { - return parentChain.deriveChildChains(c) - }), - - c.Engine.WithNonEmptyValue(func(engineInstance *engine.Engine) (shutdown func()) { - return c.deriveOutOfSyncThreshold(engineInstance) - }), + c.ForkingPoint.WithValue(c.deriveParentChain), + c.ParentChain.WithNonEmptyValue(lo.Bind(c, (*Chain).deriveChildChains)), + c.Engine.WithNonEmptyValue(c.deriveOutOfSyncThreshold), ) } diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 4ce7b79da..1e0898be4 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -272,16 +272,16 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting // warp sync, and we are the directly above the latest verified Commitment. func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { - return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, parentIsFullyBooked bool, isFullyBooked bool) bool { - return engineInstance != nil && warpSyncModeEnabled && parentIsFullyBooked && !isFullyBooked + return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { + return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an // engine, is no longer requesting warp sync, and we are above the latest verified Commitment. func (c *Commitment) deriveReplayDroppedBlocks(chain *Chain) func() { - return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncModeEnabled bool, isAboveLatestVerifiedCommitment bool) bool { - return engineInstance != nil && !warpSyncModeEnabled && isAboveLatestVerifiedCommitment + return c.ReplayDroppedBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, engineInstance *engine.Engine, warpSyncing bool, isAboveLatestVerifiedCommitment bool) bool { + return engineInstance != nil && !warpSyncing && isAboveLatestVerifiedCommitment }, chain.Engine, chain.WarpSyncMode, c.IsAboveLatestVerifiedCommitment)) } diff --git a/pkg/protocol/engine/clock/blocktime/clock.go b/pkg/protocol/engine/clock/blocktime/clock.go index 51801187b..506387df0 100644 --- a/pkg/protocol/engine/clock/blocktime/clock.go +++ b/pkg/protocol/engine/clock/blocktime/clock.go @@ -55,9 +55,7 @@ func NewProvider(opts ...options.Option[Clock]) module.Provider[*engine.Engine, asyncOpt := event.WithWorkerPool(c.workerPool) c.HookStopped(lo.Batch( e.Events.BlockGadget.BlockAccepted.Hook(func(block *blocks.Block) { - block.Notarized().OnTrigger(func() { - c.acceptedTime.Advance(block.IssuingTime()) - }) + c.acceptedTime.Advance(block.IssuingTime()) }, asyncOpt).Unhook, e.Events.BlockGadget.BlockConfirmed.Hook(func(block *blocks.Block) { diff --git a/pkg/protocol/engines.go b/pkg/protocol/engines.go index e30cfe19f..781dc0d32 100644 --- a/pkg/protocol/engines.go +++ b/pkg/protocol/engines.go @@ -9,7 +9,6 @@ import ( "github.com/iotaledger/hive.go/ds/reactive" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" - "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/ioutils" "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/workerpool" @@ -159,8 +158,6 @@ func (e *Engines) ForkAtSlot(slot iotago.SlotIndex) (*engine.Engine, error) { return nil, ierrors.Wrap(err, "error while rolling back attestations storage on candidate engine") } - e.Log("forked engine", log.LevelTrace, "name", newEngineAlias[0:8], "slot", slot) - return candidateEngine, nil } diff --git a/pkg/tests/loss_of_acceptance_test.go b/pkg/tests/loss_of_acceptance_test.go index 93ec48b57..eae12a80c 100644 --- a/pkg/tests/loss_of_acceptance_test.go +++ b/pkg/tests/loss_of_acceptance_test.go @@ -32,7 +32,7 @@ func TestLossOfAcceptanceFromGenesis(t *testing.T) { 5, ), ), - testsuite.WithWaitFor(30*time.Second), + testsuite.WithWaitFor(15*time.Second), ) defer ts.Shutdown() From 2b819f9908662d578b677c3263fce95cad92caf2 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 00:29:37 +0100 Subject: [PATCH 45/55] Refactor: reverted more --- pkg/protocol/chain.go | 2 +- pkg/protocol/commitment.go | 22 +++++++++++----------- pkg/protocol/commitments.go | 2 +- pkg/protocol/protocol_warp_sync.go | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index a2f2671bd..00054ce72 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -313,7 +313,7 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), - newCommitment.IsCommitted.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), + newCommitment.IsVerified.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedSlot.Set(newCommitment.Slot()) }), ) } diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 1e0898be4..e120b02f7 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -62,9 +62,9 @@ type Commitment struct { // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). IsCommittable reactive.Event - // IsCommitted contains a flag indicating if we Commitment produced this Commitment ourselves by replaying all the - // blocks of the Commitment. - IsCommitted reactive.Event + // IsVerified contains a flag indicating if this Commitment is verified (we produced this Commitment ourselves by + // booking all the contained blocks and transactions). + IsVerified reactive.Event // IsAboveLatestVerifiedCommitment contains a flag indicating if this Commitment is above the latest verified // Commitment. @@ -102,7 +102,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen IsAttested: reactive.NewEvent(), IsFullyBooked: reactive.NewEvent(), IsCommittable: reactive.NewEvent(), - IsCommitted: reactive.NewEvent(), + IsVerified: reactive.NewEvent(), IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), ReplayDroppedBlocks: reactive.NewVariable[bool](), IsEvicted: reactive.NewEvent(), @@ -145,7 +145,7 @@ func (c *Commitment) initLogger() (shutdown func()) { c.IsAttested.LogUpdates(c, log.LevelTrace, "IsAttested"), c.IsFullyBooked.LogUpdates(c, log.LevelTrace, "IsFullyBooked"), c.IsCommittable.LogUpdates(c, log.LevelTrace, "IsCommittable"), - c.IsCommitted.LogUpdates(c, log.LevelTrace, "IsCommitted"), + c.IsVerified.LogUpdates(c, log.LevelTrace, "IsVerified"), c.ReplayDroppedBlocks.LogUpdates(c, log.LevelTrace, "ReplayDroppedBlocks"), c.IsEvicted.LogUpdates(c, log.LevelTrace, "IsEvicted"), @@ -156,12 +156,12 @@ func (c *Commitment) initLogger() (shutdown func()) { // initDerivedProperties initializes the behavior of this Commitment by setting up the relations between its properties. func (c *Commitment) initDerivedProperties() (shutdown func()) { return lo.Batch( - // mark commitments that are marked as root as committed - c.IsCommitted.InheritFrom(c.IsRoot), + // mark commitments that are marked as root as verified + c.IsVerified.InheritFrom(c.IsRoot), - // mark commitments that are marked as committed as attested and fully booked - c.IsAttested.InheritFrom(c.IsCommitted), - c.IsFullyBooked.InheritFrom(c.IsCommitted), + // mark commitments that are marked as verified as attested and fully booked + c.IsAttested.InheritFrom(c.IsVerified), + c.IsFullyBooked.InheritFrom(c.IsVerified), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent @@ -258,7 +258,7 @@ func (c *Commitment) deriveCumulativeAttestedWeight(parent *Commitment) func() { func (c *Commitment) deriveIsAboveLatestVerifiedCommitment(parent *Commitment) func() { return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsCommitted bool, isCommitted bool) bool { return parentAboveLatestVerifiedCommitment || (parentIsCommitted && !isCommitted) - }, parent.IsAboveLatestVerifiedCommitment, parent.IsCommitted, c.IsCommitted)) + }, parent.IsAboveLatestVerifiedCommitment, parent.IsVerified, c.IsVerified)) } // deriveRequestAttestations derives the RequestAttestations flag of this Commitment which is true if our Chain is diff --git a/pkg/protocol/commitments.go b/pkg/protocol/commitments.go index 4066fc01d..857b1c00d 100644 --- a/pkg/protocol/commitments.go +++ b/pkg/protocol/commitments.go @@ -155,7 +155,7 @@ func (c *Commitments) publishEngineCommitments(chain *Chain, engine *engine.Engi // mark it as produced by ourselves and force it to be on the right chain (in case our chain produced a // different commitment than the one we erroneously expected it to be - we always trust our engine most). publishedCommitment.AttestedWeight.Set(publishedCommitment.Weight.Get()) - publishedCommitment.IsCommitted.Set(true) + publishedCommitment.IsVerified.Set(true) publishedCommitment.forceChain(chain) } }) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index ae2cb7576..4c9e6d466 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -259,7 +259,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // force commit one by one and wait for the parent to be committed before we can commit the next one commitment.Parent.WithNonEmptyValue(func(parent *Commitment) (teardown func()) { - return parent.IsCommitted.WithNonEmptyValue(func(_ bool) (teardown func()) { + return parent.IsVerified.WithNonEmptyValue(func(_ bool) (teardown func()) { return commitment.IsCommittable.OnTrigger(forceCommitmentFunc) }) }) From 75240f00ed6a406db79356a893518c920680f086 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 00:32:01 +0100 Subject: [PATCH 46/55] Refactor: revert --- pkg/protocol/commitment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index e120b02f7..49a8936a0 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -256,8 +256,8 @@ func (c *Commitment) deriveCumulativeAttestedWeight(parent *Commitment) func() { // deriveIsAboveLatestVerifiedCommitment derives the IsAboveLatestVerifiedCommitment flag of this Commitment which is // true if the parent is already above the latest verified Commitment or if the parent is verified and we are not. func (c *Commitment) deriveIsAboveLatestVerifiedCommitment(parent *Commitment) func() { - return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsCommitted bool, isCommitted bool) bool { - return parentAboveLatestVerifiedCommitment || (parentIsCommitted && !isCommitted) + return c.IsAboveLatestVerifiedCommitment.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, parentAboveLatestVerifiedCommitment bool, parentIsVerified bool, isVerified bool) bool { + return parentAboveLatestVerifiedCommitment || (parentIsVerified && !isVerified) }, parent.IsAboveLatestVerifiedCommitment, parent.IsVerified, c.IsVerified)) } From e83a78ee6d148d5a891df279059b71d1837e255f Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 00:38:18 +0100 Subject: [PATCH 47/55] Fix: fix possible nil pointer exception --- pkg/protocol/commitment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 49a8936a0..e49ad25ea 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -289,7 +289,7 @@ func (c *Commitment) deriveReplayDroppedBlocks(chain *Chain) func() { // the parent is on the target Chain. func (c *Commitment) forceChain(targetChain *Chain) { if currentChain := c.Chain.Get(); currentChain != targetChain { - if parent := c.Parent.Get(); parent.Chain.Get() == targetChain { + if parent := c.Parent.Get(); parent != nil && parent.Chain.Get() == targetChain { parent.MainChild.Set(c) } } From 90c1fd3f7dd9fe49eb2a1b1598b8135471bd092c Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 01:25:00 +0100 Subject: [PATCH 48/55] Refactor: renamed flags --- pkg/protocol/chain.go | 20 ++++++++++---------- pkg/protocol/commitment.go | 12 ++++++------ pkg/protocol/protocol_warp_sync.go | 6 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/protocol/chain.go b/pkg/protocol/chain.go index 00054ce72..3ec93212b 100644 --- a/pkg/protocol/chain.go +++ b/pkg/protocol/chain.go @@ -48,8 +48,8 @@ type Chain struct { // WarpSyncMode contains a flag that indicates whether this chain is in warp sync mode. WarpSyncMode reactive.Variable[bool] - // LatestFullyBookedSlot contains the latest commitment of this chain for which all blocks were booked. - LatestFullyBookedSlot reactive.Variable[iotago.SlotIndex] + // LatestSyncedSlot contains the latest commitment of this chain for which all blocks were booked. + LatestSyncedSlot reactive.Variable[iotago.SlotIndex] // OutOfSyncThreshold contains the slot at which the chain will consider itself to be out of sync and switch to warp // sync mode. It is derived from the latest network slot minus two times the max committable age. @@ -92,7 +92,7 @@ func newChain(chains *Chains) *Chain { AttestedWeight: reactive.NewVariable[uint64](), VerifiedWeight: reactive.NewVariable[uint64](), WarpSyncMode: reactive.NewVariable[bool]().Init(true), - LatestFullyBookedSlot: reactive.NewVariable[iotago.SlotIndex](), + LatestSyncedSlot: reactive.NewVariable[iotago.SlotIndex](), OutOfSyncThreshold: reactive.NewVariable[iotago.SlotIndex](), RequestAttestations: reactive.NewVariable[bool](), StartEngine: reactive.NewVariable[bool](), @@ -186,7 +186,7 @@ func (c *Chain) initLogger() (shutdown func()) { return lo.Batch( c.WarpSyncMode.LogUpdates(c, log.LevelTrace, "WarpSyncMode"), - c.LatestFullyBookedSlot.LogUpdates(c, log.LevelTrace, "LatestFullyBookedSlot"), + c.LatestSyncedSlot.LogUpdates(c, log.LevelTrace, "LatestSyncedSlot"), c.OutOfSyncThreshold.LogUpdates(c, log.LevelTrace, "OutOfSyncThreshold"), c.ForkingPoint.LogUpdates(c, log.LevelTrace, "ForkingPoint", (*Commitment).LogName), c.ClaimedWeight.LogUpdates(c, log.LevelTrace, "ClaimedWeight"), @@ -220,15 +220,15 @@ func (c *Chain) initDerivedProperties() (shutdown func()) { // deriveWarpSyncMode defines how a chain determines whether it is in warp sync mode or not. func (c *Chain) deriveWarpSyncMode() func() { - return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestFullyBookedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { - // if warp sync mode is enabled, keep it enabled until we have fully booked all slots + return c.WarpSyncMode.DeriveValueFrom(reactive.NewDerivedVariable3(func(warpSyncMode bool, latestSyncedSlot iotago.SlotIndex, latestSeenSlot iotago.SlotIndex, outOfSyncThreshold iotago.SlotIndex) bool { + // if warp sync mode is enabled, keep it enabled until we have synced all slots if warpSyncMode { - return latestFullyBookedSlot < latestSeenSlot + return latestSyncedSlot < latestSeenSlot } // if warp sync mode is disabled, enable it only if we fall below the out of sync threshold - return latestFullyBookedSlot < outOfSyncThreshold - }, c.LatestFullyBookedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) + return latestSyncedSlot < outOfSyncThreshold + }, c.LatestSyncedSlot, c.chains.LatestSeenSlot, c.OutOfSyncThreshold, c.WarpSyncMode.Get())) } // deriveClaimedWeight defines how a chain determines its claimed weight (by setting the cumulative weight of the @@ -314,7 +314,7 @@ func (c *Chain) addCommitment(newCommitment *Commitment) (shutdown func()) { return lo.Batch( newCommitment.IsAttested.OnTrigger(func() { c.LatestAttestedCommitment.Set(newCommitment) }), newCommitment.IsVerified.OnTrigger(func() { c.LatestProducedCommitment.Set(newCommitment) }), - newCommitment.IsFullyBooked.OnTrigger(func() { c.LatestFullyBookedSlot.Set(newCommitment.Slot()) }), + newCommitment.IsSynced.OnTrigger(func() { c.LatestSyncedSlot.Set(newCommitment.Slot()) }), ) } diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index e49ad25ea..eca4199ee 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -56,8 +56,8 @@ type Commitment struct { // IsAttested contains a flag indicating if we have received attestations for this Commitment. IsAttested reactive.Event - // IsFullyBooked contains a flag indicating if we have received all blocks for this Commitment. - IsFullyBooked reactive.Event + // IsSynced contains a flag indicating if we have downloaded all objects enclosed in this Commitment. + IsSynced reactive.Event // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). IsCommittable reactive.Event @@ -100,7 +100,7 @@ func newCommitment(commitments *Commitments, model *model.Commitment) *Commitmen CumulativeAttestedWeight: reactive.NewVariable[uint64](), IsRoot: reactive.NewEvent(), IsAttested: reactive.NewEvent(), - IsFullyBooked: reactive.NewEvent(), + IsSynced: reactive.NewEvent(), IsCommittable: reactive.NewEvent(), IsVerified: reactive.NewEvent(), IsAboveLatestVerifiedCommitment: reactive.NewVariable[bool](), @@ -143,7 +143,7 @@ func (c *Commitment) initLogger() (shutdown func()) { c.CumulativeAttestedWeight.LogUpdates(c, log.LevelTrace, "CumulativeAttestedWeight"), c.IsRoot.LogUpdates(c, log.LevelTrace, "IsRoot"), c.IsAttested.LogUpdates(c, log.LevelTrace, "IsAttested"), - c.IsFullyBooked.LogUpdates(c, log.LevelTrace, "IsFullyBooked"), + c.IsSynced.LogUpdates(c, log.LevelTrace, "IsSynced"), c.IsCommittable.LogUpdates(c, log.LevelTrace, "IsCommittable"), c.IsVerified.LogUpdates(c, log.LevelTrace, "IsVerified"), c.ReplayDroppedBlocks.LogUpdates(c, log.LevelTrace, "ReplayDroppedBlocks"), @@ -161,7 +161,7 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { // mark commitments that are marked as verified as attested and fully booked c.IsAttested.InheritFrom(c.IsVerified), - c.IsFullyBooked.InheritFrom(c.IsVerified), + c.IsSynced.InheritFrom(c.IsVerified), c.Parent.WithNonEmptyValue(func(parent *Commitment) func() { // the weight can be fixed as a one time operation (as it only relies on static information from the parent @@ -274,7 +274,7 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked - }, chain.Engine, chain.WarpSyncMode, parent.IsFullyBooked, c.IsFullyBooked)) + }, chain.Engine, chain.WarpSyncMode, parent.IsSynced, c.IsSynced)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 4c9e6d466..37946b6fd 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -249,7 +249,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // Once all blocks are fully booked we can mark the commitment that is minCommittableAge older as this // commitment to be committable. - commitment.IsFullyBooked.OnUpdateOnce(func(_ bool, _ bool) { + commitment.IsSynced.OnUpdateOnce(func(_ bool, _ bool) { if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - targetEngine.LatestAPI().ProtocolParameters().MinCommittableAge()); exists { w.workerPool.Submit(func() { committableCommitment.IsCommittable.Set(true) @@ -266,7 +266,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo if totalBlocks == 0 { commitment.IsCommittable.Set(true) - commitment.IsFullyBooked.Set(true) + commitment.IsSynced.Set(true) return blocksToWarpSync } @@ -291,7 +291,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - commitment.IsFullyBooked.Set(true) + commitment.IsSynced.Set(true) }) } } From 8b5b5dc46b2b6745eb7eb03dd0ee06108cf35a4e Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 01:40:18 +0100 Subject: [PATCH 49/55] Refactor: finished rename --- pkg/protocol/commitment.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index eca4199ee..2113592d9 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -178,7 +178,11 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { c.Chain.WithNonEmptyValue(func(chain *Chain) func() { return lo.Batch( c.deriveRequestAttestations(chain, parent), - c.deriveWarpSyncBlocks(chain, parent), + + // only start requesting blocks once the engine is ready + chain.WithInitializedEngine(func(_ *engine.Engine) (shutdown func()) { + return c.deriveWarpSyncBlocks(chain, parent) + }), ) }), ) @@ -272,9 +276,9 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting // warp sync, and we are the directly above the latest verified Commitment. func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { - return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable4(func(_ bool, engineInstance *engine.Engine, warpSync bool, parentIsFullyBooked bool, isFullyBooked bool) bool { - return engineInstance != nil && warpSync && parentIsFullyBooked && !isFullyBooked - }, chain.Engine, chain.WarpSyncMode, parent.IsSynced, c.IsSynced)) + return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, warpSyncMode bool, parentIsSynced bool, isSynced bool) bool { + return warpSyncMode && parentIsSynced && !isSynced + }, chain.WarpSyncMode, parent.IsSynced, c.IsSynced)) } // deriveReplayDroppedBlocks derives the ReplayDroppedBlocks flag of this Commitment which is true if our Chain has an From 524162284dda3cfd2ec65f3af2afdbc5e0649458 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 01:55:39 +0100 Subject: [PATCH 50/55] Refactor: refactored warpsync protocol --- pkg/protocol/protocol_warp_sync.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index 37946b6fd..c2a560d56 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -176,7 +176,7 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // 1. Mark all transactions as accepted // 2. Mark all blocks as accepted // 3. Force commitment of the slot - forceCommitmentFunc := func() { + commitmentFunc := func() { if !chain.WarpSyncMode.Get() { return } @@ -250,21 +250,23 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo // Once all blocks are fully booked we can mark the commitment that is minCommittableAge older as this // commitment to be committable. commitment.IsSynced.OnUpdateOnce(func(_ bool, _ bool) { - if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - targetEngine.LatestAPI().ProtocolParameters().MinCommittableAge()); exists { - w.workerPool.Submit(func() { + // update the flag in a worker since it can potentially cause a commit + w.workerPool.Submit(func() { + if committableCommitment, exists := chain.Commitment(commitmentID.Slot() - targetEngine.LatestAPI().ProtocolParameters().MinCommittableAge()); exists { committableCommitment.IsCommittable.Set(true) - }) - } + } + }) }) - // force commit one by one and wait for the parent to be committed before we can commit the next one + // force commit one by one and wait for the parent to be verified before we commit the next one commitment.Parent.WithNonEmptyValue(func(parent *Commitment) (teardown func()) { return parent.IsVerified.WithNonEmptyValue(func(_ bool) (teardown func()) { - return commitment.IsCommittable.OnTrigger(forceCommitmentFunc) + return commitment.IsCommittable.OnTrigger(commitmentFunc) }) }) if totalBlocks == 0 { + // mark empty slots as committable and synced commitment.IsCommittable.Set(true) commitment.IsSynced.Set(true) From c6a724238ef226830fbfe9e4eca417a7c0e22e90 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 01:59:22 +0100 Subject: [PATCH 51/55] Feat: updated comments --- pkg/protocol/commitment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 2113592d9..8d0439066 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -56,10 +56,10 @@ type Commitment struct { // IsAttested contains a flag indicating if we have received attestations for this Commitment. IsAttested reactive.Event - // IsSynced contains a flag indicating if we have downloaded all objects enclosed in this Commitment. + // IsSynced contains a flag that indicates if a Commitment was fully downloaded and processed. IsSynced reactive.Event - // IsCommittable contains a flag indicating if this Commitment is committable (we have received all blocks and all attestations). + // IsCommittable contains a flag that indicates if a Commitment is ready to be committed by the warp sync process. IsCommittable reactive.Event // IsVerified contains a flag indicating if this Commitment is verified (we produced this Commitment ourselves by From 5fb3535b883e9b354345026310c9f436ade6d335 Mon Sep 17 00:00:00 2001 From: Hans Moog <3293976+hmoog@users.noreply.github.com> Date: Mon, 4 Dec 2023 02:00:41 +0100 Subject: [PATCH 52/55] Refactor: updated comments --- pkg/protocol/commitment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index 8d0439066..ec8a8804d 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -159,7 +159,7 @@ func (c *Commitment) initDerivedProperties() (shutdown func()) { // mark commitments that are marked as root as verified c.IsVerified.InheritFrom(c.IsRoot), - // mark commitments that are marked as verified as attested and fully booked + // mark commitments that are marked as verified as attested and synced c.IsAttested.InheritFrom(c.IsVerified), c.IsSynced.InheritFrom(c.IsVerified), From 9576fd68cc80714a97b88367cdbb3e1c21b91a9a Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 4 Dec 2023 19:54:12 +0800 Subject: [PATCH 53/55] Do not mark blocks as root blocks when warp syncing as that is done automatically within the warp sync window --- pkg/protocol/protocol_warp_sync.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index c2a560d56..e6522ffa4 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -229,21 +229,6 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return } - - // TODO: maybe we don't need to set root blocks as we leave a window for warp sync now. - for slotCommitmentID, blockIDs := range blockIDsBySlotCommitment { - for _, blockID := range blockIDs { - block, exists := targetEngine.BlockCache.Block(blockID) - if !exists { // this should never happen as we just booked these blocks in this slot. - continue - } - - // We need to make sure that we add all blocks as root blocks because we don't know which blocks are root blocks without - // blocks from future slots. We're committing the current slot which then leads to the eviction of the blocks from the - // block cache and thus if not root blocks no block in the next slot can become solid. - targetEngine.EvictionState.AddRootBlock(block.ID(), slotCommitmentID) - } - } }) } From d8d6430f194ba0f54d114d86f41e3a6ff731515e Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:16:45 +0800 Subject: [PATCH 54/55] Address review comments --- pkg/protocol/commitment.go | 4 ++-- pkg/protocol/protocol_warp_sync.go | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/protocol/commitment.go b/pkg/protocol/commitment.go index ec8a8804d..6c2bb4584 100644 --- a/pkg/protocol/commitment.go +++ b/pkg/protocol/commitment.go @@ -71,7 +71,7 @@ type Commitment struct { IsAboveLatestVerifiedCommitment reactive.Variable[bool] // ReplayDroppedBlocks contains a flag indicating if we should replay the blocks that were dropped while the - //Commitment was pending. + // Commitment was pending. ReplayDroppedBlocks reactive.Variable[bool] // IsEvicted contains a flag indicating if this Commitment was evicted from the Protocol. @@ -274,7 +274,7 @@ func (c *Commitment) deriveRequestAttestations(chain *Chain, parent *Commitment) } // deriveWarpSyncBlocks derives the WarpSyncBlocks flag of this Commitment which is true if our Chain is requesting -// warp sync, and we are the directly above the latest verified Commitment. +// warp sync, and we are the directly above the latest commitment that is synced (has downloaded everything). func (c *Commitment) deriveWarpSyncBlocks(chain *Chain, parent *Commitment) func() { return c.WarpSyncBlocks.DeriveValueFrom(reactive.NewDerivedVariable3(func(_ bool, warpSyncMode bool, parentIsSynced bool, isSynced bool) bool { return warpSyncMode && parentIsSynced && !isSynced diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index e6522ffa4..ac1326b84 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -167,11 +167,6 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo return blocksToWarpSync } - // make sure the engine is clean and requires a warp-sync before we start processing the blocks - if targetEngine.Storage.Settings().LatestCommitment().ID().Slot() > commitmentID.Slot() { - return blocksToWarpSync - } - // Once all blocks are booked we // 1. Mark all transactions as accepted // 2. Mark all blocks as accepted From ac6a99359b8a1dbd092ca78e66ff4c86f705950a Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:59:47 +0800 Subject: [PATCH 55/55] Fix deadlock --- pkg/protocol/protocol_warp_sync.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/protocol/protocol_warp_sync.go b/pkg/protocol/protocol_warp_sync.go index ac1326b84..e42f0c6e8 100644 --- a/pkg/protocol/protocol_warp_sync.go +++ b/pkg/protocol/protocol_warp_sync.go @@ -210,20 +210,24 @@ func (w *WarpSyncProtocol) ProcessResponse(commitmentID iotago.CommitmentID, blo } allBlocksNotarized.OnTrigger(func() { - // 3. Force commitment of the slot - producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) - if err != nil { - w.protocol.LogError("failed to force commitment", "commitmentID", commitmentID, "err", err) + // This needs to happen in a separate worker since the trigger for block notarized while the lock in + // the notarization is still held. + w.workerPool.Submit(func() { + // 3. Force commitment of the slot + producedCommitment, err := targetEngine.Notarization.ForceCommit(commitmentID.Slot()) + if err != nil { + w.protocol.LogError("failed to force commitment", "commitmentID", commitmentID, "err", err) - return - } + return + } - // 4. Verify that the produced commitment is the same as the initially requested one - if producedCommitment.ID() != commitmentID { - w.protocol.LogError("commitment does not match", "expectedCommitmentID", commitmentID, "producedCommitmentID", producedCommitment.ID()) + // 4. Verify that the produced commitment is the same as the initially requested one + if producedCommitment.ID() != commitmentID { + w.protocol.LogError("commitment does not match", "expectedCommitmentID", commitmentID, "producedCommitmentID", producedCommitment.ID()) - return - } + return + } + }) }) }