From eeca6843fab6b76155a7d89dce2543c3bdff6c7d Mon Sep 17 00:00:00 2001 From: MariusC Date: Wed, 29 Jan 2025 15:53:02 +0200 Subject: [PATCH 01/20] FEAT: RestoreBlockIntoPools removes last cross notarized headers based on extended headers --- process/block/sovereignChainBlock.go | 33 ++++++++++++++++++- .../track/sovereignChainShardBlockTrack.go | 10 ++++++ testscommon/extendedShardHeaderTrackerStub.go | 16 +++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 9e5597bd858..36be981679d 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -33,6 +33,8 @@ var rootHash = "uncomputed root hash" type extendedShardHeaderTrackHandler interface { ComputeLongestExtendedShardChainFromLastNotarized() ([]data.HeaderHandler, [][]byte, error) IsGenesisLastCrossNotarizedHeader() bool + RemoveLastCrossNotarizedHeaders() + RemoveLastSelfNotarizedHeaders() } type extendedShardHeaderRequestHandler interface { @@ -58,6 +60,8 @@ type sovereignChainBlockProcessor struct { epochEconomics process.EndOfEpochEconomics mainChainNotarizationStartRound uint64 + + ct int } // ArgsSovereignChainBlockProcessor is a struct placeholder for args needed to create a new sovereign chain block processor @@ -717,6 +721,13 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header return nil, nil, process.ErrNilHaveTimeHandler } + scbp.ct++ + + //if scbp.ct == 35 { + // scbp.RestoreBlockIntoPools(headerHandler, bodyHandler) + // return headerHandler, bodyHandler, nil + //} + scbp.processStatusHandler.SetBusy("sovereignChainBlockProcessor.ProcessBlock") defer scbp.processStatusHandler.SetIdle() @@ -1944,7 +1955,27 @@ func (scbp *sovereignChainBlockProcessor) saveSovereignMetricsForCommittedBlock( // RestoreBlockIntoPools restores block into pools func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.HeaderHandler, body data.BodyHandler) error { scbp.restoreBlockBody(header, body) - scbp.blockTracker.RemoveLastNotarizedHeaders() + // THIS SHOULD BE ERROR + + // 2. restore cross chain txs/incoming scr???? + // 3. restore extended shard headers from hashes??? like RestoreBlockIntoPools from shard + + sovChainHdr, castOk := header.(data.SovereignChainHeaderHandler) + if !castOk { + return fmt.Errorf("%w in sovereignChainBlockProcessor.RestoreBlockIntoPools", errors.ErrWrongTypeAssertion) + } + + scbp.extendedShardHeaderTracker.RemoveLastSelfNotarizedHeaders() + numOfNotarizedExtendedHeaders := len(sovChainHdr.GetExtendedShardHeaderHashes()) + log.Debug("sovereignChainBlockProcessor.RestoreBlockIntoPools", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) + if numOfNotarizedExtendedHeaders == 0 { + return nil + } + + for i := 0; i < numOfNotarizedExtendedHeaders; i++ { + scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() + } + return nil } diff --git a/process/track/sovereignChainShardBlockTrack.go b/process/track/sovereignChainShardBlockTrack.go index 07f7792b7fa..e4d149eff7c 100644 --- a/process/track/sovereignChainShardBlockTrack.go +++ b/process/track/sovereignChainShardBlockTrack.go @@ -255,3 +255,13 @@ func (scsbt *sovereignChainShardBlockTrack) IsShardStuck(_ uint32) bool { // ComputeCrossInfo does nothing for sovereign chain func (scsbt *sovereignChainShardBlockTrack) ComputeCrossInfo(_ []data.HeaderHandler) { } + +// RemoveLastCrossNotarizedHeaders removes last cross notarized headers from tracker list +func (scsbt *sovereignChainShardBlockTrack) RemoveLastCrossNotarizedHeaders() { + scsbt.crossNotarizer.RemoveLastNotarizedHeader() +} + +// RemoveLastSelfNotarizedHeaders removes last self notarized headers from tracker list +func (scsbt *sovereignChainShardBlockTrack) RemoveLastSelfNotarizedHeaders() { + scsbt.selfNotarizer.RemoveLastNotarizedHeader() +} diff --git a/testscommon/extendedShardHeaderTrackerStub.go b/testscommon/extendedShardHeaderTrackerStub.go index 56345087bec..685ab097cfd 100644 --- a/testscommon/extendedShardHeaderTrackerStub.go +++ b/testscommon/extendedShardHeaderTrackerStub.go @@ -8,6 +8,8 @@ import ( type ExtendedShardHeaderTrackerStub struct { BlockTrackerStub ComputeLongestExtendedShardChainFromLastNotarizedCalled func() ([]data.HeaderHandler, [][]byte, error) + RemoveLastCrossNotarizedHeadersCalled func() + RemoveLastSelfNotarizedHeadersCalled func() } // ComputeLongestExtendedShardChainFromLastNotarized - @@ -18,6 +20,20 @@ func (eshts *ExtendedShardHeaderTrackerStub) ComputeLongestExtendedShardChainFro return nil, nil, nil } +// RemoveLastCrossNotarizedHeaders - +func (eshts *ExtendedShardHeaderTrackerStub) RemoveLastCrossNotarizedHeaders() { + if eshts.RemoveLastCrossNotarizedHeadersCalled != nil { + eshts.RemoveLastCrossNotarizedHeadersCalled() + } +} + +// RemoveLastSelfNotarizedHeaders - +func (eshts *ExtendedShardHeaderTrackerStub) RemoveLastSelfNotarizedHeaders() { + if eshts.RemoveLastSelfNotarizedHeadersCalled != nil { + eshts.RemoveLastSelfNotarizedHeadersCalled() + } +} + // IsGenesisLastCrossNotarizedHeader - func (eshts *ExtendedShardHeaderTrackerStub) IsGenesisLastCrossNotarizedHeader() bool { return false From 1c9fd9acc8404ebc838de71df8879721ab2698df Mon Sep 17 00:00:00 2001 From: MariusC Date: Wed, 29 Jan 2025 17:50:13 +0200 Subject: [PATCH 02/20] FEAT: Add restoreExtendedHeaderIntoPool in RestoreBlockIntoPools --- process/block/sovereignChainBlock.go | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 36be981679d..49c760aba0a 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -1959,12 +1959,18 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head // 2. restore cross chain txs/incoming scr???? // 3. restore extended shard headers from hashes??? like RestoreBlockIntoPools from shard + // 4. what to do if we forever loose an extended header? we should request it from notifier? sovChainHdr, castOk := header.(data.SovereignChainHeaderHandler) if !castOk { return fmt.Errorf("%w in sovereignChainBlockProcessor.RestoreBlockIntoPools", errors.ErrWrongTypeAssertion) } + err := scbp.restoreExtendedHeaderIntoPool(sovChainHdr.GetExtendedShardHeaderHashes()) + if err != nil { + return err + } + scbp.extendedShardHeaderTracker.RemoveLastSelfNotarizedHeaders() numOfNotarizedExtendedHeaders := len(sovChainHdr.GetExtendedShardHeaderHashes()) log.Debug("sovereignChainBlockProcessor.RestoreBlockIntoPools", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) @@ -1979,6 +1985,56 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head return nil } +func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extendedShardHeaderHashes [][]byte) error { + headersPool := scbp.dataPool.Headers() + + for _, extendedHdrHash := range extendedShardHeaderHashes { + extendedHdr, errNotCritical := process.GetExtendedShardHeaderFromStorage(extendedHdrHash, scbp.marshalizer, scbp.store) + if errNotCritical != nil { + log.Debug("extended header is not fully processed yet and not committed in ExtendedShardHeadersUnit", + "hash", extendedHdrHash) + continue + } + + headersPool.AddHeader(extendedHdrHash, extendedHdr) + + extendedHdrStorer, err := scbp.store.GetStorer(dataRetriever.ExtendedShardHeadersUnit) + if err != nil { + log.Error("unable to get storage unit", + "unit", dataRetriever.ExtendedShardHeadersUnit.String()) + return err + } + + err = extendedHdrStorer.Remove(extendedHdrHash) + if err != nil { + log.Error("unable to remove hash from MetaBlockUnit", + "hash", extendedHdrHash) + return err + } + + nonceToByteSlice := scbp.uint64Converter.ToByteSlice(extendedHdr.GetNonce()) + extendedHdrNonceHashStorer, err := scbp.store.GetStorer(dataRetriever.ExtendedShardHeadersNonceHashDataUnit) + if err != nil { + log.Error("unable to get storage unit", + "unit", dataRetriever.ExtendedShardHeadersNonceHashDataUnit.String()) + return err + } + + errNotCritical = extendedHdrNonceHashStorer.Remove(nonceToByteSlice) + if errNotCritical != nil { + log.Debug("extendedHdrNonceHashStorer.Remove error not critical", + "error", errNotCritical.Error()) + } + + log.Debug("extended block has been restored successfully", + "round", extendedHdr.GetRound(), + "nonce", extendedHdr.GetNonce(), + "hash", extendedHdr) + } + + return nil +} + // RevertStateToBlock reverts state in tries func (scbp *sovereignChainBlockProcessor) RevertStateToBlock(header data.HeaderHandler, rootHash []byte) error { rootHashHolder := holders.NewDefaultRootHashesHolder(rootHash) From 1d8a36672e11c5f188f56cb70ce53f5b7adafc36 Mon Sep 17 00:00:00 2001 From: MariusC Date: Fri, 31 Jan 2025 11:35:04 +0200 Subject: [PATCH 03/20] FEAT: Reset last cross notarized from prev header --- process/block/sovereignChainBlock.go | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 49c760aba0a..900b21d7a8c 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -896,8 +896,15 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity() er "extendedShardHeader nonce", extendedShardHdr.GetNonce(), "extendedShardHeader round", extendedShardHdr.GetRound(), ) + err = scbp.headerValidator.IsHeaderConstructionValid(extendedShardHdr, lastCrossNotarizedHeader) if err != nil { + log.Error("sovereignChainBlockProcessor.checkExtendedShardHeadersValidity", "error", err, + "extendedShardHdr.Nonce", extendedShardHdr.GetNonce(), + "lastCrossNotarizedHeader.Nonce", lastCrossNotarizedHeader.GetNonce(), + "extendedShardHdr.Round", extendedShardHdr.GetRound(), + "lastCrossNotarizedHeader.Round", lastCrossNotarizedHeader.GetRound(), + ) return fmt.Errorf("%w : checkExtendedShardHeadersValidity -> isHdrConstructionValid", err) } @@ -1982,6 +1989,31 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() } + return scbp.resetLastCrossNotarizedFromPrevHdr(sovChainHdr) +} + +func (scbp *sovereignChainBlockProcessor) resetLastCrossNotarizedFromPrevHdr(sovChainHdr data.SovereignChainHeaderHandler) error { + prevHdrHash := sovChainHdr.GetPrevHash() + prevHdr, err := scbp.dataPool.Headers().GetHeaderByHash(prevHdrHash) + if err != nil { + return err + } + + prevSovChainHdr, castOk := prevHdr.(data.SovereignChainHeaderHandler) + if !castOk { + return fmt.Errorf("%w in sovereignChainBlockProcessor.resetLastCrossNotarizedFromPrevHdr", errors.ErrWrongTypeAssertion) + } + + numOfNotarizedExtendedHeaders := len(prevSovChainHdr.GetExtendedShardHeaderHashes()) + log.Debug("sovereignChainBlockProcessor.resetLastCrossNotarizedFromPrevHdr", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) + if numOfNotarizedExtendedHeaders <= 1 { + return nil + } + + for i := 0; i < numOfNotarizedExtendedHeaders; i++ { + scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() + } + return nil } From 69b3e4f368a8938fdfb6b5bac57f72519e82cf39 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 10:40:49 +0200 Subject: [PATCH 04/20] FEAT: Try restoreExtendedHeaderIntoPool for previous header --- process/block/sovereignChainBlock.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 900b21d7a8c..795466bf167 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -1989,6 +1989,8 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() } + // restore last cross notarized at previous ???? + return scbp.resetLastCrossNotarizedFromPrevHdr(sovChainHdr) } @@ -2006,8 +2008,10 @@ func (scbp *sovereignChainBlockProcessor) resetLastCrossNotarizedFromPrevHdr(sov numOfNotarizedExtendedHeaders := len(prevSovChainHdr.GetExtendedShardHeaderHashes()) log.Debug("sovereignChainBlockProcessor.resetLastCrossNotarizedFromPrevHdr", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) - if numOfNotarizedExtendedHeaders <= 1 { - return nil + + err = scbp.restoreExtendedHeaderIntoPool(prevSovChainHdr.GetExtendedShardHeaderHashes()) + if err != nil { + return err } for i := 0; i < numOfNotarizedExtendedHeaders; i++ { From 82995486a445a52a94d15082fd45c84f72c99063 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 11:20:20 +0200 Subject: [PATCH 05/20] FEAT: Do not restore headers from previous header --- process/block/sovereignChainBlock.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 795466bf167..cdd43b7e66b 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -1990,8 +1990,8 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head } // restore last cross notarized at previous ???? - - return scbp.resetLastCrossNotarizedFromPrevHdr(sovChainHdr) + return nil + // return scbp.resetLastCrossNotarizedFromPrevHdr(sovChainHdr) } func (scbp *sovereignChainBlockProcessor) resetLastCrossNotarizedFromPrevHdr(sovChainHdr data.SovereignChainHeaderHandler) error { From 0d3b23489dfeb66346915ab5415be8b4be237255 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 12:07:34 +0200 Subject: [PATCH 06/20] FEAT: Do not RemoveLastCrossNotarizedHeaders from current header --- process/block/sovereignChainBlock.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index cdd43b7e66b..fd2ae829c23 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -1985,9 +1985,9 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head return nil } - for i := 0; i < numOfNotarizedExtendedHeaders; i++ { - scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() - } + //for i := 0; i < numOfNotarizedExtendedHeaders; i++ { + // scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() + //} // restore last cross notarized at previous ???? return nil From 9c0e7d4f0df454bc0d5ffb184a13610243583487 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 12:08:50 +0200 Subject: [PATCH 07/20] FIX: Ext hdr hash in debug --- process/block/sovereignChainBlock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index fd2ae829c23..f6f39bf5b1c 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -2065,7 +2065,7 @@ func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extended log.Debug("extended block has been restored successfully", "round", extendedHdr.GetRound(), "nonce", extendedHdr.GetNonce(), - "hash", extendedHdr) + "hash", extendedHdrHash) } return nil From b9a7fbac04e31cbd442e9d0faf05b84fd95c2ebf Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 15:04:23 +0200 Subject: [PATCH 08/20] FEAT: Try to only sort extended headers used in block --- process/block/sovereignChainBlock.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index f6f39bf5b1c..8b7712adbe0 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -892,7 +892,7 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity() er } for _, extendedShardHdr := range extendedShardHdrs { - log.Trace("checkExtendedShardHeadersValidity", + log.Debug("checkExtendedShardHeadersValidity", "extendedShardHeader nonce", extendedShardHdr.GetNonce(), "extendedShardHeader round", extendedShardHdr.GetRound(), ) @@ -1140,7 +1140,14 @@ func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBloc scbp.hdrsForCurrBlock.mutHdrsForBlock.RLock() for _, headerInfo := range scbp.hdrsForCurrBlock.hdrHashAndInfo { - hdrsForCurrentBlock = append(hdrsForCurrentBlock, headerInfo.hdr) + log.Debug("sovereignChainBlockProcessor.sortExtendedShardHeadersForCurrentBlockByNonce", + "headerInfo.hdr.GetNonce()", headerInfo.hdr.GetNonce(), + "headerInfo.usedInBlock", headerInfo.usedInBlock, + ) + + if headerInfo.usedInBlock { + hdrsForCurrentBlock = append(hdrsForCurrentBlock, headerInfo.hdr) + } } scbp.hdrsForCurrBlock.mutHdrsForBlock.RUnlock() From 51aaae23fd45e5a5017338f08466f86b7f92c0c8 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 3 Feb 2025 16:27:00 +0200 Subject: [PATCH 09/20] FEAT: Check extended headers from within header itself --- process/block/sovereignChainBlock.go | 33 ++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 8b7712adbe0..bf30a062bca 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -810,7 +810,7 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header go scbp.checkAndRequestIfExtendedShardHeadersMissing() }() - err = scbp.checkExtendedShardHeadersValidity() + err = scbp.checkExtendedShardHeadersValidity(sovereignChainHeader) if err != nil { return nil, nil, err } @@ -860,7 +860,9 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header } // checkExtendedShardHeadersValidity checks if used extended shard headers are valid as construction -func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity() error { +func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity( + sovChainHeader data.SovereignChainHeaderHandler, +) error { lastCrossNotarizedHeader, _, err := scbp.blockTracker.GetLastCrossNotarizedHeader(core.MainChainShardId) if err != nil { return err @@ -871,7 +873,11 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity() er "lastCrossNotarizedHeader round", lastCrossNotarizedHeader.GetRound(), ) - extendedShardHdrs := scbp.sortExtendedShardHeadersForCurrentBlockByNonce() + extendedShardHdrs, err := scbp.sortExtendedShardHeadersForCurrentBlockByNonce(sovChainHeader) + if err != nil { + return err + } + if len(extendedShardHdrs) == 0 { return nil } @@ -1135,11 +1141,20 @@ func (scbp *sovereignChainBlockProcessor) getExtraMissingNoncesToRequest(_ data. return make([]uint64, 0) } -func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBlockByNonce() []data.HeaderHandler { +func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBlockByNonce( + sovChainHeader data.SovereignChainHeaderHandler, +) ([]data.HeaderHandler, error) { hdrsForCurrentBlock := make([]data.HeaderHandler, 0) scbp.hdrsForCurrBlock.mutHdrsForBlock.RLock() - for _, headerInfo := range scbp.hdrsForCurrBlock.hdrHashAndInfo { + for _, extendedShardHeaderHash := range sovChainHeader.GetExtendedShardHeaderHashes() { + headerInfo, found := scbp.hdrsForCurrBlock.hdrHashAndInfo[string(extendedShardHeaderHash)] + if !found { + scbp.hdrsForCurrBlock.mutHdrsForBlock.RUnlock() + return nil, fmt.Errorf("%w : in sovereignChainBlockProcessor.sortExtendedShardHeadersForCurrentBlockByNonce = %s", + process.ErrMissingHeader, extendedShardHeaderHash) + } + log.Debug("sovereignChainBlockProcessor.sortExtendedShardHeadersForCurrentBlockByNonce", "headerInfo.hdr.GetNonce()", headerInfo.hdr.GetNonce(), "headerInfo.usedInBlock", headerInfo.usedInBlock, @@ -1153,7 +1168,7 @@ func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBloc process.SortHeadersByNonce(hdrsForCurrentBlock) - return hdrsForCurrentBlock + return hdrsForCurrentBlock, nil } func (scbp *sovereignChainBlockProcessor) verifyCrossShardMiniBlockDstMe(sovereignChainHeader data.SovereignChainHeaderHandler) error { @@ -1992,9 +2007,9 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head return nil } - //for i := 0; i < numOfNotarizedExtendedHeaders; i++ { - // scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() - //} + for i := 0; i < numOfNotarizedExtendedHeaders; i++ { + scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() + } // restore last cross notarized at previous ???? return nil From e339b2fddbfb6747465c9302f428a4262dcbdfcc Mon Sep 17 00:00:00 2001 From: MariusC Date: Tue, 4 Feb 2025 11:45:01 +0200 Subject: [PATCH 10/20] FIX: On restoreExtendedHeaderIntoPool use AddHeaderInShard --- process/block/sovereignChainBlock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index bf30a062bca..9e16df8d23f 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -2054,7 +2054,7 @@ func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extended continue } - headersPool.AddHeader(extendedHdrHash, extendedHdr) + headersPool.AddHeaderInShard(extendedHdrHash, extendedHdr, core.MainChainShardId) extendedHdrStorer, err := scbp.store.GetStorer(dataRetriever.ExtendedShardHeadersUnit) if err != nil { From e01a637bf3294a001f233587d1df052871e120d6 Mon Sep 17 00:00:00 2001 From: MariusC Date: Tue, 4 Feb 2025 12:39:26 +0200 Subject: [PATCH 11/20] FEAT: Try to delete extended headers by hash --- process/block/sovereignChainBlock.go | 7 +++++-- process/track/blockNotarizer.go | 15 +++++++++++++++ process/track/sovereignChainShardBlockTrack.go | 10 ++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 9e16df8d23f..6475d71fdd4 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -35,6 +35,7 @@ type extendedShardHeaderTrackHandler interface { IsGenesisLastCrossNotarizedHeader() bool RemoveLastCrossNotarizedHeaders() RemoveLastSelfNotarizedHeaders() + RemoveLastCrossNotarizedHeaderByHash(hash []byte) } type extendedShardHeaderRequestHandler interface { @@ -2007,8 +2008,10 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head return nil } - for i := 0; i < numOfNotarizedExtendedHeaders; i++ { - scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() + for i := numOfNotarizedExtendedHeaders - 1; i > 0; i-- { + extendedHdrHash := sovChainHdr.GetExtendedShardHeaderHashes()[i] + log.Debug("CALLING DELETE ", "extendedHdrHash", extendedHdrHash) + scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaderByHash(extendedHdrHash) } // restore last cross notarized at previous ???? diff --git a/process/track/blockNotarizer.go b/process/track/blockNotarizer.go index 0d6b581fd6e..c9d6326ed69 100644 --- a/process/track/blockNotarizer.go +++ b/process/track/blockNotarizer.go @@ -1,6 +1,7 @@ package track import ( + "bytes" "sort" "sync" @@ -9,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/sharding" ) @@ -248,6 +250,19 @@ func (bn *blockNotarizer) RemoveLastNotarizedHeader() { bn.mutNotarizedHeaders.Unlock() } +// RemoveLastNotarizedHeaderByHash removes last notarized header by hash +func (bn *blockNotarizer) RemoveLastNotarizedHeaderByHash(hash []byte) { + bn.mutNotarizedHeaders.Lock() + for shardID := range bn.notarizedHeaders { + notarizedHeadersCount := len(bn.notarizedHeaders[shardID]) + if notarizedHeadersCount > 1 && bytes.Equal(hash, bn.notarizedHeaders[shardID][notarizedHeadersCount-1].Hash) { + bn.notarizedHeaders[shardID] = bn.notarizedHeaders[shardID][:notarizedHeadersCount-1] + log.Debug("DELETED ", "extendedHdrHash", hash) + } + } + bn.mutNotarizedHeaders.Unlock() +} + // RestoreNotarizedHeadersToGenesis restores all notarized headers from each shard to the genesis value (nonce 0) func (bn *blockNotarizer) RestoreNotarizedHeadersToGenesis() { bn.mutNotarizedHeaders.Lock() diff --git a/process/track/sovereignChainShardBlockTrack.go b/process/track/sovereignChainShardBlockTrack.go index e4d149eff7c..d6ebd479645 100644 --- a/process/track/sovereignChainShardBlockTrack.go +++ b/process/track/sovereignChainShardBlockTrack.go @@ -261,6 +261,16 @@ func (scsbt *sovereignChainShardBlockTrack) RemoveLastCrossNotarizedHeaders() { scsbt.crossNotarizer.RemoveLastNotarizedHeader() } +type extendedCrossNotarizer interface { + blockNotarizerHandler + RemoveLastNotarizedHeaderByHash(hash []byte) +} + +// RemoveLastCrossNotarizedHeaderByHash removes last cross notarized header by hash, if exists +func (scsbt *sovereignChainShardBlockTrack) RemoveLastCrossNotarizedHeaderByHash(hash []byte) { + scsbt.crossNotarizer.(extendedCrossNotarizer).RemoveLastNotarizedHeaderByHash(hash) +} + // RemoveLastSelfNotarizedHeaders removes last self notarized headers from tracker list func (scsbt *sovereignChainShardBlockTrack) RemoveLastSelfNotarizedHeaders() { scsbt.selfNotarizer.RemoveLastNotarizedHeader() From 727ac7a01e3ff6d6373570c283c3aa16eeecce00 Mon Sep 17 00:00:00 2001 From: MariusC Date: Tue, 4 Feb 2025 12:48:22 +0200 Subject: [PATCH 12/20] FIX: For iteration extended --- process/block/sovereignChainBlock.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 6475d71fdd4..9f4e0d654d5 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -724,8 +724,15 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header scbp.ct++ - //if scbp.ct == 35 { - // scbp.RestoreBlockIntoPools(headerHandler, bodyHandler) + //if scbp.ct > 35 { + // prevHsh := headerHandler.GetPrevHash() + // + // prevHdr, _ := scbp.dataPool.Headers().GetHeaderByHash(prevHsh) + // + // if prevHdr != nil && len(prevHdr.(data.SovereignChainHeaderHandler).GetExtendedShardHeaderHashes()) > 0 { + // scbp.RestoreBlockIntoPools(prevHdr, bodyHandler) + // } + // // return headerHandler, bodyHandler, nil //} @@ -2008,7 +2015,7 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head return nil } - for i := numOfNotarizedExtendedHeaders - 1; i > 0; i-- { + for i := numOfNotarizedExtendedHeaders - 1; i >= 0; i-- { extendedHdrHash := sovChainHdr.GetExtendedShardHeaderHashes()[i] log.Debug("CALLING DELETE ", "extendedHdrHash", extendedHdrHash) scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaderByHash(extendedHdrHash) From 357dc06d0d43db1a59f7a6df821b92e241e96579 Mon Sep 17 00:00:00 2001 From: MariusC Date: Thu, 6 Feb 2025 13:11:18 +0200 Subject: [PATCH 13/20] FEAT: Initial test TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork --- process/block/sovereignChainBlock_test.go | 167 +++++++++++++++++++++- 1 file changed, 165 insertions(+), 2 deletions(-) diff --git a/process/block/sovereignChainBlock_test.go b/process/block/sovereignChainBlock_test.go index 508388a9892..bc4798b0946 100644 --- a/process/block/sovereignChainBlock_test.go +++ b/process/block/sovereignChainBlock_test.go @@ -1,22 +1,37 @@ package block_test import ( + "math/big" "testing" "time" + "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-core-go/marshal" + "github.com/stretchr/testify/assert" + + "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/requestHandlers" "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" blproc "github.com/multiversx/mx-chain-go/process/block" + "github.com/multiversx/mx-chain-go/process/coordinator" + "github.com/multiversx/mx-chain-go/process/factory/shard" + shardData "github.com/multiversx/mx-chain-go/process/factory/shard/data" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/track" + "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" + commonMock "github.com/multiversx/mx-chain-go/testscommon/common" dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/economicsmocks" + "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" + "github.com/multiversx/mx-chain-go/testscommon/processMocks" "github.com/multiversx/mx-chain-go/testscommon/sovereign" - "github.com/multiversx/mx-chain-go/testscommon/storage" + storageStub "github.com/multiversx/mx-chain-go/testscommon/storage" + storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" @@ -83,7 +98,7 @@ func CreateSovereignChainShardTrackerMockArguments() track.ArgShardTracker { RequestHandler: &testscommon.ExtendedShardHeaderRequestHandlerStub{}, RoundHandler: &testscommon.RoundHandlerMock{}, ShardCoordinator: &testscommon.ShardsCoordinatorMock{}, - Store: &storage.ChainStorerStub{}, + Store: &storageStub.ChainStorerStub{}, StartHeaders: createGenesisBlocks(&testscommon.ShardsCoordinatorMock{NoShards: 1}), PoolsHolder: dataRetrieverMock.NewPoolsHolderMock(), WhitelistHandler: &testscommon.WhiteListHandlerStub{}, @@ -343,4 +358,152 @@ func TestSovereignChainBlockProcessor_createAndSetOutGoingMiniBlock(t *testing.T require.Equal(t, expectedSovChainHeader, sovChainHdr) } +func createTxCoordinator( + store dataRetriever.StorageService, + marshaller marshal.Marshalizer, + Hasher hashing.Hasher, + dataPool dataRetriever.PoolsHolder, +) process.TransactionCoordinator { + args := shardData.ArgPreProcessorsContainerFactory{ + ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), + Store: store, + Marshaller: marshaller, + Hasher: Hasher, + DataPool: dataPool, + PubkeyConverter: createMockPubkeyConverter(), + Accounts: initAccountsMock(), + RequestHandler: &testscommon.RequestHandlerStub{}, + TxProcessor: &testscommon.TxProcessorMock{}, + ScProcessor: &testscommon.SCProcessorMock{}, + ScResultProcessor: &testscommon.SmartContractResultsProcessorMock{}, + RewardsTxProcessor: &testscommon.RewardTxProcessorMock{}, + EconomicsFee: &economicsmocks.EconomicsHandlerStub{}, + GasHandler: &testscommon.GasHandlerStub{}, + BlockTracker: &mock.BlockTrackerMock{}, + BlockSizeComputation: &testscommon.BlockSizeComputationStub{}, + BalanceComputation: &testscommon.BalanceComputationStub{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + TxTypeHandler: &testscommon.TxTypeHandlerMock{}, + ScheduledTxsExecutionHandler: &testscommon.ScheduledTxsExecutionStub{}, + ProcessedMiniBlocksTracker: &testscommon.ProcessedMiniBlocksTrackerStub{}, + TxExecutionOrderHandler: &commonMock.TxExecutionOrderHandlerStub{}, + RunTypeComponents: processMocks.NewRunTypeComponentsStub(), + } + factory, _ := shard.NewPreProcessorsContainerFactory(args) + container, _ := factory.Create() + + argsTransactionCoordinator := createMockTransactionCoordinatorArguments(initAccountsMock(), dataPool, container) + tc, _ := coordinator.NewTransactionCoordinator(argsTransactionCoordinator) + return tc +} + +func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing.T) { + t.Parallel() + + txHash := []byte("tx hash 1") + + datapool := dataRetrieverMock.NewPoolsHolderMock() + marshalizerMock := &mock.MarshalizerMock{} + hasherMock := &mock.HasherStub{} + + body := &block.Body{} + tx := &transaction.Transaction{ + Nonce: 1, + Value: big.NewInt(0), + } + buffTx, _ := marshalizerMock.Marshal(tx) + + store := &storageStubs.ChainStorerStub{ + GetAllCalled: func(unitType dataRetriever.UnitType, keys [][]byte) (map[string][]byte, error) { + m := make(map[string][]byte) + m[string(txHash)] = buffTx + return m, nil + }, + } + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + dataComponents.DataPool = datapool + dataComponents.Storage = store + coreComponents.Hash = hasherMock + coreComponents.IntMarsh = marshalizerMock + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.TxCoordinator = createTxCoordinator(store, marshalizerMock, hasherMock, datapool) + + shardArguments := CreateSovereignChainShardTrackerMockArguments() + sbt, _ := track.NewShardBlockTrack(shardArguments) + arguments.BlockTracker, _ = track.NewSovereignChainShardBlockTrack(sbt) + rrh, _ := requestHandlers.NewResolverRequestHandler( + &dataRetrieverMock.RequestersFinderStub{}, + &mock.RequestedItemsHandlerStub{}, + &testscommon.WhiteListHandlerStub{}, + 1, + 0, + time.Second, + ) + arguments.RequestHandler, _ = requestHandlers.NewSovereignResolverRequestHandler(rrh) + + sp, _ := blproc.NewShardProcessor(arguments) + + argsSovProc := createSovChainBlockProcessorArgs() + argsSovProc.ShardProcessor = sp + sovBlockProc, _ := blproc.NewSovereignChainBlockProcessor(argsSovProc) + + txHashes := make([][]byte, 0) + txHashes = append(txHashes, txHash) + miniblock := block.MiniBlock{ + ReceiverShardID: 0, + SenderShardID: 1, + TxHashes: txHashes, + } + body.MiniBlocks = append(body.MiniBlocks, &miniblock) + + miniblockHash := []byte("mini block hash 1") + hasherMock.ComputeCalled = func(s string) []byte { + return miniblockHash + } + + extendedHeaderHash := []byte("meta block hash 1") + extendedHeader := &block.ShardHeaderExtended{ + Header: &block.HeaderV2{ + ScheduledRootHash: []byte("root hash"), + }, + IncomingMiniBlocks: nil, + IncomingEvents: nil, + } + datapool.Headers().AddHeader(extendedHeaderHash, extendedHeader) + + store.GetStorerCalled = func(unitType dataRetriever.UnitType) (storage.Storer, error) { + return &storageStubs.StorerStub{ + RemoveCalled: func(key []byte) error { + return nil + }, + GetCalled: func(key []byte) ([]byte, error) { + return marshalizerMock.Marshal(extendedHeader) + }, + }, nil + } + + miniBlockHeader := block.MiniBlockHeader{ + Hash: miniblockHash, + SenderShardID: miniblock.SenderShardID, + ReceiverShardID: miniblock.ReceiverShardID, + } + + sovHdr := &block.SovereignChainHeader{ + Header: &block.Header{ + MiniBlockHeaders: []block.MiniBlockHeader{miniBlockHeader}, + }, + ExtendedShardHeaderHashes: [][]byte{extendedHeaderHash}, + } + + err := sovBlockProc.RestoreBlockIntoPools(sovHdr, body) + assert.Nil(t, err) + + miniblockFromPool, _ := datapool.MiniBlocks().Get(miniblockHash) + txFromPool, _ := datapool.Transactions().SearchFirstData(txHash) + assert.Nil(t, err) + assert.Equal(t, &miniblock, miniblockFromPool) + assert.Equal(t, tx, txFromPool) +} + //TODO: More unit tests should be added. Created PR https://multiversxlabs.atlassian.net/browse/MX-14149 From cc244b9bf4f2738d01813cdfcc55debc65971f2e Mon Sep 17 00:00:00 2001 From: MariusC Date: Thu, 6 Feb 2025 16:05:13 +0200 Subject: [PATCH 14/20] CLN: General cleanup solution --- process/block/sovereignChainBlock.go | 66 ++----------------- process/track/blockNotarizer.go | 14 ---- .../track/sovereignChainShardBlockTrack.go | 10 --- 3 files changed, 6 insertions(+), 84 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 9f4e0d654d5..7bb0a197294 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -35,7 +35,6 @@ type extendedShardHeaderTrackHandler interface { IsGenesisLastCrossNotarizedHeader() bool RemoveLastCrossNotarizedHeaders() RemoveLastSelfNotarizedHeaders() - RemoveLastCrossNotarizedHeaderByHash(hash []byte) } type extendedShardHeaderRequestHandler interface { @@ -61,8 +60,6 @@ type sovereignChainBlockProcessor struct { epochEconomics process.EndOfEpochEconomics mainChainNotarizationStartRound uint64 - - ct int } // ArgsSovereignChainBlockProcessor is a struct placeholder for args needed to create a new sovereign chain block processor @@ -722,20 +719,6 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header return nil, nil, process.ErrNilHaveTimeHandler } - scbp.ct++ - - //if scbp.ct > 35 { - // prevHsh := headerHandler.GetPrevHash() - // - // prevHdr, _ := scbp.dataPool.Headers().GetHeaderByHash(prevHsh) - // - // if prevHdr != nil && len(prevHdr.(data.SovereignChainHeaderHandler).GetExtendedShardHeaderHashes()) > 0 { - // scbp.RestoreBlockIntoPools(prevHdr, bodyHandler) - // } - // - // return headerHandler, bodyHandler, nil - //} - scbp.processStatusHandler.SetBusy("sovereignChainBlockProcessor.ProcessBlock") defer scbp.processStatusHandler.SetIdle() @@ -1992,11 +1975,9 @@ func (scbp *sovereignChainBlockProcessor) saveSovereignMetricsForCommittedBlock( // RestoreBlockIntoPools restores block into pools func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.HeaderHandler, body data.BodyHandler) error { scbp.restoreBlockBody(header, body) - // THIS SHOULD BE ERROR - // 2. restore cross chain txs/incoming scr???? - // 3. restore extended shard headers from hashes??? like RestoreBlockIntoPools from shard - // 4. what to do if we forever loose an extended header? we should request it from notifier? + // TODO: MX-16507: check how/if to restore incoming scrs/cross chain txs once we have a testnet setup for it + // We should probably have something similar to (sp *shardProcessor) rollBackProcessedMiniBlocksInfo sovChainHdr, castOk := header.(data.SovereignChainHeaderHandler) if !castOk { @@ -2009,53 +1990,18 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head } scbp.extendedShardHeaderTracker.RemoveLastSelfNotarizedHeaders() + numOfNotarizedExtendedHeaders := len(sovChainHdr.GetExtendedShardHeaderHashes()) log.Debug("sovereignChainBlockProcessor.RestoreBlockIntoPools", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) if numOfNotarizedExtendedHeaders == 0 { return nil } - for i := numOfNotarizedExtendedHeaders - 1; i >= 0; i-- { - extendedHdrHash := sovChainHdr.GetExtendedShardHeaderHashes()[i] - log.Debug("CALLING DELETE ", "extendedHdrHash", extendedHdrHash) - scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaderByHash(extendedHdrHash) - } - - // restore last cross notarized at previous ???? - return nil - // return scbp.resetLastCrossNotarizedFromPrevHdr(sovChainHdr) -} - -func (scbp *sovereignChainBlockProcessor) resetLastCrossNotarizedFromPrevHdr(sovChainHdr data.SovereignChainHeaderHandler) error { - prevHdrHash := sovChainHdr.GetPrevHash() - prevHdr, err := scbp.dataPool.Headers().GetHeaderByHash(prevHdrHash) - if err != nil { - return err - } - - prevSovChainHdr, castOk := prevHdr.(data.SovereignChainHeaderHandler) - if !castOk { - return fmt.Errorf("%w in sovereignChainBlockProcessor.resetLastCrossNotarizedFromPrevHdr", errors.ErrWrongTypeAssertion) - } - - numOfNotarizedExtendedHeaders := len(prevSovChainHdr.GetExtendedShardHeaderHashes()) - log.Debug("sovereignChainBlockProcessor.resetLastCrossNotarizedFromPrevHdr", "numOfNotarizedExtendedHeaders", numOfNotarizedExtendedHeaders) - - err = scbp.restoreExtendedHeaderIntoPool(prevSovChainHdr.GetExtendedShardHeaderHashes()) - if err != nil { - return err - } - - for i := 0; i < numOfNotarizedExtendedHeaders; i++ { - scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() - } - + scbp.extendedShardHeaderTracker.RemoveLastCrossNotarizedHeaders() return nil } func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extendedShardHeaderHashes [][]byte) error { - headersPool := scbp.dataPool.Headers() - for _, extendedHdrHash := range extendedShardHeaderHashes { extendedHdr, errNotCritical := process.GetExtendedShardHeaderFromStorage(extendedHdrHash, scbp.marshalizer, scbp.store) if errNotCritical != nil { @@ -2064,7 +2010,7 @@ func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extended continue } - headersPool.AddHeaderInShard(extendedHdrHash, extendedHdr, core.MainChainShardId) + scbp.dataPool.Headers().AddHeaderInShard(extendedHdrHash, extendedHdr, core.MainChainShardId) extendedHdrStorer, err := scbp.store.GetStorer(dataRetriever.ExtendedShardHeadersUnit) if err != nil { @@ -2075,7 +2021,7 @@ func (scbp *sovereignChainBlockProcessor) restoreExtendedHeaderIntoPool(extended err = extendedHdrStorer.Remove(extendedHdrHash) if err != nil { - log.Error("unable to remove hash from MetaBlockUnit", + log.Error("unable to remove hash from ExtendedShardHeadersUnit", "hash", extendedHdrHash) return err } diff --git a/process/track/blockNotarizer.go b/process/track/blockNotarizer.go index c9d6326ed69..6520d3ff810 100644 --- a/process/track/blockNotarizer.go +++ b/process/track/blockNotarizer.go @@ -1,7 +1,6 @@ package track import ( - "bytes" "sort" "sync" @@ -250,19 +249,6 @@ func (bn *blockNotarizer) RemoveLastNotarizedHeader() { bn.mutNotarizedHeaders.Unlock() } -// RemoveLastNotarizedHeaderByHash removes last notarized header by hash -func (bn *blockNotarizer) RemoveLastNotarizedHeaderByHash(hash []byte) { - bn.mutNotarizedHeaders.Lock() - for shardID := range bn.notarizedHeaders { - notarizedHeadersCount := len(bn.notarizedHeaders[shardID]) - if notarizedHeadersCount > 1 && bytes.Equal(hash, bn.notarizedHeaders[shardID][notarizedHeadersCount-1].Hash) { - bn.notarizedHeaders[shardID] = bn.notarizedHeaders[shardID][:notarizedHeadersCount-1] - log.Debug("DELETED ", "extendedHdrHash", hash) - } - } - bn.mutNotarizedHeaders.Unlock() -} - // RestoreNotarizedHeadersToGenesis restores all notarized headers from each shard to the genesis value (nonce 0) func (bn *blockNotarizer) RestoreNotarizedHeadersToGenesis() { bn.mutNotarizedHeaders.Lock() diff --git a/process/track/sovereignChainShardBlockTrack.go b/process/track/sovereignChainShardBlockTrack.go index d6ebd479645..e4d149eff7c 100644 --- a/process/track/sovereignChainShardBlockTrack.go +++ b/process/track/sovereignChainShardBlockTrack.go @@ -261,16 +261,6 @@ func (scsbt *sovereignChainShardBlockTrack) RemoveLastCrossNotarizedHeaders() { scsbt.crossNotarizer.RemoveLastNotarizedHeader() } -type extendedCrossNotarizer interface { - blockNotarizerHandler - RemoveLastNotarizedHeaderByHash(hash []byte) -} - -// RemoveLastCrossNotarizedHeaderByHash removes last cross notarized header by hash, if exists -func (scsbt *sovereignChainShardBlockTrack) RemoveLastCrossNotarizedHeaderByHash(hash []byte) { - scsbt.crossNotarizer.(extendedCrossNotarizer).RemoveLastNotarizedHeaderByHash(hash) -} - // RemoveLastSelfNotarizedHeaders removes last self notarized headers from tracker list func (scsbt *sovereignChainShardBlockTrack) RemoveLastSelfNotarizedHeaders() { scsbt.selfNotarizer.RemoveLastNotarizedHeader() From ba1274b19692a771a46ca7f53df54f90ebffd7f8 Mon Sep 17 00:00:00 2001 From: MariusC Date: Thu, 6 Feb 2025 16:11:45 +0200 Subject: [PATCH 15/20] CLN: Log levels --- process/block/sovereignChainBlock.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index 7bb0a197294..efaa63e677f 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -889,7 +889,7 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity( } for _, extendedShardHdr := range extendedShardHdrs { - log.Debug("checkExtendedShardHeadersValidity", + log.Trace("checkExtendedShardHeadersValidity", "extendedShardHeader nonce", extendedShardHdr.GetNonce(), "extendedShardHeader round", extendedShardHdr.GetRound(), ) @@ -1146,7 +1146,7 @@ func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBloc process.ErrMissingHeader, extendedShardHeaderHash) } - log.Debug("sovereignChainBlockProcessor.sortExtendedShardHeadersForCurrentBlockByNonce", + log.Trace("sovereignChainBlockProcessor.sortExtendedShardHeadersForCurrentBlockByNonce", "headerInfo.hdr.GetNonce()", headerInfo.hdr.GetNonce(), "headerInfo.usedInBlock", headerInfo.usedInBlock, ) @@ -1977,7 +1977,9 @@ func (scbp *sovereignChainBlockProcessor) RestoreBlockIntoPools(header data.Head scbp.restoreBlockBody(header, body) // TODO: MX-16507: check how/if to restore incoming scrs/cross chain txs once we have a testnet setup for it - // We should probably have something similar to (sp *shardProcessor) rollBackProcessedMiniBlocksInfo + // 1. For incoming scrs We should probably have something similar to (sp *shardProcessor) rollBackProcessedMiniBlocksInfo + // 2. For outgoing operations, we should analyse how to revert operations or delay them before sending (wait for them + // to be finalized and impossible to rollback maybe?) sovChainHdr, castOk := header.(data.SovereignChainHeaderHandler) if !castOk { From 47f0c2f544b7d454722b7916a4dacf5416e1cf85 Mon Sep 17 00:00:00 2001 From: MariusC Date: Fri, 7 Feb 2025 11:41:08 +0200 Subject: [PATCH 16/20] FIX: After merge --- process/block/sovereignChainBlock.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/process/block/sovereignChainBlock.go b/process/block/sovereignChainBlock.go index e4b94830d24..be06a6591a7 100644 --- a/process/block/sovereignChainBlock.go +++ b/process/block/sovereignChainBlock.go @@ -800,7 +800,7 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header "nonce", headerHandler.GetNonce(), ) - sovereignChainHeader, ok := headerHandler.(data.SovereignChainHeaderHandler) + sovChainHeader, ok := headerHandler.(data.SovereignChainHeaderHandler) if !ok { return nil, nil, process.ErrWrongTypeAssertion } @@ -827,7 +827,7 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header scbp.blockChainHook.SetCurrentHeader(headerHandler) scbp.txCoordinator.RequestBlockTransactions(body) - requestedExtendedShardHdrs := scbp.requestExtendedShardHeaders(sovereignChainHeader) + requestedExtendedShardHdrs := scbp.requestExtendedShardHeaders(sovChainHeader) if haveTime() < 0 { return nil, nil, process.ErrTimeIsOut @@ -854,7 +854,7 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header go scbp.checkAndRequestIfExtendedShardHeadersMissing() }() - err = scbp.checkExtendedShardHeadersValidity(sovereignChainHeader) + err = scbp.checkExtendedShardHeadersValidity(sovChainHeader) if err != nil { return nil, nil, err } @@ -904,7 +904,9 @@ func (scbp *sovereignChainBlockProcessor) ProcessBlock(headerHandler data.Header } // checkExtendedShardHeadersValidity checks if used extended shard headers are valid as construction -func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity(sovereignChainHeader data.SovereignChainHeaderHandler) error { +func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity( + sovChainHeader data.SovereignChainHeaderHandler, +) error { lastCrossNotarizedHeader, _, err := scbp.blockTracker.GetLastCrossNotarizedHeader(core.MainChainShardId) if err != nil { return err @@ -915,7 +917,7 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity(sove "lastCrossNotarizedHeader round", lastCrossNotarizedHeader.GetRound(), ) - extendedShardHdrs, err := scbp.sortExtendedShardHeadersForCurrentBlockByNonce(sovereignChainHeader) + extendedShardHdrs, err := scbp.sortExtendedShardHeadersForCurrentBlockByNonce(sovChainHeader) if err != nil { return err } @@ -925,7 +927,7 @@ func (scbp *sovereignChainBlockProcessor) checkExtendedShardHeadersValidity(sove } // we should not have an epoch start block with main chain headers to be processed - if sovereignChainHeader.IsStartOfEpochBlock() { + if sovChainHeader.IsStartOfEpochBlock() { return errors.ErrReceivedSovereignEpochStartBlockWithExtendedHeaders } @@ -1206,7 +1208,8 @@ func (scbp *sovereignChainBlockProcessor) sortExtendedShardHeadersForCurrentBloc ) if headerInfo.usedInBlock { - if headerInfo.usedInBlock {hdrsForCurrentBlock = append(hdrsForCurrentBlock, headerInfo.hdr) + hdrsForCurrentBlock = append(hdrsForCurrentBlock, headerInfo.hdr) + } } scbp.hdrsForCurrBlock.mutHdrsForBlock.RUnlock() From 2ce24695478d9e5758da09d01f23f6f7988076a2 Mon Sep 17 00:00:00 2001 From: MariusC Date: Fri, 7 Feb 2025 14:25:59 +0200 Subject: [PATCH 17/20] FEAT: Cleaned refactored unit test RestoreBlockIntoPool --- process/block/sovereignChainBlock_test.go | 219 ++++++++++------------ 1 file changed, 94 insertions(+), 125 deletions(-) diff --git a/process/block/sovereignChainBlock_test.go b/process/block/sovereignChainBlock_test.go index bc4798b0946..c988151d82a 100644 --- a/process/block/sovereignChainBlock_test.go +++ b/process/block/sovereignChainBlock_test.go @@ -1,34 +1,23 @@ package block_test import ( - "math/big" + "errors" "testing" "time" - "github.com/multiversx/mx-chain-core-go/data/transaction" - "github.com/multiversx/mx-chain-core-go/hashing" - "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/stretchr/testify/assert" - "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/dataRetriever/requestHandlers" - "github.com/multiversx/mx-chain-go/errors" + errMx "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/process" blproc "github.com/multiversx/mx-chain-go/process/block" - "github.com/multiversx/mx-chain-go/process/coordinator" - "github.com/multiversx/mx-chain-go/process/factory/shard" - shardData "github.com/multiversx/mx-chain-go/process/factory/shard/data" "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/track" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" - commonMock "github.com/multiversx/mx-chain-go/testscommon/common" dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/economicsmocks" - "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" - "github.com/multiversx/mx-chain-go/testscommon/processMocks" "github.com/multiversx/mx-chain-go/testscommon/sovereign" storageStub "github.com/multiversx/mx-chain-go/testscommon/storage" storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" @@ -142,7 +131,7 @@ func TestSovereignBlockProcessor_NewSovereignChainBlockProcessorShouldWork(t *te scbp, err := blproc.NewSovereignChainBlockProcessor(args) require.Nil(t, scbp) - require.ErrorIs(t, err, errors.ErrNilOutgoingOperationsFormatter) + require.ErrorIs(t, err, errMx.ErrNilOutgoingOperationsFormatter) }) t.Run("should error when outgoing operation pool is nil", func(t *testing.T) { @@ -153,7 +142,7 @@ func TestSovereignBlockProcessor_NewSovereignChainBlockProcessorShouldWork(t *te scbp, err := blproc.NewSovereignChainBlockProcessor(args) require.Nil(t, scbp) - require.Equal(t, errors.ErrNilOutGoingOperationsPool, err) + require.Equal(t, errMx.ErrNilOutGoingOperationsPool, err) }) t.Run("should error when operations hasher is nil", func(t *testing.T) { @@ -164,7 +153,7 @@ func TestSovereignBlockProcessor_NewSovereignChainBlockProcessorShouldWork(t *te scbp, err := blproc.NewSovereignChainBlockProcessor(args) require.Nil(t, scbp) - require.Equal(t, errors.ErrNilOperationsHasher, err) + require.Equal(t, errMx.ErrNilOperationsHasher, err) }) t.Run("should error when epoch start data creator is nil", func(t *testing.T) { @@ -358,135 +347,108 @@ func TestSovereignChainBlockProcessor_createAndSetOutGoingMiniBlock(t *testing.T require.Equal(t, expectedSovChainHeader, sovChainHdr) } -func createTxCoordinator( - store dataRetriever.StorageService, - marshaller marshal.Marshalizer, - Hasher hashing.Hasher, - dataPool dataRetriever.PoolsHolder, -) process.TransactionCoordinator { - args := shardData.ArgPreProcessorsContainerFactory{ - ShardCoordinator: mock.NewMultiShardsCoordinatorMock(3), - Store: store, - Marshaller: marshaller, - Hasher: Hasher, - DataPool: dataPool, - PubkeyConverter: createMockPubkeyConverter(), - Accounts: initAccountsMock(), - RequestHandler: &testscommon.RequestHandlerStub{}, - TxProcessor: &testscommon.TxProcessorMock{}, - ScProcessor: &testscommon.SCProcessorMock{}, - ScResultProcessor: &testscommon.SmartContractResultsProcessorMock{}, - RewardsTxProcessor: &testscommon.RewardTxProcessorMock{}, - EconomicsFee: &economicsmocks.EconomicsHandlerStub{}, - GasHandler: &testscommon.GasHandlerStub{}, - BlockTracker: &mock.BlockTrackerMock{}, - BlockSizeComputation: &testscommon.BlockSizeComputationStub{}, - BalanceComputation: &testscommon.BalanceComputationStub{}, - EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - TxTypeHandler: &testscommon.TxTypeHandlerMock{}, - ScheduledTxsExecutionHandler: &testscommon.ScheduledTxsExecutionStub{}, - ProcessedMiniBlocksTracker: &testscommon.ProcessedMiniBlocksTrackerStub{}, - TxExecutionOrderHandler: &commonMock.TxExecutionOrderHandlerStub{}, - RunTypeComponents: processMocks.NewRunTypeComponentsStub(), - } - factory, _ := shard.NewPreProcessorsContainerFactory(args) - container, _ := factory.Create() - - argsTransactionCoordinator := createMockTransactionCoordinatorArguments(initAccountsMock(), dataPool, container) - tc, _ := coordinator.NewTransactionCoordinator(argsTransactionCoordinator) - return tc -} - func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing.T) { t.Parallel() - txHash := []byte("tx hash 1") - - datapool := dataRetrieverMock.NewPoolsHolderMock() - marshalizerMock := &mock.MarshalizerMock{} - hasherMock := &mock.HasherStub{} - - body := &block.Body{} - tx := &transaction.Transaction{ - Nonce: 1, - Value: big.NewInt(0), - } - buffTx, _ := marshalizerMock.Marshal(tx) - - store := &storageStubs.ChainStorerStub{ - GetAllCalled: func(unitType dataRetriever.UnitType, keys [][]byte) (map[string][]byte, error) { - m := make(map[string][]byte) - m[string(txHash)] = buffTx - return m, nil + store := &storageStubs.ChainStorerStub{} + marshaller := &mock.MarshalizerMock{} + expectedBody := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + { + SenderShardID: core.MainChainShardId, + ReceiverShardID: core.SovereignChainShardId, + TxHashes: [][]byte{[]byte("txHash1")}, + }, }, } + extendedHdrNonce := uint64(4) + extendedHdrNonceToBytes := []byte("toByteData") + dataPool := dataRetrieverMock.NewPoolsHolderMock() + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() - dataComponents.DataPool = datapool + coreComponents.UInt64ByteSliceConv = &testscommon.Uint64ByteSliceConverterStub{ + ToByteSliceCalled: func(u uint64) []byte { + require.Equal(t, extendedHdrNonce, u) + return extendedHdrNonceToBytes + }, + } + dataComponents.DataPool = dataPool dataComponents.Storage = store - coreComponents.Hash = hasherMock - coreComponents.IntMarsh = marshalizerMock + coreComponents.IntMarsh = marshaller arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) - arguments.TxCoordinator = createTxCoordinator(store, marshalizerMock, hasherMock, datapool) + arguments.RequestHandler = &testscommon.ExtendedShardHeaderRequestHandlerStub{} - shardArguments := CreateSovereignChainShardTrackerMockArguments() - sbt, _ := track.NewShardBlockTrack(shardArguments) - arguments.BlockTracker, _ = track.NewSovereignChainShardBlockTrack(sbt) - rrh, _ := requestHandlers.NewResolverRequestHandler( - &dataRetrieverMock.RequestersFinderStub{}, - &mock.RequestedItemsHandlerStub{}, - &testscommon.WhiteListHandlerStub{}, - 1, - 0, - time.Second, - ) - arguments.RequestHandler, _ = requestHandlers.NewSovereignResolverRequestHandler(rrh) + wasRestoreBlockDataFromStorageCalled := false + arguments.TxCoordinator = &testscommon.TransactionCoordinatorMock{ + RestoreBlockDataFromStorageCalled: func(body *block.Body) (int, error) { + wasRestoreBlockDataFromStorageCalled = true + require.Equal(t, body, expectedBody) + return 0, nil + }, + } - sp, _ := blproc.NewShardProcessor(arguments) + wasLastSelfNotarizedHeaderRemoved := false + wasLastCrossNotarizedHeaderRemoved := false + arguments.BlockTracker = &testscommon.ExtendedShardHeaderTrackerStub{ + RemoveLastSelfNotarizedHeadersCalled: func() { + wasLastSelfNotarizedHeaderRemoved = true + }, + RemoveLastCrossNotarizedHeadersCalled: func() { + wasLastCrossNotarizedHeaderRemoved = true + }, + } + sp, _ := blproc.NewShardProcessor(arguments) argsSovProc := createSovChainBlockProcessorArgs() argsSovProc.ShardProcessor = sp sovBlockProc, _ := blproc.NewSovereignChainBlockProcessor(argsSovProc) - txHashes := make([][]byte, 0) - txHashes = append(txHashes, txHash) - miniblock := block.MiniBlock{ - ReceiverShardID: 0, - SenderShardID: 1, - TxHashes: txHashes, + extendedHeaderHash := []byte("extendedHdrHash") + extendedHeader := &block.ShardHeaderExtended{ + Header: &block.HeaderV2{ + ScheduledRootHash: []byte("rootHash"), + Header: &block.Header{ + Nonce: extendedHdrNonce, + }, + }, } - body.MiniBlocks = append(body.MiniBlocks, &miniblock) - miniblockHash := []byte("mini block hash 1") - hasherMock.ComputeCalled = func(s string) []byte { - return miniblockHash + wasExtendedHdrStorerRemoved := false + storerExtendedHdr := &storageStubs.StorerStub{ + RemoveCalled: func(key []byte) error { + wasExtendedHdrStorerRemoved = true + require.Equal(t, extendedHeaderHash, key) + return nil + }, + GetCalled: func(key []byte) ([]byte, error) { + return marshaller.Marshal(extendedHeader) + }, } - extendedHeaderHash := []byte("meta block hash 1") - extendedHeader := &block.ShardHeaderExtended{ - Header: &block.HeaderV2{ - ScheduledRootHash: []byte("root hash"), + wasNonceHashExtendedHdrStorerRemoved := false + storerNonceHashExtendedHdr := &storageStubs.StorerStub{ + RemoveCalled: func(key []byte) error { + wasNonceHashExtendedHdrStorerRemoved = true + require.Equal(t, extendedHdrNonceToBytes, key) + return nil }, - IncomingMiniBlocks: nil, - IncomingEvents: nil, } - datapool.Headers().AddHeader(extendedHeaderHash, extendedHeader) store.GetStorerCalled = func(unitType dataRetriever.UnitType) (storage.Storer, error) { - return &storageStubs.StorerStub{ - RemoveCalled: func(key []byte) error { - return nil - }, - GetCalled: func(key []byte) ([]byte, error) { - return marshalizerMock.Marshal(extendedHeader) - }, - }, nil + switch unitType { + case dataRetriever.ExtendedShardHeadersUnit: + return storerExtendedHdr, nil + case dataRetriever.ExtendedShardHeadersNonceHashDataUnit: + return storerNonceHashExtendedHdr, nil + } + + return nil, errors.New("invalid storer type") } miniBlockHeader := block.MiniBlockHeader{ - Hash: miniblockHash, - SenderShardID: miniblock.SenderShardID, - ReceiverShardID: miniblock.ReceiverShardID, + SenderShardID: expectedBody.MiniBlocks[0].SenderShardID, + ReceiverShardID: expectedBody.MiniBlocks[0].ReceiverShardID, } sovHdr := &block.SovereignChainHeader{ @@ -496,14 +458,21 @@ func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing ExtendedShardHeaderHashes: [][]byte{extendedHeaderHash}, } - err := sovBlockProc.RestoreBlockIntoPools(sovHdr, body) - assert.Nil(t, err) + retrievedHdr, err := dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.NotNil(t, err) + require.Nil(t, retrievedHdr) - miniblockFromPool, _ := datapool.MiniBlocks().Get(miniblockHash) - txFromPool, _ := datapool.Transactions().SearchFirstData(txHash) - assert.Nil(t, err) - assert.Equal(t, &miniblock, miniblockFromPool) - assert.Equal(t, tx, txFromPool) + err = sovBlockProc.RestoreBlockIntoPools(sovHdr, expectedBody) + require.Nil(t, err) + require.True(t, wasRestoreBlockDataFromStorageCalled) + require.True(t, wasLastSelfNotarizedHeaderRemoved) + require.True(t, wasLastCrossNotarizedHeaderRemoved) + require.True(t, wasExtendedHdrStorerRemoved) + require.True(t, wasNonceHashExtendedHdrStorerRemoved) + + retrievedHdr, err = dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.Nil(t, err) + require.Equal(t, extendedHeader, retrievedHdr) } //TODO: More unit tests should be added. Created PR https://multiversxlabs.atlassian.net/browse/MX-14149 From caab04eeacf25e5fee84ecfdddbdff6f34e3999c Mon Sep 17 00:00:00 2001 From: MariusC Date: Fri, 7 Feb 2025 14:37:21 +0200 Subject: [PATCH 18/20] FEAT: Extend test as func with or without extended header --- process/block/sovereignChainBlock_test.go | 32 ++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/process/block/sovereignChainBlock_test.go b/process/block/sovereignChainBlock_test.go index c988151d82a..b894f752a47 100644 --- a/process/block/sovereignChainBlock_test.go +++ b/process/block/sovereignChainBlock_test.go @@ -350,6 +350,11 @@ func TestSovereignChainBlockProcessor_createAndSetOutGoingMiniBlock(t *testing.T func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing.T) { t.Parallel() + testRestoreBlockIntoPools(t, true) + testRestoreBlockIntoPools(t, false) +} + +func testRestoreBlockIntoPools(t *testing.T, withExtendedHeader bool) { store := &storageStubs.ChainStorerStub{} marshaller := &mock.MarshalizerMock{} expectedBody := &block.Body{ @@ -458,6 +463,10 @@ func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing ExtendedShardHeaderHashes: [][]byte{extendedHeaderHash}, } + if !withExtendedHeader { + sovHdr.ExtendedShardHeaderHashes = nil + } + retrievedHdr, err := dataPool.Headers().GetHeaderByHash(extendedHeaderHash) require.NotNil(t, err) require.Nil(t, retrievedHdr) @@ -466,13 +475,24 @@ func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing require.Nil(t, err) require.True(t, wasRestoreBlockDataFromStorageCalled) require.True(t, wasLastSelfNotarizedHeaderRemoved) - require.True(t, wasLastCrossNotarizedHeaderRemoved) - require.True(t, wasExtendedHdrStorerRemoved) - require.True(t, wasNonceHashExtendedHdrStorerRemoved) - retrievedHdr, err = dataPool.Headers().GetHeaderByHash(extendedHeaderHash) - require.Nil(t, err) - require.Equal(t, extendedHeader, retrievedHdr) + if withExtendedHeader { + require.True(t, wasLastCrossNotarizedHeaderRemoved) + require.True(t, wasExtendedHdrStorerRemoved) + require.True(t, wasNonceHashExtendedHdrStorerRemoved) + + retrievedHdr, err = dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.Nil(t, err) + require.Equal(t, extendedHeader, retrievedHdr) + } else { + require.False(t, wasLastCrossNotarizedHeaderRemoved) + require.False(t, wasExtendedHdrStorerRemoved) + require.False(t, wasNonceHashExtendedHdrStorerRemoved) + + retrievedHdr, err = dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.NotNil(t, err) + require.Nil(t, retrievedHdr) + } } //TODO: More unit tests should be added. Created PR https://multiversxlabs.atlassian.net/browse/MX-14149 From ccea95209b5101501d9429e9e72d4e1cfcf66e64 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 10 Feb 2025 15:06:10 +0200 Subject: [PATCH 19/20] FEAT: Test RestoreBlockIntoPoolsInvalidHeaderType --- process/block/sovereignChainBlock_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/process/block/sovereignChainBlock_test.go b/process/block/sovereignChainBlock_test.go index b894f752a47..d79d85ff2fc 100644 --- a/process/block/sovereignChainBlock_test.go +++ b/process/block/sovereignChainBlock_test.go @@ -347,6 +347,16 @@ func TestSovereignChainBlockProcessor_createAndSetOutGoingMiniBlock(t *testing.T require.Equal(t, expectedSovChainHeader, sovChainHdr) } +func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsInvalidHeaderType(t *testing.T) { + t.Parallel() + + sovBlockProcArgs := createSovChainBlockProcessorArgs() + sovBlockProc, _ := blproc.NewSovereignChainBlockProcessor(sovBlockProcArgs) + + err := sovBlockProc.RestoreBlockIntoPools(&block.HeaderV2{}, &block.Body{}) + require.ErrorIs(t, err, errMx.ErrWrongTypeAssertion) +} + func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing.T) { t.Parallel() From ddb81349d2d418d8d294afdea86bda4572144143 Mon Sep 17 00:00:00 2001 From: MariusC Date: Mon, 10 Feb 2025 16:56:12 +0200 Subject: [PATCH 20/20] FEAT: RestoreBlockIntoPoolsShouldWorkWhenHeaderNotCommitted --- process/block/sovereignChainBlock_test.go | 97 ++++++++++++++++++++--- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/process/block/sovereignChainBlock_test.go b/process/block/sovereignChainBlock_test.go index d79d85ff2fc..994b3fa9d78 100644 --- a/process/block/sovereignChainBlock_test.go +++ b/process/block/sovereignChainBlock_test.go @@ -357,6 +357,76 @@ func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsInvalidHeaderType(t * require.ErrorIs(t, err, errMx.ErrWrongTypeAssertion) } +func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWorkWhenHeaderNotCommitted(t *testing.T) { + t.Parallel() + + store := &storageStubs.ChainStorerStub{ + GetStorerCalled: func(unitType dataRetriever.UnitType) (storage.Storer, error) { + return nil, errors.New("header not fully processed yet") + }, + } + expectedBody := &block.Body{ + MiniBlocks: []*block.MiniBlock{ + { + SenderShardID: core.MainChainShardId, + ReceiverShardID: core.SovereignChainShardId, + TxHashes: [][]byte{[]byte("txHash1")}, + }, + }, + } + + dataPool := dataRetrieverMock.NewPoolsHolderMock() + + coreComponents, dataComponents, bootstrapComponents, statusComponents := createComponentHolderMocks() + dataComponents.DataPool = dataPool + dataComponents.Storage = store + arguments := CreateMockArguments(coreComponents, dataComponents, bootstrapComponents, statusComponents) + arguments.RequestHandler = &testscommon.ExtendedShardHeaderRequestHandlerStub{} + + wasRestoreBlockDataFromStorageCalled := false + arguments.TxCoordinator = &testscommon.TransactionCoordinatorMock{ + RestoreBlockDataFromStorageCalled: func(body *block.Body) (int, error) { + wasRestoreBlockDataFromStorageCalled = true + require.Equal(t, body, expectedBody) + return 0, nil + }, + } + + wasLastSelfNotarizedHeaderRemoved := false + wasLastCrossNotarizedHeaderRemoved := false + arguments.BlockTracker = &testscommon.ExtendedShardHeaderTrackerStub{ + RemoveLastSelfNotarizedHeadersCalled: func() { + wasLastSelfNotarizedHeaderRemoved = true + }, + RemoveLastCrossNotarizedHeadersCalled: func() { + wasLastCrossNotarizedHeaderRemoved = true + }, + } + + sp, _ := blproc.NewShardProcessor(arguments) + argsSovProc := createSovChainBlockProcessorArgs() + argsSovProc.ShardProcessor = sp + sovBlockProc, _ := blproc.NewSovereignChainBlockProcessor(argsSovProc) + + extendedHeaderHash := []byte("extendedHdrHash") + sovHdr := createSovHeaderForRestoreBlocksTest(expectedBody, extendedHeaderHash) + + retrievedHdr, err := dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.NotNil(t, err) + require.Nil(t, retrievedHdr) + + err = sovBlockProc.RestoreBlockIntoPools(sovHdr, expectedBody) + require.Nil(t, err) + require.True(t, wasRestoreBlockDataFromStorageCalled) + require.True(t, wasLastSelfNotarizedHeaderRemoved) + require.True(t, wasLastCrossNotarizedHeaderRemoved) + + retrievedHdr, err = dataPool.Headers().GetHeaderByHash(extendedHeaderHash) + require.NotNil(t, err) + require.Nil(t, retrievedHdr) + +} + func TestSovereignChainBlockProcessor_RestoreBlockIntoPoolsShouldWork(t *testing.T) { t.Parallel() @@ -461,18 +531,7 @@ func testRestoreBlockIntoPools(t *testing.T, withExtendedHeader bool) { return nil, errors.New("invalid storer type") } - miniBlockHeader := block.MiniBlockHeader{ - SenderShardID: expectedBody.MiniBlocks[0].SenderShardID, - ReceiverShardID: expectedBody.MiniBlocks[0].ReceiverShardID, - } - - sovHdr := &block.SovereignChainHeader{ - Header: &block.Header{ - MiniBlockHeaders: []block.MiniBlockHeader{miniBlockHeader}, - }, - ExtendedShardHeaderHashes: [][]byte{extendedHeaderHash}, - } - + sovHdr := createSovHeaderForRestoreBlocksTest(expectedBody, extendedHeaderHash) if !withExtendedHeader { sovHdr.ExtendedShardHeaderHashes = nil } @@ -505,4 +564,18 @@ func testRestoreBlockIntoPools(t *testing.T, withExtendedHeader bool) { } } +func createSovHeaderForRestoreBlocksTest(body *block.Body, extendedHeaderHash []byte) *block.SovereignChainHeader { + miniBlockHeader := block.MiniBlockHeader{ + SenderShardID: body.MiniBlocks[0].SenderShardID, + ReceiverShardID: body.MiniBlocks[0].ReceiverShardID, + } + + return &block.SovereignChainHeader{ + Header: &block.Header{ + MiniBlockHeaders: []block.MiniBlockHeader{miniBlockHeader}, + }, + ExtendedShardHeaderHashes: [][]byte{extendedHeaderHash}, + } +} + //TODO: More unit tests should be added. Created PR https://multiversxlabs.atlassian.net/browse/MX-14149