From cdc4fa40c9560226273024a0df13c9ebbda898b8 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 22 Aug 2023 19:13:02 -1000 Subject: [PATCH 01/63] feat: Add SectorDealIDs with CBOR serialization - Introduced `SectorDealIDs` struct to store DealIDs associated with a sector in the market module. - Implemented CBOR marshaling and unmarshaling for `SectorDealIDs`. - Updated related references and dependencies in the market package. --- builtin/v12/gen/gen.go | 1 + builtin/v12/market/cbor_gen.go | 89 ++++++++++++++++++++++++++++++ builtin/v12/market/market_state.go | 4 ++ 3 files changed, 94 insertions(+) diff --git a/builtin/v12/gen/gen.go b/builtin/v12/gen/gen.go index 55b8ec6b..999b382b 100644 --- a/builtin/v12/gen/gen.go +++ b/builtin/v12/gen/gen.go @@ -128,6 +128,7 @@ func main() { market.GetDealTermReturn{}, market.GetDealActivationReturn{}, market.OnMinerSectorsTerminateParams{}, + market.SectorDealIDs{}, // other types market.DealProposal{}, diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 5857440c..4ac835a1 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -1784,6 +1784,95 @@ func (t *OnMinerSectorsTerminateParams) UnmarshalCBOR(r io.Reader) (err error) { return nil } +var lengthBufSectorDealIDs = []byte{129} + +func (t *SectorDealIDs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufSectorDealIDs); err != nil { + return err + } + + // t.DealIDs ([]abi.DealID) (slice) + if len(t.DealIDs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.DealIDs was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.DealIDs))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cw.CborWriteHeader(cbg.MajUnsignedInt, uint64(v)); err != nil { + return err + } + } + return nil +} + +func (t *SectorDealIDs) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorDealIDs{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.DealIDs ([]abi.DealID) (slice) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DealIDs = make([]abi.DealID, extra) + } + + for i := 0; i < int(extra); i++ { + + maj, val, err := cr.ReadHeader() + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = abi.DealID(val) + } + + return nil +} + var lengthBufDealProposal = []byte{139} func (t *DealProposal) MarshalCBOR(w io.Writer) error { diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 7bcf82ca..5c516008 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -101,6 +101,10 @@ func ConstructState(store adt.Store) (*State, error) { }, nil } +type SectorDealIDs struct { + DealIDs []abi.DealID +} + // A specialization of a array to deals. // It is an error to query for a key that doesn't exist. type DealArray struct { From c1a758f32f750df6f43c34c85df39e1665779bdd Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 22 Aug 2023 19:37:23 -1000 Subject: [PATCH 02/63] feat: Introduce SectorDeals mapping in market state - Enhanced `State` struct in the market module to include `SectorDeals`, a mapping of sector IDs to deal IDs grouped by storage provider. - Updated CBOR serialization logic in `cbor_gen.go`: - Adjusted `lengthBufState` to reflect new field. - Implemented CBOR marshaling and unmarshaling for `SectorDeals`. - This addition facilitates more efficient queries for sector-specific deal data. --- builtin/v12/market/cbor_gen.go | 22 ++++++++++++++++++++-- builtin/v12/market/market_state.go | 4 ++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 4ac835a1..6bb8cf20 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -20,7 +20,7 @@ var _ = cid.Undef var _ = math.E var _ = sort.Sort -var lengthBufState = []byte{140} +var lengthBufState = []byte{141} func (t *State) MarshalCBOR(w io.Writer) error { if t == nil { @@ -108,6 +108,12 @@ func (t *State) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.PendingDealAllocationIds: %w", err) } + // t.SectorDeals (cid.Cid) (struct) + + if err := cbg.WriteCid(cw, t.SectorDeals); err != nil { + return xerrors.Errorf("failed to write cid field t.SectorDeals: %w", err) + } + return nil } @@ -130,7 +136,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 13 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -283,6 +289,18 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { t.PendingDealAllocationIds = c + } + // t.SectorDeals (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SectorDeals: %w", err) + } + + t.SectorDeals = c + } return nil } diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 5c516008..75e60a6d 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -55,6 +55,10 @@ type State struct { // Verified registry allocation IDs for deals that are not yet activated. PendingDealAllocationIds cid.Cid // HAMT[DealID]AllocationID + + // New mapping of sector IDs to deal IDS, grouped by storage provider. + SectorDeals cid.Cid // HAMT[Address]HAMT[SectorNumber]SectorDeals + } func ConstructState(store adt.Store) (*State, error) { From a08aa8fcd47eb925e0ea9e8811cdce3b045e6525 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 22 Aug 2023 19:45:44 -1000 Subject: [PATCH 03/63] temp: Add logging for sectorToDealIdHamtCid and minerAddr in migration - Extended sector migration functions in `miner.go` to support `minerAddr`. - Introduced `sectorToDealIdHamt` to map sector numbers to their associated deal IDs. - Added console print to log `sectorToDealIdHamtCid` and `minerAddr` for subsequent construction of `sector_deal_ids_index`. - This is a preliminary commit, and the logging will be removed after constructing the required index. --- builtin/v12/migration/miner.go | 49 +++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 7252fe60..5c99532b 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -2,9 +2,12 @@ package migration import ( "context" + "fmt" + "github.com/filecoin-project/go-state-types/builtin" miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" + "github.com/filecoin-project/go-state-types/builtin/v12/market" miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner" adt12 "github.com/filecoin-project/go-state-types/builtin/v12/util/adt" "github.com/filecoin-project/go-state-types/migration" @@ -175,7 +178,7 @@ func migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migra if okIn && okOut { // we have previous work, but the AMT has changed -- diff them - outRoot, err = migrateSectorsWithDiff(ctx, store, inRoot, prevInRoot, prevOutRoot) + outRoot, err = migrateSectorsWithDiff(ctx, store, minerAddr, inRoot, prevInRoot, prevOutRoot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } @@ -185,7 +188,7 @@ func migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migra if err != nil { return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } - outArray, err := migrateSectorsFromScratch(ctx, store, inArray) + outArray, err := migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } @@ -224,7 +227,7 @@ delta = prevInRoot - inRoot merge( prevOutRoot, Migrated(delta) ) */ -func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid) (cid.Cid, error) { +func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, minerAddr address.Address, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid) (cid.Cid, error) { // we have previous work, but the AMT has changed -- diff them diffs, err := amt.Diff(ctx, store, store, prevInRoot, inRoot, amt.UseTreeBitWidth(miner11.SectorsAmtBitwidth)) if err != nil { @@ -260,7 +263,7 @@ func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, inRoot cid.C return cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) } - if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info)); err != nil { + if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info, store)); err != nil { return cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) } } @@ -269,19 +272,33 @@ func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, inRoot cid.C return prevOutSectors.Root() } -func migrateSectorsFromScratch(ctx context.Context, store adt11.Store, inArray *adt11.Array) (*adt12.Array, error) { +func migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, error) { outArray, err := adt12.MakeEmptyArray(store, miner12.SectorsAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to construct new sectors array: %w", err) } + sectorToDealIdHamt, err := builtin.NewTree(store) + if err != nil { + return nil, xerrors.Errorf("creating new state tree: %w", err) + } + var sectorInfo miner11.SectorOnChainInfo if err = inArray.ForEach(§orInfo, func(k int64) error { - return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) + + err = addSectorNumberToDealIdHAMT(sectorToDealIdHamt, sectorInfo, store) + + return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo, store)) }); err != nil { return nil, err } + sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() + if err != nil { + return nil, err + } + fmt.Println(sectorToDealIdHamtCid.String(), minerAddr.String()) + return outArray, err } @@ -319,7 +336,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, var outSnapshotRoot cid.Cid if okIn && okOut { - outSnapshotRoot, err = migrateSectorsWithDiff(ctx, store, inDeadline.SectorsSnapshot, currentInRoot, currentOutRoot) + outSnapshotRoot, err = migrateSectorsWithDiff(ctx, store, minerAddr, inDeadline.SectorsSnapshot, currentInRoot, currentOutRoot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } @@ -328,7 +345,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, if err != nil { return cid.Undef, err } - outSnapshot, err := migrateSectorsFromScratch(ctx, store, inSectorsSnapshot) + outSnapshot, err := migrateSectorsFromScratch(ctx, store, minerAddr, inSectorsSnapshot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors: %w", err) } @@ -371,7 +388,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, return store.Put(ctx, &outDeadlines) } -func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo) *miner12.SectorOnChainInfo { +func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo, store adt11.Store) *miner12.SectorOnChainInfo { // For a sector that has not been updated: the Activation is correct and ReplacedSectorAge is zero. // For a sector that has been updated through SnapDeals: Activation is the epoch at which it was upgraded, and ReplacedSectorAge is delta since the true activation. // For a sector that has been updated through the old CC path: Activation is correct @@ -405,3 +422,17 @@ func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo) *miner12.SectorOnCh SimpleQAPower: sectorInfo.SimpleQAPower, } } + +func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { + //sectorInfo.DealIDs,sectorInfo.SectorNumber + // - todo make SectorNumber and explicit type, not just cast to int64 + // + // - todo for sector use array not CborInt + + cborDealIDs := market.SectorDealIDs{DealIDs: sectorInfo.DealIDs} + err := xap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &cborDealIDs) + if err != nil { + return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) + } + return nil +} From f7b3f39d45a7334b848c8e80c652fbc207bd274e Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 23 Aug 2023 16:52:09 -1000 Subject: [PATCH 04/63] Add HamtCid struct and its CBOR serialization methods The HamtCid struct was introduced in the market module, along with the necessary methods to marshal and unmarshal it using CBOR. Since this information is auto-generated, additional detailed comments have been omitted. - Added HamtCid type definition in market_state.go - Implemented MarshalCBOR and UnmarshalCBOR methods for HamtCid in cbor_gen.go - Updated gen.go to include the HamtCid type in the main function --- builtin/v12/gen/gen.go | 1 + builtin/v12/market/cbor_gen.go | 61 ++++++++++++++++++++++++++++++ builtin/v12/market/market_state.go | 4 ++ 3 files changed, 66 insertions(+) diff --git a/builtin/v12/gen/gen.go b/builtin/v12/gen/gen.go index 999b382b..938dd57b 100644 --- a/builtin/v12/gen/gen.go +++ b/builtin/v12/gen/gen.go @@ -129,6 +129,7 @@ func main() { market.GetDealActivationReturn{}, market.OnMinerSectorsTerminateParams{}, market.SectorDealIDs{}, + market.HamtCid{}, // other types market.DealProposal{}, diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 6bb8cf20..e1fc04a5 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -1891,6 +1891,67 @@ func (t *SectorDealIDs) UnmarshalCBOR(r io.Reader) (err error) { return nil } +var lengthBufHamtCid = []byte{129} + +func (t *HamtCid) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufHamtCid); err != nil { + return err + } + + // t.Cid (cid.Cid) (struct) + + if err := cbg.WriteCid(cw, t.Cid); err != nil { + return xerrors.Errorf("failed to write cid field t.Cid: %w", err) + } + + return nil +} + +func (t *HamtCid) UnmarshalCBOR(r io.Reader) (err error) { + *t = HamtCid{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Cid (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Cid: %w", err) + } + + t.Cid = c + + } + return nil +} + var lengthBufDealProposal = []byte{139} func (t *DealProposal) MarshalCBOR(w io.Writer) error { diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 75e60a6d..405b057a 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -109,6 +109,10 @@ type SectorDealIDs struct { DealIDs []abi.DealID } +type HamtCid struct { + Cid cid.Cid +} + // A specialization of a array to deals. // It is an error to query for a key that doesn't exist. type DealArray struct { From d7f9e7647bb09cdf41f88cc37fadc7f8215580c1 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 23 Aug 2023 16:54:21 -1000 Subject: [PATCH 05/63] commit -m "Enhance miner migration and introduce market migration - Add `sectorDeals` field in `minerMigrator` for mapping of sector IDs to deal IDs. - Modify sector migration methods to be methods of the `minerMigrator` struct. - Introduce a mechanism to map sector numbers to their associated deal IDs and add to the state tree. - Correct manifest actor matching for `market11Cid`. - Initiate a second migration for the market actor post miner migration. " --- builtin/v12/migration/miner.go | 36 +++++++++++++++++++++++++------- builtin/v12/migration/top.go | 38 +++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 5c99532b..7cf0dd89 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -2,7 +2,6 @@ package migration import ( "context" - "fmt" "github.com/filecoin-project/go-state-types/builtin" miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner" @@ -31,6 +30,7 @@ type minerMigrator struct { emptyDeadlinesV11 cid.Cid emptyDeadlineV12 cid.Cid emptyDeadlinesV12 cid.Cid + sectorDeals *builtin.ActorTree OutCodeCID cid.Cid } @@ -70,11 +70,18 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid } + // New mapping of sector IDs to deal IDS, grouped by storage provider. + sectorDeals, err := builtin.NewTree(ctxStore) + if err != nil { + return nil, xerrors.Errorf("creating new state tree: %w", err) + } + return &minerMigrator{ emptyDeadlineV11: edv11cid, emptyDeadlinesV11: edsv11cid, emptyDeadlineV12: edv12cid, emptyDeadlinesV12: edsv12cid, + sectorDeals: sectorDeals, OutCodeCID: outCode, }, nil } @@ -123,7 +130,7 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i wrappedStore := adt11.WrapStore(ctx, store) - newSectors, err := migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) + newSectors, err := m.migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) if err != nil { return nil, xerrors.Errorf("failed to migrate sectors for miner: %s: %w", in.Address, err) } @@ -162,7 +169,7 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i }, nil } -func migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, error) { +func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, error) { return cache.Load(migration.SectorsAmtKey(inRoot), func() (cid.Cid, error) { okIn, prevInRoot, err := cache.Read(migration.MinerPrevSectorsInKey(minerAddr)) @@ -188,7 +195,7 @@ func migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migra if err != nil { return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } - outArray, err := migrateSectorsFromScratch(ctx, store, minerAddr, inArray) + outArray, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } @@ -272,7 +279,7 @@ func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, minerAddr ad return prevOutSectors.Root() } -func migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, error) { +func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, error) { outArray, err := adt12.MakeEmptyArray(store, miner12.SectorsAmtBitwidth) if err != nil { return nil, xerrors.Errorf("failed to construct new sectors array: %w", err) @@ -297,7 +304,11 @@ func migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr if err != nil { return nil, err } - fmt.Println(sectorToDealIdHamtCid.String(), minerAddr.String()) + + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) + if err != nil { + return nil, err + } return outArray, err } @@ -345,7 +356,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, if err != nil { return cid.Undef, err } - outSnapshot, err := migrateSectorsFromScratch(ctx, store, minerAddr, inSectorsSnapshot) + outSnapshot, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inSectorsSnapshot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors: %w", err) } @@ -436,3 +447,14 @@ func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.Sect } return nil } + +func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { + + cborCid := market.HamtCid{Cid: hamtCid} + err := m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), &cborCid) + + if err != nil { + return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) + } + return nil +} diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index b6344cbd..09815fcc 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -66,11 +66,15 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID deferredCodeIDs := make(map[cid.Cid]struct{}) miner11Cid := cid.Undef + market11Cid := cid.Undef for _, oldEntry := range oldManifestData.Entries { if oldEntry.Name == manifest.MinerKey { miner11Cid = oldEntry.Code } + if oldEntry.Name == manifest.MinerKey { + market11Cid = oldEntry.Code + } newCodeCID, ok := newManifest.Get(oldEntry.Name) if !ok { return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", oldEntry.Name) @@ -78,6 +82,14 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID migrations[oldEntry.Code] = migration.CachedMigration(cache, migration.CodeMigrator{OutCodeCID: newCodeCID}) } + if !miner11Cid.Defined() { + return cid.Undef, xerrors.Errorf("didn't find miner actor in old manifest") + } + + if !market11Cid.Defined() { + return cid.Undef, xerrors.Errorf("didn't find market actor in old manifest") + } + // migrations that migrate both code and state, override entries in `migrations` // The System Actor @@ -95,11 +107,11 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") } - mm, err := newMinerMigrator(ctx, store, miner12Cid) + minerMigrator, err := newMinerMigrator(ctx, store, miner12Cid) if err != nil { return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err) } - migrations[miner11Cid] = migration.CachedMigration(cache, *mm) + migrations[miner11Cid] = migration.CachedMigration(cache, *minerMigrator) if len(migrations)+len(deferredCodeIDs) != len(oldManifestData.Entries) { return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) @@ -110,7 +122,27 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) } - outCid, err := actorsOut.Flush() + //todo run second migration here for market + migrations2 := make(map[cid.Cid]migration.ActorMigration) + + // The Market Actor + market12Cid, ok := newManifest.Get(manifest.MarketKey) + if !ok { + return cid.Undef, xerrors.Errorf("code cid for market actor not found in new manifest") + } + + marketMigrator, err := newMarketMigrator(ctx, store, market12Cid, minerMigrator) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create market migrator: %w", err) + } + migrations2[market11Cid] = migration.CachedMigration(cache, *marketMigrator) + + actorsOut2, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsOut, migrations2) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) + } + + outCid, err := actorsOut2.Flush() if err != nil { return cid.Undef, xerrors.Errorf("failed to flush actorsOut: %w", err) } From f4616ba7ebea933ebc41018c1daf52e46cab47ea Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 23 Aug 2023 17:27:11 -1000 Subject: [PATCH 06/63] forgot the file for market migration --- builtin/v12/migration/market.go | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 builtin/v12/migration/market.go diff --git a/builtin/v12/migration/market.go b/builtin/v12/migration/market.go new file mode 100644 index 00000000..78a472bd --- /dev/null +++ b/builtin/v12/migration/market.go @@ -0,0 +1,76 @@ +package migration + +import ( + "context" + + market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" + market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" + + "github.com/filecoin-project/go-state-types/migration" + + "golang.org/x/xerrors" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" +) + +// The marketMigrator performs the following migrations: +// TOOD + +func newMarketMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, x *minerMigrator) (*marketMigrator, error) { + sectorToDealIdHamtCid, err := x.sectorDeals.Map.Root() + if err != nil { + return nil, err + } + + return &marketMigrator{ + OutCodeCID: outCode, + sectorDealIDs: sectorToDealIdHamtCid, + }, nil +} + +type marketMigrator struct { + sectorDealIDs cid.Cid + + OutCodeCID cid.Cid +} + +func (m marketMigrator) MigratedCodeCID() cid.Cid { + return m.OutCodeCID +} + +func (m marketMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, in migration.ActorMigrationInput) (*migration.ActorMigrationResult, error) { + var inState market11.State + if err := store.Get(ctx, in.Head, &inState); err != nil { + return nil, xerrors.Errorf("failed to load miner state for %s: %w", in.Address, err) + } + + newSectorDeals := m.sectorDealIDs + + // Create the new state + outState := market12.State{ + Proposals: inState.Proposals, + States: inState.States, + PendingProposals: inState.PendingProposals, + EscrowTable: inState.EscrowTable, + LockedTable: inState.LockedTable, + NextID: inState.NextID, + DealOpsByEpoch: inState.DealOpsByEpoch, + LastCron: inState.LastCron, + TotalClientLockedCollateral: inState.TotalClientLockedCollateral, + TotalProviderLockedCollateral: inState.TotalProviderLockedCollateral, + TotalClientStorageFee: inState.TotalClientStorageFee, + PendingDealAllocationIds: inState.PendingDealAllocationIds, + SectorDeals: newSectorDeals, // Updated value + } + + newHead, err := store.Put(ctx, &outState) + if err != nil { + return nil, xerrors.Errorf("failed to put new state: %w", err) + } + + return &migration.ActorMigrationResult{ + NewCodeCID: m.MigratedCodeCID(), + NewHead: newHead, + }, nil +} From b7c927f80dcb0feef947a4b4f3d32f8da45486dd Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 24 Aug 2023 11:37:21 -1000 Subject: [PATCH 07/63] Refactor migration logic and streamline market actor updates. - Removed `builtin/v12/migration/market.go` as the migration logic is now centralized in `top.go`. - Enhanced error messages in the migration process for better debugging. - Added safety checks before updating sector deal IDs in the market actor. --- builtin/v12/migration/market.go | 76 --------------------------------- builtin/v12/migration/top.go | 43 ++++++++++++------- 2 files changed, 28 insertions(+), 91 deletions(-) delete mode 100644 builtin/v12/migration/market.go diff --git a/builtin/v12/migration/market.go b/builtin/v12/migration/market.go deleted file mode 100644 index 78a472bd..00000000 --- a/builtin/v12/migration/market.go +++ /dev/null @@ -1,76 +0,0 @@ -package migration - -import ( - "context" - - market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" - market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" - - "github.com/filecoin-project/go-state-types/migration" - - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" -) - -// The marketMigrator performs the following migrations: -// TOOD - -func newMarketMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, x *minerMigrator) (*marketMigrator, error) { - sectorToDealIdHamtCid, err := x.sectorDeals.Map.Root() - if err != nil { - return nil, err - } - - return &marketMigrator{ - OutCodeCID: outCode, - sectorDealIDs: sectorToDealIdHamtCid, - }, nil -} - -type marketMigrator struct { - sectorDealIDs cid.Cid - - OutCodeCID cid.Cid -} - -func (m marketMigrator) MigratedCodeCID() cid.Cid { - return m.OutCodeCID -} - -func (m marketMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, in migration.ActorMigrationInput) (*migration.ActorMigrationResult, error) { - var inState market11.State - if err := store.Get(ctx, in.Head, &inState); err != nil { - return nil, xerrors.Errorf("failed to load miner state for %s: %w", in.Address, err) - } - - newSectorDeals := m.sectorDealIDs - - // Create the new state - outState := market12.State{ - Proposals: inState.Proposals, - States: inState.States, - PendingProposals: inState.PendingProposals, - EscrowTable: inState.EscrowTable, - LockedTable: inState.LockedTable, - NextID: inState.NextID, - DealOpsByEpoch: inState.DealOpsByEpoch, - LastCron: inState.LastCron, - TotalClientLockedCollateral: inState.TotalClientLockedCollateral, - TotalProviderLockedCollateral: inState.TotalProviderLockedCollateral, - TotalClientStorageFee: inState.TotalClientStorageFee, - PendingDealAllocationIds: inState.PendingDealAllocationIds, - SectorDeals: newSectorDeals, // Updated value - } - - newHead, err := store.Put(ctx, &outState) - if err != nil { - return nil, xerrors.Errorf("failed to put new state: %w", err) - } - - return &migration.ActorMigrationResult{ - NewCodeCID: m.MigratedCodeCID(), - NewHead: newHead, - }, nil -} diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 09815fcc..51c7d455 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" + market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" "github.com/filecoin-project/go-state-types/manifest" @@ -122,30 +123,42 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) } - //todo run second migration here for market - migrations2 := make(map[cid.Cid]migration.ActorMigration) + // Load the state of the market actor from v11 for migration purposes. + marketActorV11, ok, err := actorsIn.GetActorV4(builtin.StorageMarketActorAddr) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get market actor: %w", err) + } - // The Market Actor - market12Cid, ok := newManifest.Get(manifest.MarketKey) if !ok { - return cid.Undef, xerrors.Errorf("code cid for market actor not found in new manifest") + return cid.Undef, xerrors.New("didn't find market actor") } - marketMigrator, err := newMarketMigrator(ctx, store, market12Cid, minerMigrator) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to create market migrator: %w", err) + var marketStateV11 market11.State + if err := store.Get(ctx, marketActorV11.Head, &marketStateV11); err != nil { + return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } - migrations2[market11Cid] = migration.CachedMigration(cache, *marketMigrator) - actorsOut2, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsOut, migrations2) + // Retrieve the sector deal IDs from the minerMigrator. These IDs are crucial for the migration of market actor state. + sectorDealIDs, err := minerMigrator.sectorDeals.Map.Root() if err != nil { - return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) + return cid.Undef, err } - outCid, err := actorsOut2.Flush() - if err != nil { - return cid.Undef, xerrors.Errorf("failed to flush actorsOut: %w", err) + // Check if sectorDealIDs is not a zero value before proceeding. + if sectorDealIDs == cid.Undef { + return cid.Undef, xerrors.New("sectorDealIDs is a zero value, cannot proceed with migration") + } + + // Update the market actor in the state tree with the newly fetched sector deal IDs. + // This ensures the market actor's state reflects the most recent sector deals. + if err = actorsOut.SetActorV4(builtin.StorageMarketActorAddr, &builtin.ActorV4{ + Code: marketActorV11.Code, + Head: sectorDealIDs, // Updated value + CallSeqNum: marketActorV11.CallSeqNum, + Balance: marketActorV11.Balance, + }); err != nil { + return cid.Undef, xerrors.Errorf("failed to set market actor: %w", err) } - return outCid, nil + return actorsOut.Flush() } From 3d217c0f7576bb5f503803fd421de8f424c0d230 Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 24 Aug 2023 14:09:57 -1000 Subject: [PATCH 08/63] Rename variables for clarity in market actor migration code --- builtin/v12/migration/top.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 51c7d455..f006b86e 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -124,7 +124,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } // Load the state of the market actor from v11 for migration purposes. - marketActorV11, ok, err := actorsIn.GetActorV4(builtin.StorageMarketActorAddr) + oldMarketActor, ok, err := actorsOut.GetActorV4(builtin.StorageMarketActorAddr) if err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor: %w", err) } @@ -133,8 +133,8 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("didn't find market actor") } - var marketStateV11 market11.State - if err := store.Get(ctx, marketActorV11.Head, &marketStateV11); err != nil { + var oldMarketState market11.State + if err := store.Get(ctx, oldMarketActor.Head, &oldMarketState); err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } @@ -152,10 +152,10 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Update the market actor in the state tree with the newly fetched sector deal IDs. // This ensures the market actor's state reflects the most recent sector deals. if err = actorsOut.SetActorV4(builtin.StorageMarketActorAddr, &builtin.ActorV4{ - Code: marketActorV11.Code, + Code: oldMarketActor.Code, Head: sectorDealIDs, // Updated value - CallSeqNum: marketActorV11.CallSeqNum, - Balance: marketActorV11.Balance, + CallSeqNum: oldMarketActor.CallSeqNum, + Balance: oldMarketActor.Balance, }); err != nil { return cid.Undef, xerrors.Errorf("failed to set market actor: %w", err) } From c4ec6a7080a6f0a065ef7ae43def7776f9fcf54e Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 24 Aug 2023 16:51:39 -1000 Subject: [PATCH 09/63] Save the new market state object to the datastore and set the migrated market object to use this new state --- builtin/v12/migration/top.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index f006b86e..10b3ae66 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -12,6 +12,7 @@ import ( market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" + market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" "github.com/filecoin-project/go-state-types/manifest" "github.com/filecoin-project/go-state-types/migration" ) @@ -149,11 +150,33 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("sectorDealIDs is a zero value, cannot proceed with migration") } + // Create the new state + newMarketState := market12.State{ + Proposals: oldMarketState.Proposals, + States: oldMarketState.States, + PendingProposals: oldMarketState.PendingProposals, + EscrowTable: oldMarketState.EscrowTable, + LockedTable: oldMarketState.LockedTable, + NextID: oldMarketState.NextID, + DealOpsByEpoch: oldMarketState.DealOpsByEpoch, + LastCron: oldMarketState.LastCron, + TotalClientLockedCollateral: oldMarketState.TotalClientLockedCollateral, + TotalProviderLockedCollateral: oldMarketState.TotalProviderLockedCollateral, + TotalClientStorageFee: oldMarketState.TotalClientStorageFee, + PendingDealAllocationIds: oldMarketState.PendingDealAllocationIds, + SectorDeals: sectorDealIDs, // Updated value + } + + newMarketStateCid, err := store.Put(ctx, &newMarketState) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to put new state: %w", err) + } + // Update the market actor in the state tree with the newly fetched sector deal IDs. // This ensures the market actor's state reflects the most recent sector deals. if err = actorsOut.SetActorV4(builtin.StorageMarketActorAddr, &builtin.ActorV4{ Code: oldMarketActor.Code, - Head: sectorDealIDs, // Updated value + Head: newMarketStateCid, // Updated value CallSeqNum: oldMarketActor.CallSeqNum, Balance: oldMarketActor.Balance, }); err != nil { From 2600de7410b86d469dafdc6d2f31e99a483c9bca Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 25 Aug 2023 10:03:22 -1000 Subject: [PATCH 10/63] bugfixes: Update migration logic to use v12 market state - Remove v11 market state import. - Transition from GetActorV4/SetActorV4 to GetActorV5/SetActorV5 methods. - Use v12 market state type for migration. - Add address field when updating the market actor in the state tree. --- builtin/v12/migration/top.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 10b3ae66..4a16ef16 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" - market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" @@ -125,7 +124,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } // Load the state of the market actor from v11 for migration purposes. - oldMarketActor, ok, err := actorsOut.GetActorV4(builtin.StorageMarketActorAddr) + oldMarketActor, ok, err := actorsOut.GetActorV5(builtin.StorageMarketActorAddr) if err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor: %w", err) } @@ -134,7 +133,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("didn't find market actor") } - var oldMarketState market11.State + var oldMarketState market12.State if err := store.Get(ctx, oldMarketActor.Head, &oldMarketState); err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } @@ -174,11 +173,12 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Update the market actor in the state tree with the newly fetched sector deal IDs. // This ensures the market actor's state reflects the most recent sector deals. - if err = actorsOut.SetActorV4(builtin.StorageMarketActorAddr, &builtin.ActorV4{ + if err = actorsOut.SetActorV5(builtin.StorageMarketActorAddr, &builtin.ActorV5{ Code: oldMarketActor.Code, Head: newMarketStateCid, // Updated value CallSeqNum: oldMarketActor.CallSeqNum, Balance: oldMarketActor.Balance, + Address: oldMarketActor.Address, }); err != nil { return cid.Undef, xerrors.Errorf("failed to set market actor: %w", err) } From ca993fe045375f80dc335535c52b4d4c97c93fb9 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 25 Aug 2023 11:23:58 -1000 Subject: [PATCH 11/63] xxx remove migration for testing --- builtin/v12/migration/top.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 4a16ef16..a15a2f6a 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -118,10 +118,15 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) } - actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) - } + /* + + XXX remove migration for testing + + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) + } + */ // Load the state of the market actor from v11 for migration purposes. oldMarketActor, ok, err := actorsOut.GetActorV5(builtin.StorageMarketActorAddr) From 435c9c8262c9ed328649a89f5caf2065a68bb11a Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 25 Aug 2023 11:30:02 -1000 Subject: [PATCH 12/63] Revert "xxx remove migration for testing" This reverts commit ca993fe045375f80dc335535c52b4d4c97c93fb9. --- builtin/v12/migration/top.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index a15a2f6a..4a16ef16 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -118,15 +118,10 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) } - /* - - XXX remove migration for testing - - actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) - } - */ + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) + } // Load the state of the market actor from v11 for migration purposes. oldMarketActor, ok, err := actorsOut.GetActorV5(builtin.StorageMarketActorAddr) From b97547c6bb18a9ab7a00d51508c2cd28eb5e393c Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 25 Aug 2023 11:35:31 -1000 Subject: [PATCH 13/63] bufix: place RunMigration after the initial set up work to find the market actor --- builtin/v12/migration/top.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 4a16ef16..c8988b32 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" + market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" @@ -118,13 +119,8 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) } - actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) - } - // Load the state of the market actor from v11 for migration purposes. - oldMarketActor, ok, err := actorsOut.GetActorV5(builtin.StorageMarketActorAddr) + oldMarketActor, ok, err := actorsIn.GetActorV5(builtin.StorageMarketActorAddr) if err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor: %w", err) } @@ -133,11 +129,16 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("didn't find market actor") } - var oldMarketState market12.State + var oldMarketState market11.State if err := store.Get(ctx, oldMarketActor.Head, &oldMarketState); err != nil { return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) + } + // Retrieve the sector deal IDs from the minerMigrator. These IDs are crucial for the migration of market actor state. sectorDealIDs, err := minerMigrator.sectorDeals.Map.Root() if err != nil { From 1909372e881405de60d4831d7e436939d10b5d83 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 29 Aug 2023 11:15:56 -1000 Subject: [PATCH 14/63] feat(miner): Implement Migration Cache in Miner Migrator - Add migration cache to newMinerMigrator constructor. - Implement sector index caching in migrateSectorsWithCache. - Optimize sector migration by leveraging diffs in cached AMT. - Manage sector-to-deal mappings using a HAMT structure. --- builtin/v12/migration/miner.go | 211 +++++++++++++++++++++++++++------ builtin/v12/migration/top.go | 2 +- migration/util.go | 8 ++ 3 files changed, 186 insertions(+), 35 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 7cf0dd89..1b8a7577 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -34,7 +34,7 @@ type minerMigrator struct { OutCodeCID cid.Cid } -func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid) (*minerMigrator, error) { +func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { ctxStore := adt11.WrapStore(ctx, store) edv11, err := miner11.ConstructDeadline(ctxStore) @@ -70,10 +70,23 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid } - // New mapping of sector IDs to deal IDS, grouped by storage provider. - sectorDeals, err := builtin.NewTree(ctxStore) + //load sector index from datastore, or create a new one if not found + okSectorIndex, prevSectorIndexRoot, err := cache.Read(migration.SectorIndexHamtKey()) if err != nil { - return nil, xerrors.Errorf("creating new state tree: %w", err) + return nil, xerrors.Errorf("failed to get previous sector index from cache: %w", err) + } + var sectorDeals *builtin.ActorTree + if okSectorIndex { + sectorDeals, err = builtin.LoadTree(ctxStore, prevSectorIndexRoot) + if err != nil { + return nil, xerrors.Errorf("reading cached sectorDeals tree: %w", err) + } + } else { + // New mapping of sector IDs to deal IDS, grouped by storage provider. + sectorDeals, err = builtin.NewTree(ctxStore) + if err != nil { + return nil, xerrors.Errorf("creating new state tree: %w", err) + } } return &minerMigrator{ @@ -181,11 +194,18 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. if err != nil { return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } + + okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.SectorIndexPrevSectorsOutKey(minerAddr)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + } + var outRoot cid.Cid + var sectorToDealIdHamtCid cid.Cid - if okIn && okOut { - // we have previous work, but the AMT has changed -- diff them - outRoot, err = migrateSectorsWithDiff(ctx, store, minerAddr, inRoot, prevInRoot, prevOutRoot) + if okIn && okOut && okSectorIndexOut { + // we have previous work -- diff them to identify if there's new work and do the new work + outRoot, sectorToDealIdHamtCid, err = m.migrateSectorsWithDiff(ctx, store, minerAddr, inRoot, prevInRoot, prevOutRoot, prevSectorIndexRoot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } @@ -195,7 +215,8 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. if err != nil { return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } - outArray, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) + + outArray, outDealHamt, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } @@ -203,6 +224,12 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. if err != nil { return cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) } + + sectorToDealIdHamtCid = outDealHamt + } + + if err = cache.Write(migration.SectorIndexPrevSectorsOutKey(minerAddr), sectorToDealIdHamtCid); err != nil { + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { @@ -213,6 +240,9 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } + if err = cache.Write(migration.SectorIndexPrevSectorsOutKey(minerAddr), sectorToDealIdHamtCid); err != nil { + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + } return outRoot, nil }) } @@ -234,60 +264,101 @@ delta = prevInRoot - inRoot merge( prevOutRoot, Migrated(delta) ) */ -func migrateSectorsWithDiff(ctx context.Context, store adt11.Store, minerAddr address.Address, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid) (cid.Cid, error) { - // we have previous work, but the AMT has changed -- diff them + +// todo modify hamt based on diff +func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.Store, minerAddr address.Address, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid, prevSectorIndexRoot cid.Cid) (cid.Cid, cid.Cid, error) { + // we have previous work + // the AMT may or may not have changed -- diff will let us iterate over all the changes diffs, err := amt.Diff(ctx, store, store, prevInRoot, inRoot, amt.UseTreeBitWidth(miner11.SectorsAmtBitwidth)) if err != nil { - return cid.Undef, xerrors.Errorf("failed to diff old and new Sector AMTs: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to diff old and new Sector AMTs: %w", err) } inSectors, err := miner11.LoadSectors(store, inRoot) if err != nil { - return cid.Undef, xerrors.Errorf("failed to load inSectors: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to load inSectors: %w", err) } - prevOutSectors, err := miner12.LoadSectors(store, prevOutRoot) if err != nil { - return cid.Undef, xerrors.Errorf("failed to load prevOutSectors: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to load prevOutSectors: %w", err) + } + + // load previous HAMT sector index for this specific minerAddr + sectorToDealIdHamt, err := builtin.LoadTree(store, prevSectorIndexRoot) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("creating new state tree: %w", err) } for _, change := range diffs { switch change.Type { case amt.Remove: if err := prevOutSectors.Delete(change.Key); err != nil { - return cid.Undef, xerrors.Errorf("failed to delete sector from prevOutSectors: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to delete sector from prevOutSectors: %w", err) } + + //remove sector from HAMT index + err = removeSectorNumberToDealIdFromHAMT(sectorToDealIdHamt, change.Key, store) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to remove sector from HAMT: %w", err) + } + case amt.Add: fallthrough case amt.Modify: sectorNo := abi.SectorNumber(change.Key) info, found, err := inSectors.Get(sectorNo) if err != nil { - return cid.Undef, xerrors.Errorf("failed to get sector %d in inSectors: %w", sectorNo, err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get sector %d in inSectors: %w", sectorNo, err) } if !found { - return cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) + return cid.Undef, cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) } if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info, store)); err != nil { - return cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) + } + + // add sector to the HAMT + err = addSectorNumberToDealIdHAMT(sectorToDealIdHamt, *info, store) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to add sector %d to HAMT: %w", sectorNo, err) } } } - return prevOutSectors.Root() + if len(diffs) > 0 { + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt, minerAddr) + if err != nil { + return cid.Undef, cid.Undef, err + } + } + + //return the hAmt + + prevOutSectorsRoot, err := prevOutSectors.Root() + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of prevOutSectors: %w", err) + } + + ret, err := sectorToDealIdHamt.Map.Root() + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of sectorToDealIdHamt: %w", err) + } + return prevOutSectorsRoot, ret, nil + } -func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, error) { +// return cid of the HAMT for this address and add to the cache outside this fn +func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, cid.Cid, error) { outArray, err := adt12.MakeEmptyArray(store, miner12.SectorsAmtBitwidth) if err != nil { - return nil, xerrors.Errorf("failed to construct new sectors array: %w", err) + return nil, cid.Undef, xerrors.Errorf("failed to construct new sectors array: %w", err) } sectorToDealIdHamt, err := builtin.NewTree(store) if err != nil { - return nil, xerrors.Errorf("creating new state tree: %w", err) + return nil, cid.Undef, xerrors.Errorf("creating new state tree: %w", err) } var sectorInfo miner11.SectorOnChainInfo @@ -297,20 +368,20 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo, store)) }); err != nil { - return nil, err + return nil, cid.Undef, err } sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() if err != nil { - return nil, err + return nil, cid.Undef, err } - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt, minerAddr) if err != nil { - return nil, err + return nil, cid.Undef, err } - return outArray, err + return outArray, sectorToDealIdHamtCid, err } func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, deadlines cid.Cid) (cid.Cid, error) { @@ -347,7 +418,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, var outSnapshotRoot cid.Cid if okIn && okOut { - outSnapshotRoot, err = migrateSectorsWithDiff(ctx, store, minerAddr, inDeadline.SectorsSnapshot, currentInRoot, currentOutRoot) + outSnapshotRoot, err = migrateDeadlineSectorsWithDiff(ctx, store, inDeadline.SectorsSnapshot, currentInRoot, currentOutRoot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } @@ -356,7 +427,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, if err != nil { return cid.Undef, err } - outSnapshot, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inSectorsSnapshot) + outSnapshot, err := migrateDeadlineSectorsFromScratch(ctx, store, inSectorsSnapshot) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors: %w", err) } @@ -434,11 +505,70 @@ func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo, store adt11.Store) } } +func migrateDeadlineSectorsWithDiff(ctx context.Context, store adt11.Store, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid) (cid.Cid, error) { + // we have previous work, but the AMT has changed -- diff them + diffs, err := amt.Diff(ctx, store, store, prevInRoot, inRoot, amt.UseTreeBitWidth(miner11.SectorsAmtBitwidth)) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to diff old and new Sector AMTs: %w", err) + } + + inSectors, err := miner11.LoadSectors(store, inRoot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load inSectors: %w", err) + } + + prevOutSectors, err := miner12.LoadSectors(store, prevOutRoot) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to load prevOutSectors: %w", err) + } + + for _, change := range diffs { + switch change.Type { + case amt.Remove: + if err := prevOutSectors.Delete(change.Key); err != nil { + return cid.Undef, xerrors.Errorf("failed to delete sector from prevOutSectors: %w", err) + } + case amt.Add: + fallthrough + case amt.Modify: + sectorNo := abi.SectorNumber(change.Key) + info, found, err := inSectors.Get(sectorNo) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get sector %d in inSectors: %w", sectorNo, err) + } + + if !found { + return cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) + } + + if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info, store)); err != nil { + return cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) + } + } + } + + return prevOutSectors.Root() +} + +func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, inArray *adt11.Array) (*adt12.Array, error) { + outArray, err := adt12.MakeEmptyArray(store, miner12.SectorsAmtBitwidth) + if err != nil { + return nil, xerrors.Errorf("failed to construct new sectors array: %w", err) + } + + var sectorInfo miner11.SectorOnChainInfo + if err = inArray.ForEach(§orInfo, func(k int64) error { + return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo, store)) + }); err != nil { + return nil, err + } + + return outArray, err +} + func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { //sectorInfo.DealIDs,sectorInfo.SectorNumber - // - todo make SectorNumber and explicit type, not just cast to int64 - // - // - todo for sector use array not CborInt + // - todo confirm types beint stored is correct cborDealIDs := market.SectorDealIDs{DealIDs: sectorInfo.DealIDs} err := xap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &cborDealIDs) @@ -448,10 +578,23 @@ func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.Sect return nil } -func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { +func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uint64, store adt11.Store) error { + err := xap.Map.Delete(abi.IntKey(int64(SectorNumber))) + if err != nil { + return xerrors.Errorf("failed to delete sector from sectorToDealIdHamt index: %w", err) + } + return nil +} +func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt *builtin.ActorTree, minerAddr address.Address) error { + + //todo we are storing the hamt cid but we MUST store the hamt type + hamtCid, err := sectorToDealIdHamt.Map.Root() + if err != nil { + return xerrors.Errorf("adding getting hamtCid: %w", err) + } cborCid := market.HamtCid{Cid: hamtCid} - err := m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), &cborCid) + err = m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), &cborCid) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index c8988b32..0f5a9bf9 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -109,7 +109,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") } - minerMigrator, err := newMinerMigrator(ctx, store, miner12Cid) + minerMigrator, err := newMinerMigrator(ctx, store, miner12Cid, cache) if err != nil { return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err) } diff --git a/migration/util.go b/migration/util.go index 77a9fc08..b0c5dea5 100644 --- a/migration/util.go +++ b/migration/util.go @@ -133,6 +133,14 @@ func ExpirationsAmtKey(expirationsAmt cid.Cid) string { return "partitionsAmt-" + expirationsAmtKey } +func SectorIndexHamtKey() string { + return "sectorIndexHamtKey" +} + +func SectorIndexPrevSectorsOutKey(m address.Address) string { + return "sectorIndexPrevSectorsOut-" + m.String() +} + func MinerPrevSectorsInKey(m address.Address) string { return "prevSectorsIn-" + m.String() } From 0c8a77a2cad20e0fed807a9eb6870ccfa21c693d Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 30 Aug 2023 17:02:29 -1000 Subject: [PATCH 15/63] save to sector index hamt to cache after doing all the work! --- builtin/v12/migration/miner.go | 3 ++- builtin/v12/migration/top.go | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 1b8a7577..5dfe8d3d 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -2,6 +2,7 @@ package migration import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/builtin" miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner" @@ -81,6 +82,7 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid if err != nil { return nil, xerrors.Errorf("reading cached sectorDeals tree: %w", err) } + fmt.Println("Loaded HAMT from cache: ", prevSectorIndexRoot) } else { // New mapping of sector IDs to deal IDS, grouped by storage provider. sectorDeals, err = builtin.NewTree(ctxStore) @@ -227,7 +229,6 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. sectorToDealIdHamtCid = outDealHamt } - if err = cache.Write(migration.SectorIndexPrevSectorsOutKey(minerAddr), sectorToDealIdHamtCid); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 0f5a9bf9..b45b54d5 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -150,6 +150,11 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.New("sectorDealIDs is a zero value, cannot proceed with migration") } + //save sectorDealIdIndex in Cache + if err = cache.Write(migration.SectorIndexHamtKey(), sectorDealIDs); err != nil { + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + } + // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, From c4ef84e73165689058a91957186950510a49921b Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 31 Aug 2023 16:18:15 -1000 Subject: [PATCH 16/63] bugfix: use market11cid for code --- builtin/v12/migration/top.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index b45b54d5..91cff7be 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -180,7 +180,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Update the market actor in the state tree with the newly fetched sector deal IDs. // This ensures the market actor's state reflects the most recent sector deals. if err = actorsOut.SetActorV5(builtin.StorageMarketActorAddr, &builtin.ActorV5{ - Code: oldMarketActor.Code, + Code: market11Cid, Head: newMarketStateCid, // Updated value CallSeqNum: oldMarketActor.CallSeqNum, Balance: oldMarketActor.Balance, From 232b1c827dbbf30bc43be510f4afd9d05d8b8bdc Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 31 Aug 2023 16:19:52 -1000 Subject: [PATCH 17/63] bugfix: use MarketKey for market11cid --- builtin/v12/migration/top.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 91cff7be..897f7598 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -74,7 +74,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID if oldEntry.Name == manifest.MinerKey { miner11Cid = oldEntry.Code } - if oldEntry.Name == manifest.MinerKey { + if oldEntry.Name == manifest.MarketKey { market11Cid = oldEntry.Code } newCodeCID, ok := newManifest.Get(oldEntry.Name) From a785f26ba34380456cd19c6224879581365d3c00 Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 31 Aug 2023 16:29:26 -1000 Subject: [PATCH 18/63] add hamt lock --- builtin/v12/migration/miner.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 5dfe8d3d..b1638626 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -3,6 +3,7 @@ package migration import ( "context" "fmt" + "sync" "github.com/filecoin-project/go-state-types/builtin" miner11 "github.com/filecoin-project/go-state-types/builtin/v11/miner" @@ -33,6 +34,7 @@ type minerMigrator struct { emptyDeadlinesV12 cid.Cid sectorDeals *builtin.ActorTree OutCodeCID cid.Cid + hamtLock sync.Mutex } func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { @@ -588,6 +590,9 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin } func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt *builtin.ActorTree, minerAddr address.Address) error { + m.hamtLock.Lock() + // Lock so only one goroutine at a time can access the map c.v. + defer m.hamtLock.Unlock() //todo we are storing the hamt cid but we MUST store the hamt type hamtCid, err := sectorToDealIdHamt.Map.Root() From b4865b9453db43e15ee4ba6d8d887a3f03a6572a Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 31 Aug 2023 16:38:55 -1000 Subject: [PATCH 19/63] use cbor gen for cid --- builtin/v12/gen/gen.go | 1 - builtin/v12/market/cbor_gen.go | 61 ------------------------------ builtin/v12/market/market_state.go | 4 -- builtin/v12/migration/miner.go | 4 +- 4 files changed, 2 insertions(+), 68 deletions(-) diff --git a/builtin/v12/gen/gen.go b/builtin/v12/gen/gen.go index 938dd57b..999b382b 100644 --- a/builtin/v12/gen/gen.go +++ b/builtin/v12/gen/gen.go @@ -129,7 +129,6 @@ func main() { market.GetDealActivationReturn{}, market.OnMinerSectorsTerminateParams{}, market.SectorDealIDs{}, - market.HamtCid{}, // other types market.DealProposal{}, diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index e1fc04a5..6bb8cf20 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -1891,67 +1891,6 @@ func (t *SectorDealIDs) UnmarshalCBOR(r io.Reader) (err error) { return nil } -var lengthBufHamtCid = []byte{129} - -func (t *HamtCid) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write(lengthBufHamtCid); err != nil { - return err - } - - // t.Cid (cid.Cid) (struct) - - if err := cbg.WriteCid(cw, t.Cid); err != nil { - return xerrors.Errorf("failed to write cid field t.Cid: %w", err) - } - - return nil -} - -func (t *HamtCid) UnmarshalCBOR(r io.Reader) (err error) { - *t = HamtCid{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Cid (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Cid: %w", err) - } - - t.Cid = c - - } - return nil -} - var lengthBufDealProposal = []byte{139} func (t *DealProposal) MarshalCBOR(w io.Writer) error { diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 405b057a..75e60a6d 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -109,10 +109,6 @@ type SectorDealIDs struct { DealIDs []abi.DealID } -type HamtCid struct { - Cid cid.Cid -} - // A specialization of a array to deals. // It is an error to query for a key that doesn't exist. type DealArray struct { diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index b1638626..3cb5d54c 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -19,6 +19,7 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-state-types/abi" ) @@ -599,8 +600,7 @@ func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt *bu if err != nil { return xerrors.Errorf("adding getting hamtCid: %w", err) } - cborCid := market.HamtCid{Cid: hamtCid} - err = m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), &cborCid) + err = m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) From 778d6778e8db8780d2aee6bcf13e24c5ef636830 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 1 Sep 2023 13:33:38 -1000 Subject: [PATCH 20/63] remove duplicate code block --- builtin/v12/migration/miner.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 3cb5d54c..ccf3622f 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -232,10 +232,6 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. sectorToDealIdHamtCid = outDealHamt } - if err = cache.Write(migration.SectorIndexPrevSectorsOutKey(minerAddr), sectorToDealIdHamtCid); err != nil { - return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) - } - if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } From e7a15059513a18b16c32e63e30a35e9efa33f6d6 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 1 Sep 2023 13:41:56 -1000 Subject: [PATCH 21/63] pass lock by reference --- builtin/v12/migration/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index ccf3622f..8818be24 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -35,7 +35,7 @@ type minerMigrator struct { emptyDeadlinesV12 cid.Cid sectorDeals *builtin.ActorTree OutCodeCID cid.Cid - hamtLock sync.Mutex + hamtLock *sync.Mutex } func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { From 676a62c3d5c527a819aad09d257e11d4cf68e3c2 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 1 Sep 2023 14:48:37 -1000 Subject: [PATCH 22/63] Optimize and refactor miner migration logic - Rename `outDealHamt` to `outDealHamtCid` for clarity. - Streamline the function `addSectorToDealIDHamtToSectorDeals` to be called once in `migrateSectorsWithCache`. - Remove redundant calls to `addSectorToDealIDHamtToSectorDeals` in `migrateSectorsFromScratch` and `migrateSectorsWithDiff`. - Pass `cid.Cid` directly to `addSectorToDealIDHamtToSectorDeals` to improve type handling. --- builtin/v12/migration/miner.go | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 8818be24..8e3f8319 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -221,7 +221,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } - outArray, outDealHamt, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) + outArray, outDealHamtCid, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } @@ -230,8 +230,15 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) } - sectorToDealIdHamtCid = outDealHamt + sectorToDealIdHamtCid = outDealHamtCid } + + //add to new address sector deal id hamt index + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to add sector to deal id hamt to for minerAddr to HAMT: %w", err) + } + if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } @@ -327,13 +334,6 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } } - if len(diffs) > 0 { - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt, minerAddr) - if err != nil { - return cid.Undef, cid.Undef, err - } - } - //return the hAmt prevOutSectorsRoot, err := prevOutSectors.Root() @@ -376,11 +376,6 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 return nil, cid.Undef, err } - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt, minerAddr) - if err != nil { - return nil, cid.Undef, err - } - return outArray, sectorToDealIdHamtCid, err } @@ -586,17 +581,12 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin return nil } -func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamt *builtin.ActorTree, minerAddr address.Address) error { +func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { m.hamtLock.Lock() // Lock so only one goroutine at a time can access the map c.v. defer m.hamtLock.Unlock() - //todo we are storing the hamt cid but we MUST store the hamt type - hamtCid, err := sectorToDealIdHamt.Map.Root() - if err != nil { - return xerrors.Errorf("adding getting hamtCid: %w", err) - } - err = m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) + err := m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) From 98270db9dc7ffa1ed09cd50acadac4f17ce7ad63 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:46:10 -1000 Subject: [PATCH 23/63] match naming conventions for MinerPrevSectorDealIndexKey --- builtin/v12/migration/miner.go | 4 ++-- migration/util.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 8e3f8319..dfdb57c5 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -200,7 +200,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } - okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.SectorIndexPrevSectorsOutKey(minerAddr)) + okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(minerAddr)) if err != nil { return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } @@ -247,7 +247,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } - if err = cache.Write(migration.SectorIndexPrevSectorsOutKey(minerAddr), sectorToDealIdHamtCid); err != nil { + if err = cache.Write(migration.MinerPrevSectorDealIndexKey(minerAddr), sectorToDealIdHamtCid); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } return outRoot, nil diff --git a/migration/util.go b/migration/util.go index b0c5dea5..74cfd35b 100644 --- a/migration/util.go +++ b/migration/util.go @@ -137,7 +137,7 @@ func SectorIndexHamtKey() string { return "sectorIndexHamtKey" } -func SectorIndexPrevSectorsOutKey(m address.Address) string { +func MinerPrevSectorDealIndexKey(m address.Address) string { return "sectorIndexPrevSectorsOut-" + m.String() } From da0545ec420e7075268c4b7fbc4c119fa69f6a34 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:47:36 -1000 Subject: [PATCH 24/63] match naming conventions for MarketSectorIndexKey --- builtin/v12/migration/miner.go | 2 +- builtin/v12/migration/top.go | 2 +- migration/util.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index dfdb57c5..9224e0a1 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -75,7 +75,7 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid } //load sector index from datastore, or create a new one if not found - okSectorIndex, prevSectorIndexRoot, err := cache.Read(migration.SectorIndexHamtKey()) + okSectorIndex, prevSectorIndexRoot, err := cache.Read(migration.MarketSectorIndexKey()) if err != nil { return nil, xerrors.Errorf("failed to get previous sector index from cache: %w", err) } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 897f7598..4009ea94 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -151,7 +151,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } //save sectorDealIdIndex in Cache - if err = cache.Write(migration.SectorIndexHamtKey(), sectorDealIDs); err != nil { + if err = cache.Write(migration.MarketSectorIndexKey(), sectorDealIDs); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } diff --git a/migration/util.go b/migration/util.go index 74cfd35b..84e2a3ba 100644 --- a/migration/util.go +++ b/migration/util.go @@ -133,12 +133,12 @@ func ExpirationsAmtKey(expirationsAmt cid.Cid) string { return "partitionsAmt-" + expirationsAmtKey } -func SectorIndexHamtKey() string { - return "sectorIndexHamtKey" +func MarketSectorIndexKey() string { + return "marketSectorIndexKey" } func MinerPrevSectorDealIndexKey(m address.Address) string { - return "sectorIndexPrevSectorsOut-" + m.String() + return "minerPrevSectorDealIndexKey-" + m.String() } func MinerPrevSectorsInKey(m address.Address) string { From d84d29357a794d20f7ad0d9e2907ba836ed546db Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:50:39 -1000 Subject: [PATCH 25/63] remove redundant comment --- builtin/v12/migration/miner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 9224e0a1..102be5da 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -583,7 +583,6 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { m.hamtLock.Lock() - // Lock so only one goroutine at a time can access the map c.v. defer m.hamtLock.Unlock() err := m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) From a697049dbd56488a60c93e66f526e34464618730 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:52:20 -1000 Subject: [PATCH 26/63] remove confusing comment --- builtin/v12/migration/miner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 102be5da..cc5dcc5b 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -349,7 +349,6 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } -// return cid of the HAMT for this address and add to the cache outside this fn func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt11.Store, minerAddr address.Address, inArray *adt11.Array) (*adt12.Array, cid.Cid, error) { outArray, err := adt12.MakeEmptyArray(store, miner12.SectorsAmtBitwidth) if err != nil { From 7c7b46978c56bf7af84b202b2640ba81d17d0c74 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:53:42 -1000 Subject: [PATCH 27/63] remove unused variable in function --- builtin/v12/migration/miner.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index cc5dcc5b..3e758bc4 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -322,7 +322,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S return cid.Undef, cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) } - if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info, store)); err != nil { + if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info)); err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) } @@ -365,7 +365,7 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 err = addSectorNumberToDealIdHAMT(sectorToDealIdHamt, sectorInfo, store) - return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo, store)) + return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) }); err != nil { return nil, cid.Undef, err } @@ -464,7 +464,7 @@ func (m minerMigrator) migrateDeadlines(ctx context.Context, store adt11.Store, return store.Put(ctx, &outDeadlines) } -func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo, store adt11.Store) *miner12.SectorOnChainInfo { +func migrateSectorInfo(sectorInfo miner11.SectorOnChainInfo) *miner12.SectorOnChainInfo { // For a sector that has not been updated: the Activation is correct and ReplacedSectorAge is zero. // For a sector that has been updated through SnapDeals: Activation is the epoch at which it was upgraded, and ReplacedSectorAge is delta since the true activation. // For a sector that has been updated through the old CC path: Activation is correct @@ -535,7 +535,7 @@ func migrateDeadlineSectorsWithDiff(ctx context.Context, store adt11.Store, inRo return cid.Undef, xerrors.Errorf("didn't find sector %d in inSectors", sectorNo) } - if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info, store)); err != nil { + if err := prevOutSectors.Set(change.Key, migrateSectorInfo(*info)); err != nil { return cid.Undef, xerrors.Errorf("failed to set migrated sector %d in prevOutSectors", sectorNo) } } @@ -552,7 +552,7 @@ func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, i var sectorInfo miner11.SectorOnChainInfo if err = inArray.ForEach(§orInfo, func(k int64) error { - return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo, store)) + return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) }); err != nil { return nil, err } From 16b4ec5145087ea2f083c5b73ada93d75c96a8ef Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 10:58:23 -1000 Subject: [PATCH 28/63] epxlain comment --- builtin/v12/migration/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 3e758bc4..4abacc43 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -334,7 +334,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } } - //return the hAmt + //return the sector to deal id Hamt prevOutSectorsRoot, err := prevOutSectors.Root() if err != nil { From ffbf314f8d5ff03b841be2742c1dd402e9ba7be2 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 11:06:19 -1000 Subject: [PATCH 29/63] online the Put command for simplicity --- builtin/v12/migration/miner.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 4abacc43..cc878eba 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -561,11 +561,7 @@ func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, i } func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { - //sectorInfo.DealIDs,sectorInfo.SectorNumber - // - todo confirm types beint stored is correct - - cborDealIDs := market.SectorDealIDs{DealIDs: sectorInfo.DealIDs} - err := xap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &cborDealIDs) + err := xap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) } From 606d9d5c62d58520e52c1a62a98437458be174d6 Mon Sep 17 00:00:00 2001 From: Alex North <445306+anorth@users.noreply.github.com> Date: Wed, 6 Sep 2023 08:47:57 +1000 Subject: [PATCH 30/63] Update deal state schema to match Rust (#215) --- builtin/v12/check.go | 16 ------------- builtin/v12/market/cbor_gen.go | 36 +++++++++++++++++++++++------- builtin/v12/market/deal.go | 1 + builtin/v12/market/invariants.go | 8 ------- builtin/v12/market/market_state.go | 2 +- builtin/v12/migration/top.go | 4 ++-- 6 files changed, 32 insertions(+), 35 deletions(-) diff --git a/builtin/v12/check.go b/builtin/v12/check.go index f7a9df55..b24ba4c0 100644 --- a/builtin/v12/check.go +++ b/builtin/v12/check.go @@ -334,22 +334,6 @@ func CheckVerifregAgainstMiners(acc *builtin.MessageAccumulator, verifregSummary } func CheckMarketAgainstVerifreg(acc *builtin.MessageAccumulator, verifregSummary *verifreg.StateSummary, marketSummary *market.StateSummary) { - // all activated verified deals with claim ids reference a claim in verifreg state - // note that it is possible for claims to exist with no matching deal if the deal expires - for claimId, dealId := range marketSummary.ClaimIdToDealId { - claim, found := verifregSummary.Claims[claimId] - acc.Require(found, "claim %d not found for activated deal %d", claimId, dealId) - - info, found := marketSummary.Deals[dealId] - acc.Require(found, "internal invariant error invalid market state references missing deal %d", dealId) - - providerId, err := address.IDFromAddress(info.Provider) - acc.RequireNoError(err, "error getting ID from provider address") - acc.Require(abi.ActorID(providerId) == claim.Provider, "mismatches providers %d %d on claim %d and deal %d", providerId, claim.Provider, claimId, dealId) - - acc.Require(info.PieceCid == claim.Data, "mismatches piece cid %s %s on claim %d and deal %d", info.PieceCid, claim.Data, claimId, dealId) - } - // all pending deal allocation ids have an associated allocation // note that it is possible for allocations to exist that don't match any deal // if they are created from a direct DataCap transfer diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 6bb8cf20..9bf8dc78 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -108,10 +108,10 @@ func (t *State) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.PendingDealAllocationIds: %w", err) } - // t.SectorDeals (cid.Cid) (struct) + // t.ProviderSectors (cid.Cid) (struct) - if err := cbg.WriteCid(cw, t.SectorDeals); err != nil { - return xerrors.Errorf("failed to write cid field t.SectorDeals: %w", err) + if err := cbg.WriteCid(cw, t.ProviderSectors); err != nil { + return xerrors.Errorf("failed to write cid field t.ProviderSectors: %w", err) } return nil @@ -290,22 +290,22 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { t.PendingDealAllocationIds = c } - // t.SectorDeals (cid.Cid) (struct) + // t.ProviderSectors (cid.Cid) (struct) { c, err := cbg.ReadCid(cr) if err != nil { - return xerrors.Errorf("failed to read cid field t.SectorDeals: %w", err) + return xerrors.Errorf("failed to read cid field t.ProviderSectors: %w", err) } - t.SectorDeals = c + t.ProviderSectors = c } return nil } -var lengthBufDealState = []byte{132} +var lengthBufDealState = []byte{133} func (t *DealState) MarshalCBOR(w io.Writer) error { if t == nil { @@ -319,6 +319,12 @@ func (t *DealState) MarshalCBOR(w io.Writer) error { return err } + // t.SectorNumber (abi.SectorNumber) (uint64) + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { + return err + } + // t.SectorStartEpoch (abi.ChainEpoch) (int64) if t.SectorStartEpoch >= 0 { if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorStartEpoch)); err != nil { @@ -380,10 +386,24 @@ func (t *DealState) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("cbor input should be of type array") } - if extra != 4 { + if extra != 5 { return fmt.Errorf("cbor input had wrong number of fields") } + // t.SectorNumber (abi.SectorNumber) (uint64) + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = abi.SectorNumber(extra) + + } // t.SectorStartEpoch (abi.ChainEpoch) (int64) { maj, extra, err := cr.ReadHeader() diff --git a/builtin/v12/market/deal.go b/builtin/v12/market/deal.go index 73ee872a..4bfca94d 100644 --- a/builtin/v12/market/deal.go +++ b/builtin/v12/market/deal.go @@ -27,6 +27,7 @@ var PieceCIDPrefix = cid.Prefix{ } type DealState struct { + SectorNumber abi.SectorNumber SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated SlashEpoch abi.ChainEpoch // -1 if deal never slashed diff --git a/builtin/v12/market/invariants.go b/builtin/v12/market/invariants.go index cb9acb0b..7fa08840 100644 --- a/builtin/v12/market/invariants.go +++ b/builtin/v12/market/invariants.go @@ -28,7 +28,6 @@ type DealSummary struct { type StateSummary struct { Deals map[abi.DealID]*DealSummary PendingDealAllocationIds map[abi.DealID]verifreg.AllocationId - ClaimIdToDealId map[verifreg.ClaimId]abi.DealID AllocIdToDealId map[verifreg.AllocationId]abi.DealID PendingProposalCount uint64 DealStateCount uint64 @@ -118,7 +117,6 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c } dealStateCount := uint64(0) - claimIdToDealId := make(map[verifreg.ClaimId]abi.DealID) if dealStates, err := adt.AsArray(store, st.States, StatesAmtBitwidth); err != nil { acc.Addf("error loading deal states: %v", err) } else { @@ -156,11 +154,6 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c acc.Require(!found, "deal %d has pending allocation", dealID) dealStateCount++ - - if dealState.VerifiedClaim != verifreg.NoAllocationID { - claimIdToDealId[verifreg.ClaimId(dealState.VerifiedClaim)] = abi.DealID(dealID) - } - return nil }) acc.RequireNoError(err, "error iterating deal states") @@ -266,7 +259,6 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c LockTableCount: lockTableCount, DealOpEpochCount: dealOpEpochCount, DealOpCount: dealOpCount, - ClaimIdToDealId: claimIdToDealId, AllocIdToDealId: allocationIdToDealId, }, acc } diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 75e60a6d..1eb6fb60 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -57,7 +57,7 @@ type State struct { PendingDealAllocationIds cid.Cid // HAMT[DealID]AllocationID // New mapping of sector IDs to deal IDS, grouped by storage provider. - SectorDeals cid.Cid // HAMT[Address]HAMT[SectorNumber]SectorDeals + ProviderSectors cid.Cid // HAMT[Address]HAMT[SectorNumber]SectorDeals } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 4009ea94..78fbb235 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -158,7 +158,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, - States: oldMarketState.States, + States: oldMarketState.States, // FIXME migrate deal states to include sector number PendingProposals: oldMarketState.PendingProposals, EscrowTable: oldMarketState.EscrowTable, LockedTable: oldMarketState.LockedTable, @@ -169,7 +169,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID TotalProviderLockedCollateral: oldMarketState.TotalProviderLockedCollateral, TotalClientStorageFee: oldMarketState.TotalClientStorageFee, PendingDealAllocationIds: oldMarketState.PendingDealAllocationIds, - SectorDeals: sectorDealIDs, // Updated value + ProviderSectors: sectorDealIDs, // Updated value } newMarketStateCid, err := store.Put(ctx, &newMarketState) From dd380ea0169d7d435d7fb3616bc5c6f6a7e47e7e Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 5 Sep 2023 14:07:44 -1000 Subject: [PATCH 31/63] feat: Enhance migration handling with deferredCodeIDs - Introduce deferredCodeIDs to RunMigration to handle actors with deferred migrations. - Add logic in v12 migration to defer specific actor migrations. - Update the market actor's code CID in v12 migration. --- builtin/v11/migration/top.go | 2 +- builtin/v12/migration/top.go | 13 +++++++++++-- migration/runner.go | 6 +++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/builtin/v11/migration/top.go b/builtin/v11/migration/top.go index fb456e38..6b58f44f 100644 --- a/builtin/v11/migration/top.go +++ b/builtin/v11/migration/top.go @@ -124,7 +124,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations), len(oldManifestData.Entries)) } - actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations, deferredCodeIDs) if err != nil { return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 78fbb235..f5456c49 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -73,9 +73,12 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID for _, oldEntry := range oldManifestData.Entries { if oldEntry.Name == manifest.MinerKey { miner11Cid = oldEntry.Code + continue } if oldEntry.Name == manifest.MarketKey { market11Cid = oldEntry.Code + deferredCodeIDs[market11Cid] = struct{}{} + continue } newCodeCID, ok := newManifest.Get(oldEntry.Name) if !ok { @@ -109,6 +112,12 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") } + // The Market Actor + market12Cid, ok := newManifest.Get(manifest.MarketKey) + if !ok { + return cid.Undef, xerrors.Errorf("code cid for market actor not found in new manifest") + } + minerMigrator, err := newMinerMigrator(ctx, store, miner12Cid, cache) if err != nil { return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err) @@ -134,7 +143,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to get market actor state: %w", err) } - actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations, deferredCodeIDs) if err != nil { return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) } @@ -180,7 +189,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Update the market actor in the state tree with the newly fetched sector deal IDs. // This ensures the market actor's state reflects the most recent sector deals. if err = actorsOut.SetActorV5(builtin.StorageMarketActorAddr, &builtin.ActorV5{ - Code: market11Cid, + Code: market12Cid, Head: newMarketStateCid, // Updated value CallSeqNum: oldMarketActor.CallSeqNum, Balance: oldMarketActor.Balance, diff --git a/migration/runner.go b/migration/runner.go index 518b7445..9d54bcfb 100644 --- a/migration/runner.go +++ b/migration/runner.go @@ -16,7 +16,7 @@ import ( "golang.org/x/xerrors" ) -func RunMigration(ctx context.Context, cfg Config, cache MigrationCache, store cbor.IpldStore, log Logger, actorsIn *builtin.ActorTree, migrations map[cid.Cid]ActorMigration) (*builtin.ActorTree, error) { +func RunMigration(ctx context.Context, cfg Config, cache MigrationCache, store cbor.IpldStore, log Logger, actorsIn *builtin.ActorTree, migrations map[cid.Cid]ActorMigration, deferredCodeIDs map[cid.Cid]struct{}) (*builtin.ActorTree, error) { startTime := time.Now() // Setup synchronization @@ -35,6 +35,10 @@ func RunMigration(ctx context.Context, cfg Config, cache MigrationCache, store c if err := actorsIn.ForEachV5(func(addr address.Address, actorIn *builtin.ActorV5) error { actorMigration, ok := migrations[actorIn.Code] if !ok { + _, isDeferred := deferredCodeIDs[actorIn.Code] + if isDeferred { + return nil + } return xerrors.Errorf("actor with code %s has no registered migration function", actorIn.Code) } From ba81ee86b0c9a1b73b4fc69947b0a8456be278f9 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 6 Sep 2023 10:13:00 -1000 Subject: [PATCH 32/63] initailize lock otherwise we have a nil pointer --- builtin/v12/migration/miner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index cc878eba..5a596d6c 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -101,6 +101,7 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid emptyDeadlinesV12: edsv12cid, sectorDeals: sectorDeals, OutCodeCID: outCode, + hamtLock: &sync.Mutex{}, }, nil } From f35b49f094b717af2d0873a580054dd54f1ca13d Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 6 Sep 2023 10:33:59 -1000 Subject: [PATCH 33/63] Refactor minerMigrator to use adt.Map for sectorDeals This commit replaces the previous data structure `builtin.ActorTree` with `adt.Map` for managing sector deals in the V12 migration of the miner actor. This change aligns the minerMigrator struct with the new data structures introduced in V12. - Import adt package from go-state-types/builtin/v12/util/adt - Update newMinerMigrator to initialize sectorDeals with adt.Map - Modify addSectorToDealIDHamtToSectorDeals and MigrateStateTree methods to use methods from adt.Map --- builtin/v12/migration/miner.go | 10 +++++----- builtin/v12/migration/top.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 5a596d6c..490064ff 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -33,7 +33,7 @@ type minerMigrator struct { emptyDeadlinesV11 cid.Cid emptyDeadlineV12 cid.Cid emptyDeadlinesV12 cid.Cid - sectorDeals *builtin.ActorTree + sectorDeals *adt12.Map OutCodeCID cid.Cid hamtLock *sync.Mutex } @@ -79,16 +79,16 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid if err != nil { return nil, xerrors.Errorf("failed to get previous sector index from cache: %w", err) } - var sectorDeals *builtin.ActorTree + var sectorDeals *adt12.Map if okSectorIndex { - sectorDeals, err = builtin.LoadTree(ctxStore, prevSectorIndexRoot) + sectorDeals, err = adt12.AsMap(ctxStore, prevSectorIndexRoot, builtin.DefaultHamtBitwidth) if err != nil { return nil, xerrors.Errorf("reading cached sectorDeals tree: %w", err) } fmt.Println("Loaded HAMT from cache: ", prevSectorIndexRoot) } else { // New mapping of sector IDs to deal IDS, grouped by storage provider. - sectorDeals, err = builtin.NewTree(ctxStore) + sectorDeals, err = adt12.MakeEmptyMap(ctxStore, builtin.DefaultHamtBitwidth) if err != nil { return nil, xerrors.Errorf("creating new state tree: %w", err) } @@ -581,7 +581,7 @@ func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, miner m.hamtLock.Lock() defer m.hamtLock.Unlock() - err := m.sectorDeals.Map.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) + err := m.sectorDeals.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index f5456c49..c6c2ee51 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -149,7 +149,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } // Retrieve the sector deal IDs from the minerMigrator. These IDs are crucial for the migration of market actor state. - sectorDealIDs, err := minerMigrator.sectorDeals.Map.Root() + sectorDealIDs, err := minerMigrator.sectorDeals.Root() if err != nil { return cid.Undef, err } From 1f8f23b67337fe4fa99cc7bdcc3db390bb10b2a1 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 6 Sep 2023 11:14:14 -1000 Subject: [PATCH 34/63] rename mutex to more explicit name --- builtin/v12/migration/miner.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 490064ff..516d6e5d 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -29,13 +29,13 @@ import ( // - #914 fix: Set ActivationEpoch to when the sector was FIRST activated, and PowerBaseEpoch to latest update epoch type minerMigrator struct { - emptyDeadlineV11 cid.Cid - emptyDeadlinesV11 cid.Cid - emptyDeadlineV12 cid.Cid - emptyDeadlinesV12 cid.Cid - sectorDeals *adt12.Map - OutCodeCID cid.Cid - hamtLock *sync.Mutex + emptyDeadlineV11 cid.Cid + emptyDeadlinesV11 cid.Cid + emptyDeadlineV12 cid.Cid + emptyDeadlinesV12 cid.Cid + sectorDeals *adt12.Map + OutCodeCID cid.Cid + marketSectorDealsIndexLock *sync.Mutex } func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { @@ -95,13 +95,13 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid } return &minerMigrator{ - emptyDeadlineV11: edv11cid, - emptyDeadlinesV11: edsv11cid, - emptyDeadlineV12: edv12cid, - emptyDeadlinesV12: edsv12cid, - sectorDeals: sectorDeals, - OutCodeCID: outCode, - hamtLock: &sync.Mutex{}, + emptyDeadlineV11: edv11cid, + emptyDeadlinesV11: edsv11cid, + emptyDeadlineV12: edv12cid, + emptyDeadlinesV12: edsv12cid, + sectorDeals: sectorDeals, + OutCodeCID: outCode, + marketSectorDealsIndexLock: &sync.Mutex{}, }, nil } @@ -578,8 +578,8 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin } func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { - m.hamtLock.Lock() - defer m.hamtLock.Unlock() + m.marketSectorDealsIndexLock.Lock() + defer m.marketSectorDealsIndexLock.Unlock() err := m.sectorDeals.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) From a6c9c22b150a3d4b37afbb804b5ba753b6935411 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 6 Sep 2023 12:04:49 -1000 Subject: [PATCH 35/63] dereference pointer before calling Lock() - not sure why previously compiled --- builtin/v12/migration/miner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 516d6e5d..4acb9707 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -578,8 +578,8 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin } func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { - m.marketSectorDealsIndexLock.Lock() - defer m.marketSectorDealsIndexLock.Unlock() + (*m.marketSectorDealsIndexLock).Lock() + defer (*m.marketSectorDealsIndexLock).Unlock() err := m.sectorDeals.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) From 69de893fb95009e23c98f1192ba5235f54e3ba4f Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 7 Sep 2023 18:23:42 -1000 Subject: [PATCH 36/63] Revert "dereference pointer before calling Lock() - not sure why previously compiled" This reverts commit a6c9c22b150a3d4b37afbb804b5ba753b6935411. --- builtin/v12/migration/miner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 4acb9707..516d6e5d 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -578,8 +578,8 @@ func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uin } func (m minerMigrator) addSectorToDealIDHamtToSectorDeals(hamtCid cid.Cid, minerAddr address.Address) error { - (*m.marketSectorDealsIndexLock).Lock() - defer (*m.marketSectorDealsIndexLock).Unlock() + m.marketSectorDealsIndexLock.Lock() + defer m.marketSectorDealsIndexLock.Unlock() err := m.sectorDeals.Put(abi.IdAddrKey(minerAddr), cbg.CborCid(hamtCid)) From 095e59780cebfff36a9f4c09b5df1c3f11c3f7c4 Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 7 Sep 2023 18:24:51 -1000 Subject: [PATCH 37/63] adding print statements for the miner address -> hamt also for the final cid of the structure to branch origin/mikers/sector_deal_ids_index_cached rename return value for migrateSectorsWithDiff to sectorToDealIdHamtCid --- builtin/v12/migration/miner.go | 6 ++++-- builtin/v12/migration/top.go | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 516d6e5d..7bf90ba7 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -342,11 +342,13 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of prevOutSectors: %w", err) } - ret, err := sectorToDealIdHamt.Map.Root() + sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of sectorToDealIdHamt: %w", err) } - return prevOutSectorsRoot, ret, nil + + fmt.Println("sector to deal id hamt for address: ", minerAddr, " cid: ", sectorToDealIdHamtCid) + return prevOutSectorsRoot, sectorToDealIdHamtCid, nil } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index c6c2ee51..123ac9e4 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -2,6 +2,7 @@ package migration import ( "context" + "fmt" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -180,6 +181,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID PendingDealAllocationIds: oldMarketState.PendingDealAllocationIds, ProviderSectors: sectorDealIDs, // Updated value } + fmt.Println("sector index: ", sectorDealIDs) newMarketStateCid, err := store.Put(ctx, &newMarketState) if err != nil { From 0df532a3e3406ab300a48e4dcd62d83bbea45b29 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Fri, 8 Sep 2023 15:42:55 -0700 Subject: [PATCH 38/63] Enhance migration logic to handle dual CIDs and extend cache functionality - Modify `migrateSectorsWithCache` in `migration/miner.go` to manage two CID returns for more robust sector data migration. - Introduce `Load2` function in `migration/util.go` to the `MigrationCache` interface to support caching two CIDs. - Refactor and organize the related code for clearer logic and error handling. --- builtin/v12/migration/miner.go | 44 ++++++++++++++++------------------ migration/util.go | 29 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 7bf90ba7..e8a66695 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -149,11 +149,17 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i wrappedStore := adt11.WrapStore(ctx, store) - newSectors, err := m.migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) + newSectors, sectorToDealIdHamtCid, err := m.migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) if err != nil { return nil, xerrors.Errorf("failed to migrate sectors for miner: %s: %w", in.Address, err) } + // Add to new address sector deal id hamt index + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, in.Address) + if err != nil { + return nil, xerrors.Errorf("failed to add sector to deal id hamt for minerAddr to HAMT: %w", err) + } + newDeadlines, err := m.migrateDeadlines(ctx, wrappedStore, in.Cache, in.Address, inState.Deadlines) if err != nil { return nil, xerrors.Errorf("failed to migrate deadlines: %w", err) @@ -187,71 +193,61 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i NewHead: newHead, }, nil } - -func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, error) { - return cache.Load(migration.SectorsAmtKey(inRoot), func() (cid.Cid, error) { +func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, cid.Cid, error) { + return cache.Load2(migration.SectorsAmtKey(inRoot), func() (cid.Cid, cid.Cid, error) { okIn, prevInRoot, err := cache.Read(migration.MinerPrevSectorsInKey(minerAddr)) if err != nil { - return cid.Undef, xerrors.Errorf("failed to get previous inRoot from cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous inRoot from cache: %w", err) } okOut, prevOutRoot, err := cache.Read(migration.MinerPrevSectorsOutKey(minerAddr)) if err != nil { - return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(minerAddr)) if err != nil { - return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } var outRoot cid.Cid var sectorToDealIdHamtCid cid.Cid if okIn && okOut && okSectorIndexOut { - // we have previous work -- diff them to identify if there's new work and do the new work outRoot, sectorToDealIdHamtCid, err = m.migrateSectorsWithDiff(ctx, store, minerAddr, inRoot, prevInRoot, prevOutRoot, prevSectorIndexRoot) if err != nil { - return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } } else { - // first time we're doing this, do all the work inArray, err := adt11.AsArray(store, inRoot, miner11.SectorsAmtBitwidth) if err != nil { - return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } outArray, outDealHamtCid, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { - return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } outRoot, err = outArray.Root() if err != nil { - return cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) } sectorToDealIdHamtCid = outDealHamtCid } - - //add to new address sector deal id hamt index - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to add sector to deal id hamt to for minerAddr to HAMT: %w", err) - } - if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { - return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } if err = cache.Write(migration.MinerPrevSectorsOutKey(minerAddr), outRoot); err != nil { - return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } if err = cache.Write(migration.MinerPrevSectorDealIndexKey(minerAddr), sectorToDealIdHamtCid); err != nil { - return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } - return outRoot, nil + return outRoot, sectorToDealIdHamtCid, nil }) } diff --git a/migration/util.go b/migration/util.go index 84e2a3ba..5b5cb581 100644 --- a/migration/util.go +++ b/migration/util.go @@ -40,6 +40,34 @@ func (m *MemMigrationCache) Read(key string) (bool, cid.Cid, error) { return true, c, nil } +func (m *MemMigrationCache) Load2(key string, loadFunc func() (cid.Cid, cid.Cid, error)) (cid.Cid, cid.Cid, error) { + type cacheVal struct { + Cid1 cid.Cid + Cid2 cid.Cid + } + + val, found := m.MigrationMap.Load(key) + if found { + cv, ok := val.(cacheVal) + if !ok { + return cid.Undef, cid.Undef, xerrors.Errorf("non expected type value in cache for Load2") + } + return cv.Cid1, cv.Cid2, nil + } + + cid1, cid2, err := loadFunc() + if err != nil { + return cid.Undef, cid.Undef, err + } + + m.MigrationMap.Store(key, cacheVal{ + Cid1: cid1, + Cid2: cid2, + }) + + return cid1, cid2, nil +} + func (m *MemMigrationCache) Load(key string, loadFunc func() (cid.Cid, error)) (cid.Cid, error) { found, c, err := m.Read(key) if err != nil { @@ -95,6 +123,7 @@ type MigrationCache interface { Write(key string, newCid cid.Cid) error Read(key string) (bool, cid.Cid, error) Load(key string, loadFunc func() (cid.Cid, error)) (cid.Cid, error) + Load2(key string, loadFunc func() (cid.Cid, cid.Cid, error)) (cid.Cid, cid.Cid, error) } func ActorHeadKey(addr address.Address, head cid.Cid) string { From a56af3dbc843dcbaa37fcc371217936c94a3a31e Mon Sep 17 00:00:00 2001 From: mike seiler Date: Sat, 9 Sep 2023 09:20:22 -0700 Subject: [PATCH 39/63] Revert "Enhance migration logic to handle dual CIDs and extend cache functionality" This reverts commit 0df532a3e3406ab300a48e4dcd62d83bbea45b29. --- builtin/v12/migration/miner.go | 44 ++++++++++++++++++---------------- migration/util.go | 29 ---------------------- 2 files changed, 24 insertions(+), 49 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index e8a66695..7bf90ba7 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -149,17 +149,11 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i wrappedStore := adt11.WrapStore(ctx, store) - newSectors, sectorToDealIdHamtCid, err := m.migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) + newSectors, err := m.migrateSectorsWithCache(ctx, wrappedStore, in.Cache, in.Address, inState.Sectors) if err != nil { return nil, xerrors.Errorf("failed to migrate sectors for miner: %s: %w", in.Address, err) } - // Add to new address sector deal id hamt index - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, in.Address) - if err != nil { - return nil, xerrors.Errorf("failed to add sector to deal id hamt for minerAddr to HAMT: %w", err) - } - newDeadlines, err := m.migrateDeadlines(ctx, wrappedStore, in.Cache, in.Address, inState.Deadlines) if err != nil { return nil, xerrors.Errorf("failed to migrate deadlines: %w", err) @@ -193,61 +187,71 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i NewHead: newHead, }, nil } -func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, cid.Cid, error) { - return cache.Load2(migration.SectorsAmtKey(inRoot), func() (cid.Cid, cid.Cid, error) { + +func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11.Store, cache migration.MigrationCache, minerAddr address.Address, inRoot cid.Cid) (cid.Cid, error) { + return cache.Load(migration.SectorsAmtKey(inRoot), func() (cid.Cid, error) { okIn, prevInRoot, err := cache.Read(migration.MinerPrevSectorsInKey(minerAddr)) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous inRoot from cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to get previous inRoot from cache: %w", err) } okOut, prevOutRoot, err := cache.Read(migration.MinerPrevSectorsOutKey(minerAddr)) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(minerAddr)) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } var outRoot cid.Cid var sectorToDealIdHamtCid cid.Cid if okIn && okOut && okSectorIndexOut { + // we have previous work -- diff them to identify if there's new work and do the new work outRoot, sectorToDealIdHamtCid, err = m.migrateSectorsWithDiff(ctx, store, minerAddr, inRoot, prevInRoot, prevOutRoot, prevSectorIndexRoot) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) + return cid.Undef, xerrors.Errorf("failed to migrate sectors from diff: %w", err) } } else { + // first time we're doing this, do all the work inArray, err := adt11.AsArray(store, inRoot, miner11.SectorsAmtBitwidth) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) + return cid.Undef, xerrors.Errorf("failed to read sectors array: %w", err) } outArray, outDealHamtCid, err := m.migrateSectorsFromScratch(ctx, store, minerAddr, inArray) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) + return cid.Undef, xerrors.Errorf("failed to migrate sectors from scratch: %w", err) } outRoot, err = outArray.Root() if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) + return cid.Undef, xerrors.Errorf("error writing new sectors AMT: %w", err) } sectorToDealIdHamtCid = outDealHamtCid } + + //add to new address sector deal id hamt index + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to add sector to deal id hamt to for minerAddr to HAMT: %w", err) + } + if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } if err = cache.Write(migration.MinerPrevSectorsOutKey(minerAddr), outRoot); err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } if err = cache.Write(migration.MinerPrevSectorDealIndexKey(minerAddr), sectorToDealIdHamtCid); err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } - return outRoot, sectorToDealIdHamtCid, nil + return outRoot, nil }) } diff --git a/migration/util.go b/migration/util.go index 5b5cb581..84e2a3ba 100644 --- a/migration/util.go +++ b/migration/util.go @@ -40,34 +40,6 @@ func (m *MemMigrationCache) Read(key string) (bool, cid.Cid, error) { return true, c, nil } -func (m *MemMigrationCache) Load2(key string, loadFunc func() (cid.Cid, cid.Cid, error)) (cid.Cid, cid.Cid, error) { - type cacheVal struct { - Cid1 cid.Cid - Cid2 cid.Cid - } - - val, found := m.MigrationMap.Load(key) - if found { - cv, ok := val.(cacheVal) - if !ok { - return cid.Undef, cid.Undef, xerrors.Errorf("non expected type value in cache for Load2") - } - return cv.Cid1, cv.Cid2, nil - } - - cid1, cid2, err := loadFunc() - if err != nil { - return cid.Undef, cid.Undef, err - } - - m.MigrationMap.Store(key, cacheVal{ - Cid1: cid1, - Cid2: cid2, - }) - - return cid1, cid2, nil -} - func (m *MemMigrationCache) Load(key string, loadFunc func() (cid.Cid, error)) (cid.Cid, error) { found, c, err := m.Read(key) if err != nil { @@ -123,7 +95,6 @@ type MigrationCache interface { Write(key string, newCid cid.Cid) error Read(key string) (bool, cid.Cid, error) Load(key string, loadFunc func() (cid.Cid, error)) (cid.Cid, error) - Load2(key string, loadFunc func() (cid.Cid, cid.Cid, error)) (cid.Cid, cid.Cid, error) } func ActorHeadKey(addr address.Address, head cid.Cid) string { From 465d700d62d376371e57d8d6591f08081bab623e Mon Sep 17 00:00:00 2001 From: mike seiler Date: Sat, 9 Sep 2023 09:35:17 -0700 Subject: [PATCH 40/63] Refactor Miner State Migration Logic - Moved the logic to add sectors to the sector-deal ID HAMT from `migrateSectorsWithCache` to `MigrateState` for better separation of concerns. - Modified `MinerPrevSectorDealIndexKey` function to generate keys using `sectorsAmt` CID instead of `minerAddr` for uniqueness, accounting for potential overlaps between multiple miners with empty AMTs. - Added error handling for potential CID-to-string conversion issues. - Removed duplicated logic and improved caching methods for efficiency. --- builtin/v12/migration/miner.go | 24 ++++++++++++++++-------- migration/util.go | 9 +++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 7bf90ba7..d7e06763 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -154,6 +154,19 @@ func (m minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, i return nil, xerrors.Errorf("failed to migrate sectors for miner: %s: %w", in.Address, err) } + okCacheRead, sectorToDealIdHamtCid, err := in.Cache.Read(migration.MinerPrevSectorDealIndexKey(inState.Sectors)) + if err != nil { + return nil, xerrors.Errorf("failed to read sector to deal id hamt from cache: %w", err) + } + if !okCacheRead { + return nil, xerrors.Errorf("sector to deal id hamt not found in cache for address: %s", in.Address) + } + //add to new address sector deal id hamt index + err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, in.Address) + if err != nil { + return nil, xerrors.Errorf("failed to add sector to deal id hamt to for minerAddr to HAMT: %w", err) + } + newDeadlines, err := m.migrateDeadlines(ctx, wrappedStore, in.Cache, in.Address, inState.Deadlines) if err != nil { return nil, xerrors.Errorf("failed to migrate deadlines: %w", err) @@ -201,7 +214,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } - okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(minerAddr)) + okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(inRoot)) if err != nil { return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } @@ -234,12 +247,6 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. sectorToDealIdHamtCid = outDealHamtCid } - //add to new address sector deal id hamt index - err = m.addSectorToDealIDHamtToSectorDeals(sectorToDealIdHamtCid, minerAddr) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to add sector to deal id hamt to for minerAddr to HAMT: %w", err) - } - if err = cache.Write(migration.MinerPrevSectorsInKey(minerAddr), inRoot); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } @@ -248,7 +255,8 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } - if err = cache.Write(migration.MinerPrevSectorDealIndexKey(minerAddr), sectorToDealIdHamtCid); err != nil { + //use the AMT stateroot and not the miner address because multiple miners can have empty AMTs + if err = cache.Write(migration.MinerPrevSectorDealIndexKey(inRoot), sectorToDealIdHamtCid); err != nil { return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } return outRoot, nil diff --git a/migration/util.go b/migration/util.go index 84e2a3ba..7e5309a6 100644 --- a/migration/util.go +++ b/migration/util.go @@ -137,8 +137,13 @@ func MarketSectorIndexKey() string { return "marketSectorIndexKey" } -func MinerPrevSectorDealIndexKey(m address.Address) string { - return "minerPrevSectorDealIndexKey-" + m.String() +func MinerPrevSectorDealIndexKey(sectorsAmt cid.Cid) string { + sectorsAmtKey, err := sectorsAmt.StringOfBase(multibase.Base32) + if err != nil { + panic(err) + } + + return "minerPrevSectorDealIndexKey-" + sectorsAmtKey } func MinerPrevSectorsInKey(m address.Address) string { From a008d024f687a6fc24fc0ebac27b4961037343a5 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 12:07:59 -0700 Subject: [PATCH 41/63] look up prevInRoot here, not inRoot. We won't have a cached result for inRoot but we should have one for prevInRoot --- builtin/v12/migration/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index d7e06763..abe2ed83 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -214,7 +214,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } - okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(inRoot)) + okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(prevInRoot)) if err != nil { return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) } From 190fe18fab4b1ce540c0a89b4ef13325124dc8b6 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 12:11:13 -0700 Subject: [PATCH 42/63] chain errors --- builtin/v12/migration/top.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 123ac9e4..4fcefc29 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -152,7 +152,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Retrieve the sector deal IDs from the minerMigrator. These IDs are crucial for the migration of market actor state. sectorDealIDs, err := minerMigrator.sectorDeals.Root() if err != nil { - return cid.Undef, err + return cid.Undef, xerrors.Errorf("failed to flush actorsOut: %w", err) } // Check if sectorDealIDs is not a zero value before proceeding. From 56dc952fa14801f5a73870d991ee94a3ef0fe8ed Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 11 Sep 2023 12:11:42 -0700 Subject: [PATCH 43/63] Update builtin/v12/migration/top.go Co-authored-by: Aayush Rajasekaran --- builtin/v12/migration/top.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 123ac9e4..5839413a 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -156,7 +156,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } // Check if sectorDealIDs is not a zero value before proceeding. - if sectorDealIDs == cid.Undef { + if !sectorDealIDs.Defined() { return cid.Undef, xerrors.New("sectorDealIDs is a zero value, cannot proceed with migration") } From 7f7613ba39d1915b5f2201e9d068ca2dca422b65 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:00:44 -0700 Subject: [PATCH 44/63] remove old todo --- builtin/v12/migration/miner.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index abe2ed83..f7b4ec6c 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -280,8 +280,6 @@ delta = prevInRoot - inRoot merge( prevOutRoot, Migrated(delta) ) */ - -// todo modify hamt based on diff func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.Store, minerAddr address.Address, inRoot cid.Cid, prevInRoot cid.Cid, prevOutRoot cid.Cid, prevSectorIndexRoot cid.Cid) (cid.Cid, cid.Cid, error) { // we have previous work // the AMT may or may not have changed -- diff will let us iterate over all the changes From a9745f459e88b38c40012ff0c580dc4c8fbd09dc Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:01:11 -0700 Subject: [PATCH 45/63] fix error message --- builtin/v12/migration/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index f7b4ec6c..0450b689 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -300,7 +300,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S // load previous HAMT sector index for this specific minerAddr sectorToDealIdHamt, err := builtin.LoadTree(store, prevSectorIndexRoot) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("creating new state tree: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("failed to create new state tree: %w", err) } for _, change := range diffs { From 4f851d59c1b66e9a2fb40cede2eb4d648a3e38e3 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:07:00 -0700 Subject: [PATCH 46/63] add error message --- builtin/v12/migration/miner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 0450b689..09b95fa9 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -376,7 +376,8 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) }); err != nil { - return nil, cid.Undef, err + return nil, cid.Undef, xerrors.Errorf("failed to write new sector info: %w", err) + } sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() From 740b22b36dba90b5c109e0e90c0e4985489065b5 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:07:13 -0700 Subject: [PATCH 47/63] better name for hamt --- builtin/v12/migration/miner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 09b95fa9..2bb6ab1d 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -570,16 +570,16 @@ func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, i return outArray, err } -func addSectorNumberToDealIdHAMT(xap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { - err := xap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) +func addSectorNumberToDealIdHAMT(hamtMap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { + err := hamtMap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) } return nil } -func removeSectorNumberToDealIdFromHAMT(xap *builtin.ActorTree, SectorNumber uint64, store adt11.Store) error { - err := xap.Map.Delete(abi.IntKey(int64(SectorNumber))) +func removeSectorNumberToDealIdFromHAMT(hamtMap *builtin.ActorTree, SectorNumber uint64, store adt11.Store) error { + err := hamtMap.Map.Delete(abi.IntKey(int64(SectorNumber))) if err != nil { return xerrors.Errorf("failed to delete sector from sectorToDealIdHamt index: %w", err) } From f7911333ffbbafdb4f567d24353728a76fefaf56 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:36:21 -0700 Subject: [PATCH 48/63] cleanup errors --- builtin/v12/migration/miner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 2bb6ab1d..3effacb1 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -216,7 +216,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. okSectorIndexOut, prevSectorIndexRoot, err := cache.Read(migration.MinerPrevSectorDealIndexKey(prevInRoot)) if err != nil { - return cid.Undef, xerrors.Errorf("failed to get previous outRoot from cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to read sector to deal id hamt from cache: %w", err) } var outRoot cid.Cid @@ -257,7 +257,7 @@ func (m minerMigrator) migrateSectorsWithCache(ctx context.Context, store adt11. //use the AMT stateroot and not the miner address because multiple miners can have empty AMTs if err = cache.Write(migration.MinerPrevSectorDealIndexKey(inRoot), sectorToDealIdHamtCid); err != nil { - return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) + return cid.Undef, xerrors.Errorf("failed to write sector to deal id hamt to cache: %w", err) } return outRoot, nil }) From 0a40559b236d0743dc96eb46faca36915f624ae2 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Mon, 11 Sep 2023 13:38:08 -0700 Subject: [PATCH 49/63] Update HAMT Handling in Miner Migration - Replaced usage of the `builtin.LoadTree`, `builtin.NewTree`, and methods on `builtin.ActorTree` handling with the adt12 package. - Modified HAMT operations such as loading, creating, and getting the root CID to use the new adt12 methods. - Adjusted method parameters to fit the new HAMT functions from adt12. - Refactored addSectorNumberToDealIdHAMT and removeSectorNumberToDealIdFromHAMT functions to use the new HAMT operations for adding and deleting entries. - Streamlined error messages and handling related to HAMT operations for clarity. --- builtin/v12/migration/miner.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 3effacb1..330931f8 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -298,7 +298,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } // load previous HAMT sector index for this specific minerAddr - sectorToDealIdHamt, err := builtin.LoadTree(store, prevSectorIndexRoot) + sectorToDealIdHamt, err := adt12.AsMap(store, prevSectorIndexRoot, builtin.DefaultHamtBitwidth) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to create new state tree: %w", err) } @@ -348,7 +348,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of prevOutSectors: %w", err) } - sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() + sectorToDealIdHamtCid, err := sectorToDealIdHamt.Root() if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to get root of sectorToDealIdHamt: %w", err) } @@ -364,7 +364,7 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 return nil, cid.Undef, xerrors.Errorf("failed to construct new sectors array: %w", err) } - sectorToDealIdHamt, err := builtin.NewTree(store) + sectorToDealIdHamt, err := adt12.MakeEmptyMap(store, builtin.DefaultHamtBitwidth) if err != nil { return nil, cid.Undef, xerrors.Errorf("creating new state tree: %w", err) } @@ -380,7 +380,7 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 } - sectorToDealIdHamtCid, err := sectorToDealIdHamt.Map.Root() + sectorToDealIdHamtCid, err := sectorToDealIdHamt.Root() if err != nil { return nil, cid.Undef, err } @@ -570,16 +570,16 @@ func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, i return outArray, err } -func addSectorNumberToDealIdHAMT(hamtMap *builtin.ActorTree, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { - err := hamtMap.Map.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) +func addSectorNumberToDealIdHAMT(hamtMap *adt12.Map, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { + err := hamtMap.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) } return nil } -func removeSectorNumberToDealIdFromHAMT(hamtMap *builtin.ActorTree, SectorNumber uint64, store adt11.Store) error { - err := hamtMap.Map.Delete(abi.IntKey(int64(SectorNumber))) +func removeSectorNumberToDealIdFromHAMT(hamtMap *adt12.Map, SectorNumber uint64, store adt11.Store) error { + err := hamtMap.Delete(abi.IntKey(int64(SectorNumber))) if err != nil { return xerrors.Errorf("failed to delete sector from sectorToDealIdHamt index: %w", err) } From 869619d4cff6978ec44580f7b90006a44757ce9b Mon Sep 17 00:00:00 2001 From: mike seiler Date: Tue, 12 Sep 2023 15:53:23 -0700 Subject: [PATCH 50/63] remove verifreg.AllocationId based on fip-0076 requirements --- builtin/v12/market/cbor_gen.go | 25 ++----------------------- builtin/v12/market/deal.go | 2 -- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 9bf8dc78..363847e3 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -305,7 +305,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { return nil } -var lengthBufDealState = []byte{133} +var lengthBufDealState = []byte{132} func (t *DealState) MarshalCBOR(w io.Writer) error { if t == nil { @@ -357,13 +357,6 @@ func (t *DealState) MarshalCBOR(w io.Writer) error { return err } } - - // t.VerifiedClaim (verifreg.AllocationId) (uint64) - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.VerifiedClaim)); err != nil { - return err - } - return nil } @@ -386,7 +379,7 @@ func (t *DealState) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -479,20 +472,6 @@ func (t *DealState) UnmarshalCBOR(r io.Reader) (err error) { t.SlashEpoch = abi.ChainEpoch(extraI) } - // t.VerifiedClaim (verifreg.AllocationId) (uint64) - - { - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.VerifiedClaim = verifreg.AllocationId(extra) - - } return nil } diff --git a/builtin/v12/market/deal.go b/builtin/v12/market/deal.go index 4bfca94d..d99d7eaa 100644 --- a/builtin/v12/market/deal.go +++ b/builtin/v12/market/deal.go @@ -15,7 +15,6 @@ import ( addr "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/builtin/v12/verifreg" acrypto "github.com/filecoin-project/go-state-types/crypto" ) @@ -31,7 +30,6 @@ type DealState struct { SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated SlashEpoch abi.ChainEpoch // -1 if deal never slashed - VerifiedClaim verifreg.AllocationId } // The DealLabel is a kinded union of string or byte slice. From eae58d8f99c1e9a3a30fabd5175cd7cf8d2c5d15 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 10:57:49 -0700 Subject: [PATCH 51/63] Enhanced miner migration logic with deal-to-sector indexing - Introduced a new `dealToSectorIndex` map in the `minerMigrator` structure to map deal IDs to sector numbers. - Modified methods to update both the HAMT and the newly introduced deal-to-sector index. - Started the foundation for market state migration logic by setting up placeholders and TODOs. - Introduced the `MarketState` type in `miner_types.go` for better type clarity. Note: More work is needed to complete the market state migration, as indicated by the TODOs. --- builtin/v11/miner/miner_state.go | 1 + builtin/v12/migration/miner.go | 23 +++++++++++++++----- builtin/v12/migration/top.go | 36 ++++++++++++++++++++++++++++++++ builtin/v12/miner/miner_types.go | 3 +++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/builtin/v11/miner/miner_state.go b/builtin/v11/miner/miner_state.go index 4e5c2a57..63c606b5 100644 --- a/builtin/v11/miner/miner_state.go +++ b/builtin/v11/miner/miner_state.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" "github.com/filecoin-project/go-state-types/dline" xc "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index 330931f8..ea0b9c72 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -36,6 +36,7 @@ type minerMigrator struct { sectorDeals *adt12.Map OutCodeCID cid.Cid marketSectorDealsIndexLock *sync.Mutex + dealToSectorIndex *map[uint64]uint64 } func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { @@ -102,6 +103,7 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid sectorDeals: sectorDeals, OutCodeCID: outCode, marketSectorDealsIndexLock: &sync.Mutex{}, + dealToSectorIndex: &map[uint64]uint64{}, }, nil } @@ -311,7 +313,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } //remove sector from HAMT index - err = removeSectorNumberToDealIdFromHAMT(sectorToDealIdHamt, change.Key, store) + err = m.removeSectorNumberToDealIdFromHAMTandDealIdSectorIndex(sectorToDealIdHamt, change.Key, store) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to remove sector from HAMT: %w", err) } @@ -334,7 +336,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } // add sector to the HAMT - err = addSectorNumberToDealIdHAMT(sectorToDealIdHamt, *info, store) + err = m.addSectorNumberToDealIdHAMTandDealIdSectorIndex(sectorToDealIdHamt, *info, store) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to add sector %d to HAMT: %w", sectorNo, err) } @@ -372,7 +374,7 @@ func (m minerMigrator) migrateSectorsFromScratch(ctx context.Context, store adt1 var sectorInfo miner11.SectorOnChainInfo if err = inArray.ForEach(§orInfo, func(k int64) error { - err = addSectorNumberToDealIdHAMT(sectorToDealIdHamt, sectorInfo, store) + err = m.addSectorNumberToDealIdHAMTandDealIdSectorIndex(sectorToDealIdHamt, sectorInfo, store) return outArray.Set(uint64(k), migrateSectorInfo(sectorInfo)) }); err != nil { @@ -570,19 +572,30 @@ func migrateDeadlineSectorsFromScratch(ctx context.Context, store adt11.Store, i return outArray, err } -func addSectorNumberToDealIdHAMT(hamtMap *adt12.Map, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { +func (m minerMigrator) addSectorNumberToDealIdHAMTandDealIdSectorIndex(hamtMap *adt12.Map, sectorInfo miner11.SectorOnChainInfo, store adt11.Store) error { err := hamtMap.Put(abi.IntKey(int64(sectorInfo.SectorNumber)), &market.SectorDealIDs{DealIDs: sectorInfo.DealIDs}) if err != nil { return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) } + for _, dealId := range sectorInfo.DealIDs { + (*m.dealToSectorIndex)[uint64(dealId)] = uint64(sectorInfo.SectorNumber) + } return nil } -func removeSectorNumberToDealIdFromHAMT(hamtMap *adt12.Map, SectorNumber uint64, store adt11.Store) error { +func (m minerMigrator) removeSectorNumberToDealIdFromHAMTandDealIdSectorIndex(hamtMap *adt12.Map, SectorNumber uint64, store adt11.Store) error { err := hamtMap.Delete(abi.IntKey(int64(SectorNumber))) if err != nil { return xerrors.Errorf("failed to delete sector from sectorToDealIdHamt index: %w", err) } + + //note that this is very slow because the index only goes one direction + //todo add a second index?? + for dealId, sectorNum := range *m.dealToSectorIndex { + if sectorNum == SectorNumber { + delete(*m.dealToSectorIndex, dealId) + } + } return nil } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 759a72f7..f85cb764 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -14,6 +14,7 @@ import ( system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" + miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner" "github.com/filecoin-project/go-state-types/manifest" "github.com/filecoin-project/go-state-types/migration" ) @@ -165,6 +166,41 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } + //migrate market.States + marketStates, err := adt11.AsMap(adtStore, oldMarketState.States, builtin.DefaultHamtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to read states property from v11 market state: %w", err) + } + + var marketState miner12.MarketState + _, _ = marketState, marketStates + + //@mikers I need help understanding these types, how to create the varables properly and how to iterate through them and how to update and save them + + //TODO loop through each marketstate.states object and migrate to v12 using the index miner.dealToSectorIndex to update the sector number + // marketStates.ForEach(&marketState, func(k string) error { + // fmt.Println(k) + // return nil + // }) + + //get DealState from v11 which looks like this: + + // type DealState struct { + // SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector + // LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated + // SlashEpoch abi.ChainEpoch // -1 if deal never slashed + // VerifiedClaim verifreg.AllocationId + // } + + // drop VerifiedClaim and find SectorNumber from the miner.dealToSectorIndex based on the key from marketStates and set the value of DealState: + + // newDealState := market12.DealState{ + // SectorNumber abi.SectorNumber, + // SectorStartEpoch: oldDealState.SectorStartEpoch, + // LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, + // SlashEpoch: oldDealState.SlashEpoch, + // } + // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, diff --git a/builtin/v12/miner/miner_types.go b/builtin/v12/miner/miner_types.go index 9f4e4bd4..a568dab4 100644 --- a/builtin/v12/miner/miner_types.go +++ b/builtin/v12/miner/miner_types.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" + adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" "github.com/filecoin-project/go-state-types/builtin/v12/power" "github.com/filecoin-project/go-state-types/builtin/v12/util/adt" "github.com/filecoin-project/go-state-types/builtin/v12/util/smoothing" @@ -261,6 +262,8 @@ type ExpirationQueue struct { quant builtin.QuantSpec } +type MarketState *adt11.Map + // Loads a queue root. // Epochs provided to subsequent method calls will be quantized upwards to quanta mod offsetSeed before being // written to/read from queue entries. From 43bda098cee9358eb385a2e09884dad7845e7787 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 11:52:44 -0700 Subject: [PATCH 52/63] remove delete from m.dealToSectorIndex because of algorithmic tradeoffs to shrinking temporary lookup table --- builtin/v12/migration/miner.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index ea0b9c72..ca1f92ae 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -313,7 +313,7 @@ func (m minerMigrator) migrateSectorsWithDiff(ctx context.Context, store adt11.S } //remove sector from HAMT index - err = m.removeSectorNumberToDealIdFromHAMTandDealIdSectorIndex(sectorToDealIdHamt, change.Key, store) + err = m.removeSectorNumberToDealIdFromHAMT(sectorToDealIdHamt, change.Key, store) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to remove sector from HAMT: %w", err) } @@ -583,19 +583,11 @@ func (m minerMigrator) addSectorNumberToDealIdHAMTandDealIdSectorIndex(hamtMap * return nil } -func (m minerMigrator) removeSectorNumberToDealIdFromHAMTandDealIdSectorIndex(hamtMap *adt12.Map, SectorNumber uint64, store adt11.Store) error { +func (m minerMigrator) removeSectorNumberToDealIdFromHAMT(hamtMap *adt12.Map, SectorNumber uint64, store adt11.Store) error { err := hamtMap.Delete(abi.IntKey(int64(SectorNumber))) if err != nil { return xerrors.Errorf("failed to delete sector from sectorToDealIdHamt index: %w", err) } - - //note that this is very slow because the index only goes one direction - //todo add a second index?? - for dealId, sectorNum := range *m.dealToSectorIndex { - if sectorNum == SectorNumber { - delete(*m.dealToSectorIndex, dealId) - } - } return nil } From f5a73450c2bea556c4d77ab17cf2451ee54a7633 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 12:09:40 -0700 Subject: [PATCH 53/63] Migrate market states to v12 and update deal states - Added imports for v12 market and utilities - Removed unused market and miner v12 imports - Migrated `oldMarketState.States` from v11 to v12: - Iterated through each old market state, used `miner.dealToSectorIndex` to update the sector number. - Created a new ADT array for the deal states in v12 and populated it with the updated deal states. - Replaced the old market state's `States` with the new deal states CID in `newMarketState`. --- builtin/v12/migration/top.go | 59 ++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index f85cb764..0eb3dbd1 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -13,8 +13,9 @@ import ( market11 "github.com/filecoin-project/go-state-types/builtin/v11/market" system11 "github.com/filecoin-project/go-state-types/builtin/v11/system" adt11 "github.com/filecoin-project/go-state-types/builtin/v11/util/adt" + "github.com/filecoin-project/go-state-types/builtin/v12/market" market12 "github.com/filecoin-project/go-state-types/builtin/v12/market" - miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner" + adt12 "github.com/filecoin-project/go-state-types/builtin/v12/util/adt" "github.com/filecoin-project/go-state-types/manifest" "github.com/filecoin-project/go-state-types/migration" ) @@ -167,44 +168,36 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } //migrate market.States - marketStates, err := adt11.AsMap(adtStore, oldMarketState.States, builtin.DefaultHamtBitwidth) + //todo confirm bitwidth + newDealStates, err := adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) if err != nil { - return cid.Undef, xerrors.Errorf("failed to read states property from v11 market state: %w", err) + return cid.Undef, xerrors.Errorf("failed to create new adt empty array newDealStates: %w", err) + } + if dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth); err != nil { + return cid.Undef, xerrors.Errorf("failed to read oldMarketState.States: %w", err) + } else { + var oldDealState market11.DealState + err = dealStates.ForEach(&oldDealState, func(dealID int64) error { + newDealState := market12.DealState{ + SectorNumber: abi.SectorNumber((*minerMigrator.dealToSectorIndex)[uint64(dealID)]), + SectorStartEpoch: oldDealState.SectorStartEpoch, + LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, + SlashEpoch: oldDealState.SlashEpoch, + } + newDealStates.Set(uint64(dealID), &newDealState) + _ = newDealStates + return nil + }) + } + newDealStatesCid, err := newDealStates.Root() + if err != nil { + return cid.Undef, xerrors.Errorf("failed to get root of newDealStatesCid: %w", err) } - var marketState miner12.MarketState - _, _ = marketState, marketStates - - //@mikers I need help understanding these types, how to create the varables properly and how to iterate through them and how to update and save them - - //TODO loop through each marketstate.states object and migrate to v12 using the index miner.dealToSectorIndex to update the sector number - // marketStates.ForEach(&marketState, func(k string) error { - // fmt.Println(k) - // return nil - // }) - - //get DealState from v11 which looks like this: - - // type DealState struct { - // SectorStartEpoch abi.ChainEpoch // -1 if not yet included in proven sector - // LastUpdatedEpoch abi.ChainEpoch // -1 if deal state never updated - // SlashEpoch abi.ChainEpoch // -1 if deal never slashed - // VerifiedClaim verifreg.AllocationId - // } - - // drop VerifiedClaim and find SectorNumber from the miner.dealToSectorIndex based on the key from marketStates and set the value of DealState: - - // newDealState := market12.DealState{ - // SectorNumber abi.SectorNumber, - // SectorStartEpoch: oldDealState.SectorStartEpoch, - // LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, - // SlashEpoch: oldDealState.SlashEpoch, - // } - // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, - States: oldMarketState.States, // FIXME migrate deal states to include sector number + States: newDealStatesCid, PendingProposals: oldMarketState.PendingProposals, EscrowTable: oldMarketState.EscrowTable, LockedTable: oldMarketState.LockedTable, From 4e83d01361ffc490fd77050faa8e62b1d9cfeb82 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 12:14:52 -0700 Subject: [PATCH 54/63] Rename 'States' field to 'DealStates' in v12 market - Updated field name in CBOR serialization and deserialization in `cbor_gen.go`. - Adjusted invariant checks to reference 'DealStates' in `invariants.go`. - Modified type definition in `market_state.go` to reflect the renaming. - Ensured migration logic in `top.go` uses the new 'DealStates' field. --- builtin/v12/market/cbor_gen.go | 12 ++++++------ builtin/v12/market/invariants.go | 2 +- builtin/v12/market/market_state.go | 4 ++-- builtin/v12/migration/top.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 363847e3..3e9fdb33 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -40,10 +40,10 @@ func (t *State) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) } - // t.States (cid.Cid) (struct) + // t.DealStates (cid.Cid) (struct) - if err := cbg.WriteCid(cw, t.States); err != nil { - return xerrors.Errorf("failed to write cid field t.States: %w", err) + if err := cbg.WriteCid(cw, t.DealStates); err != nil { + return xerrors.Errorf("failed to write cid field t.DealStates: %w", err) } // t.PendingProposals (cid.Cid) (struct) @@ -152,16 +152,16 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { t.Proposals = c } - // t.States (cid.Cid) (struct) + // t.DealStates (cid.Cid) (struct) { c, err := cbg.ReadCid(cr) if err != nil { - return xerrors.Errorf("failed to read cid field t.States: %w", err) + return xerrors.Errorf("failed to read cid field t.DealStates: %w", err) } - t.States = c + t.DealStates = c } // t.PendingProposals (cid.Cid) (struct) diff --git a/builtin/v12/market/invariants.go b/builtin/v12/market/invariants.go index 7fa08840..dfb647ce 100644 --- a/builtin/v12/market/invariants.go +++ b/builtin/v12/market/invariants.go @@ -117,7 +117,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c } dealStateCount := uint64(0) - if dealStates, err := adt.AsArray(store, st.States, StatesAmtBitwidth); err != nil { + if dealStates, err := adt.AsArray(store, st.DealStates, StatesAmtBitwidth); err != nil { acc.Addf("error loading deal states: %v", err) } else { var dealState DealState diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 1eb6fb60..54560204 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -26,7 +26,7 @@ type State struct { // States contains state for deals that have been activated and not yet cleaned up after expiry or termination. // After expiration, the state exists until the proposal is cleaned up too. // Invariant: keys(States) ⊆ keys(Proposals). - States cid.Cid // AMT[DealID]DealState + DealStates cid.Cid // AMT[DealID]DealState // PendingProposals tracks dealProposals that have not yet reached their deal start date. // We track them here to ensure that miners can't publish the same deal proposal twice @@ -90,7 +90,7 @@ func ConstructState(store adt.Store) (*State, error) { return &State{ Proposals: emptyProposalsArrayCid, - States: emptyStatesArrayCid, + DealStates: emptyStatesArrayCid, PendingProposals: emptyPendingProposalsMapCid, EscrowTable: emptyBalanceTableCid, LockedTable: emptyBalanceTableCid, diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 0eb3dbd1..44d46d69 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -197,7 +197,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, - States: newDealStatesCid, + DealStates: newDealStatesCid, PendingProposals: oldMarketState.PendingProposals, EscrowTable: oldMarketState.EscrowTable, LockedTable: oldMarketState.LockedTable, From badebde1ff682ceb3b4c37b399349cf9903ce292 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 12:28:20 -0700 Subject: [PATCH 55/63] lotus fails to build these errors: go: downloading google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 chain/actors/builtin/market/v12.go:80:18: s.State.States undefined (type "github.com/filecoin-project/go-state-types/builtin/v12/market".State has no field or method States) chain/actors/builtin/market/v12.go:80:51: otherState12.State.States undefined (type "github.com/filecoin-project/go-state-types/builtin/v12/market".State has no field or method States) chain/actors/builtin/market/v12.go:84:52: s.State.States undefined (type "github.com/filecoin-project/go-state-types/builtin/v12/market".State has no field or method States) chain/actors/builtin/market/v12.go:197:53: v12.VerifiedClaim undefined (type "github.com/filecoin-project/go-state-types/builtin/v12/market".DealState has no field or method VerifiedClaim) make: *** [Makefile:89: lotus] Error 1 Revert "Rename 'States' field to 'DealStates' in v12 market" This reverts commit 4e83d01361ffc490fd77050faa8e62b1d9cfeb82. --- builtin/v12/market/cbor_gen.go | 12 ++++++------ builtin/v12/market/invariants.go | 2 +- builtin/v12/market/market_state.go | 4 ++-- builtin/v12/migration/top.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/builtin/v12/market/cbor_gen.go b/builtin/v12/market/cbor_gen.go index 3e9fdb33..363847e3 100644 --- a/builtin/v12/market/cbor_gen.go +++ b/builtin/v12/market/cbor_gen.go @@ -40,10 +40,10 @@ func (t *State) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) } - // t.DealStates (cid.Cid) (struct) + // t.States (cid.Cid) (struct) - if err := cbg.WriteCid(cw, t.DealStates); err != nil { - return xerrors.Errorf("failed to write cid field t.DealStates: %w", err) + if err := cbg.WriteCid(cw, t.States); err != nil { + return xerrors.Errorf("failed to write cid field t.States: %w", err) } // t.PendingProposals (cid.Cid) (struct) @@ -152,16 +152,16 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { t.Proposals = c } - // t.DealStates (cid.Cid) (struct) + // t.States (cid.Cid) (struct) { c, err := cbg.ReadCid(cr) if err != nil { - return xerrors.Errorf("failed to read cid field t.DealStates: %w", err) + return xerrors.Errorf("failed to read cid field t.States: %w", err) } - t.DealStates = c + t.States = c } // t.PendingProposals (cid.Cid) (struct) diff --git a/builtin/v12/market/invariants.go b/builtin/v12/market/invariants.go index dfb647ce..7fa08840 100644 --- a/builtin/v12/market/invariants.go +++ b/builtin/v12/market/invariants.go @@ -117,7 +117,7 @@ func CheckStateInvariants(st *State, store adt.Store, balance abi.TokenAmount, c } dealStateCount := uint64(0) - if dealStates, err := adt.AsArray(store, st.DealStates, StatesAmtBitwidth); err != nil { + if dealStates, err := adt.AsArray(store, st.States, StatesAmtBitwidth); err != nil { acc.Addf("error loading deal states: %v", err) } else { var dealState DealState diff --git a/builtin/v12/market/market_state.go b/builtin/v12/market/market_state.go index 54560204..1eb6fb60 100644 --- a/builtin/v12/market/market_state.go +++ b/builtin/v12/market/market_state.go @@ -26,7 +26,7 @@ type State struct { // States contains state for deals that have been activated and not yet cleaned up after expiry or termination. // After expiration, the state exists until the proposal is cleaned up too. // Invariant: keys(States) ⊆ keys(Proposals). - DealStates cid.Cid // AMT[DealID]DealState + States cid.Cid // AMT[DealID]DealState // PendingProposals tracks dealProposals that have not yet reached their deal start date. // We track them here to ensure that miners can't publish the same deal proposal twice @@ -90,7 +90,7 @@ func ConstructState(store adt.Store) (*State, error) { return &State{ Proposals: emptyProposalsArrayCid, - DealStates: emptyStatesArrayCid, + States: emptyStatesArrayCid, PendingProposals: emptyPendingProposalsMapCid, EscrowTable: emptyBalanceTableCid, LockedTable: emptyBalanceTableCid, diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 44d46d69..0eb3dbd1 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -197,7 +197,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, - DealStates: newDealStatesCid, + States: newDealStatesCid, PendingProposals: oldMarketState.PendingProposals, EscrowTable: oldMarketState.EscrowTable, LockedTable: oldMarketState.LockedTable, From 88f5efeefd179e0c959a520d541fecd50d2c2123 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Wed, 13 Sep 2023 13:52:41 -0700 Subject: [PATCH 56/63] Refactor dealToSectorIndex to use sync.Map - Modified `dealToSectorIndex` from a pointer to a map to a pointer to a `sync.Map`. - Adjusted relevant methods to use `Load` and `Store` from `sync.Map` for concurrency safety. - Updated logic in `MigrateStateTree` to handle loading sector numbers using the `sync.Map`. --- builtin/v12/migration/miner.go | 6 +++--- builtin/v12/migration/top.go | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/builtin/v12/migration/miner.go b/builtin/v12/migration/miner.go index ca1f92ae..e238f9f8 100644 --- a/builtin/v12/migration/miner.go +++ b/builtin/v12/migration/miner.go @@ -36,7 +36,7 @@ type minerMigrator struct { sectorDeals *adt12.Map OutCodeCID cid.Cid marketSectorDealsIndexLock *sync.Mutex - dealToSectorIndex *map[uint64]uint64 + dealToSectorIndex *sync.Map } func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid, cache migration.MigrationCache) (*minerMigrator, error) { @@ -103,7 +103,7 @@ func newMinerMigrator(ctx context.Context, store cbor.IpldStore, outCode cid.Cid sectorDeals: sectorDeals, OutCodeCID: outCode, marketSectorDealsIndexLock: &sync.Mutex{}, - dealToSectorIndex: &map[uint64]uint64{}, + dealToSectorIndex: &sync.Map{}, }, nil } @@ -578,7 +578,7 @@ func (m minerMigrator) addSectorNumberToDealIdHAMTandDealIdSectorIndex(hamtMap * return xerrors.Errorf("adding sector number and deal ids to state tree: %w", err) } for _, dealId := range sectorInfo.DealIDs { - (*m.dealToSectorIndex)[uint64(dealId)] = uint64(sectorInfo.SectorNumber) + m.dealToSectorIndex.Store(uint64(dealId), uint64(sectorInfo.SectorNumber)) } return nil } diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 0eb3dbd1..94b28de6 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -178,8 +178,12 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } else { var oldDealState market11.DealState err = dealStates.ForEach(&oldDealState, func(dealID int64) error { + sectorNumber, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) + if !ok { + return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) + } newDealState := market12.DealState{ - SectorNumber: abi.SectorNumber((*minerMigrator.dealToSectorIndex)[uint64(dealID)]), + SectorNumber: abi.SectorNumber(sectorNumber.(uint64)), SectorStartEpoch: oldDealState.SectorStartEpoch, LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, SlashEpoch: oldDealState.SlashEpoch, From d7316661cc15e6a02aa1495ac02f414e99949147 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 15:04:35 -0700 Subject: [PATCH 57/63] Refactor migration logic to incorporate caching previous deal states - Implement caching mechanism for previous market deal states. - Enhance deal state migration logic to prioritize cached states. - Add new utility function for previous market states key. --- builtin/v12/migration/top.go | 41 ++++++++++++++++++++++++++++++++---- migration/util.go | 4 ++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 94b28de6..d188bba4 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -167,6 +167,17 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to write inkey to cache: %w", err) } + var prevDealStates *adt12.Array + prevDealStatesOk, prevDealStatesCid, err := cache.Read(migration.PrevMarketStatesAmtKey()) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to read prev market states amt key from cache: %w", err) + } + if prevDealStatesOk { + prevDealStates, err = adt12.AsArray(adtStore, prevDealStatesCid, market.StatesAmtBitwidth) + } else { + prevDealStates, err = adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) + } + //migrate market.States //todo confirm bitwidth newDealStates, err := adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) @@ -178,12 +189,30 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } else { var oldDealState market11.DealState err = dealStates.ForEach(&oldDealState, func(dealID int64) error { - sectorNumber, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) - if !ok { - return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) + var prevRunDealState market12.DealState + var sectorNumber uint64 + found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) + if err != nil { + return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) + + } + if found { + // we have found this deal id in our previous deal state amt, so extract the sector Number from there + sectorNumber = uint64(prevRunDealState.SectorNumber) + } else { + // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error + sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) + if !ok { + return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) + } + sectorNumber, ok = sectorNumberAny.(uint64) + if !ok { + return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) + } } + newDealState := market12.DealState{ - SectorNumber: abi.SectorNumber(sectorNumber.(uint64)), + SectorNumber: abi.SectorNumber(sectorNumber), SectorStartEpoch: oldDealState.SectorStartEpoch, LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, SlashEpoch: oldDealState.SlashEpoch, @@ -198,6 +227,10 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("failed to get root of newDealStatesCid: %w", err) } + if err = cache.Write(migration.PrevMarketStatesAmtKey(), newDealStatesCid); err != nil { + return cid.Undef, xerrors.Errorf("failed to write prev market states amt key to cache: %w", err) + } + // Create the new state newMarketState := market12.State{ Proposals: oldMarketState.Proposals, diff --git a/migration/util.go b/migration/util.go index 7e5309a6..d810d678 100644 --- a/migration/util.go +++ b/migration/util.go @@ -137,6 +137,10 @@ func MarketSectorIndexKey() string { return "marketSectorIndexKey" } +func PrevMarketStatesAmtKey() string { + return "prevMarketStatesAmtOutput" +} + func MinerPrevSectorDealIndexKey(sectorsAmt cid.Cid) string { sectorsAmtKey, err := sectorsAmt.StringOfBase(multibase.Base32) if err != nil { From 99548cafdea8f8275c66c7e8c09e46c2c1e9a67b Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 15:12:04 -0700 Subject: [PATCH 58/63] handle errors --- builtin/v12/migration/top.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index d188bba4..0c58cfdf 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -174,8 +174,14 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } if prevDealStatesOk { prevDealStates, err = adt12.AsArray(adtStore, prevDealStatesCid, market.StatesAmtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to convert prevDealStatesCid to array: %w", err) + } } else { prevDealStates, err = adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create empty array for prevDealStates: %w", err) + } } //migrate market.States @@ -217,8 +223,9 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, SlashEpoch: oldDealState.SlashEpoch, } - newDealStates.Set(uint64(dealID), &newDealState) - _ = newDealStates + if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { + return xerrors.Errorf("failed to set new deal state: %w", err) + } return nil }) } From 6856ec71213454cfcf5322dc16d9ab4eba726792 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 15:16:10 -0700 Subject: [PATCH 59/63] add err handler and minor refactor formatting if statements --- builtin/v12/migration/top.go | 74 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 0c58cfdf..60338c33 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -184,50 +184,52 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } } - //migrate market.States - //todo confirm bitwidth + // Migrate market.States + // TODO Confirm bitwidth newDealStates, err := adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new adt empty array newDealStates: %w", err) } - if dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth); err != nil { + dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth) + if err != nil { return cid.Undef, xerrors.Errorf("failed to read oldMarketState.States: %w", err) - } else { - var oldDealState market11.DealState - err = dealStates.ForEach(&oldDealState, func(dealID int64) error { - var prevRunDealState market12.DealState - var sectorNumber uint64 - found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) - if err != nil { - return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) - + } + var oldDealState market11.DealState + err = dealStates.ForEach(&oldDealState, func(dealID int64) error { + var prevRunDealState market12.DealState + var sectorNumber uint64 + found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) + if err != nil { + return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) + } + if found { + // We have found this deal id in our previous deal state amt, so extract the sector Number from there + sectorNumber = uint64(prevRunDealState.SectorNumber) + } else { + // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error + sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) + if !ok { + return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) } - if found { - // we have found this deal id in our previous deal state amt, so extract the sector Number from there - sectorNumber = uint64(prevRunDealState.SectorNumber) - } else { - // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error - sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) - if !ok { - return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) - } - sectorNumber, ok = sectorNumberAny.(uint64) - if !ok { - return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) - } + sectorNumber, ok = sectorNumberAny.(uint64) + if !ok { + return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) } + } - newDealState := market12.DealState{ - SectorNumber: abi.SectorNumber(sectorNumber), - SectorStartEpoch: oldDealState.SectorStartEpoch, - LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, - SlashEpoch: oldDealState.SlashEpoch, - } - if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { - return xerrors.Errorf("failed to set new deal state: %w", err) - } - return nil - }) + newDealState := market12.DealState{ + SectorNumber: abi.SectorNumber(sectorNumber), + SectorStartEpoch: oldDealState.SectorStartEpoch, + LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, + SlashEpoch: oldDealState.SlashEpoch, + } + if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { + return xerrors.Errorf("failed to set new deal state: %w", err) + } + return nil + }) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to iterate over deal states: %w", err) } newDealStatesCid, err := newDealStates.Root() if err != nil { From 143bddb6a04a48cc8f014e7435cedfab48080046 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 16:33:20 -0700 Subject: [PATCH 60/63] Revert "add err handler and minor refactor formatting if statements" This reverts commit 6856ec71213454cfcf5322dc16d9ab4eba726792. --- builtin/v12/migration/top.go | 74 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 60338c33..0c58cfdf 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -184,52 +184,50 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } } - // Migrate market.States - // TODO Confirm bitwidth + //migrate market.States + //todo confirm bitwidth newDealStates, err := adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) if err != nil { return cid.Undef, xerrors.Errorf("failed to create new adt empty array newDealStates: %w", err) } - dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth) - if err != nil { + if dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth); err != nil { return cid.Undef, xerrors.Errorf("failed to read oldMarketState.States: %w", err) - } - var oldDealState market11.DealState - err = dealStates.ForEach(&oldDealState, func(dealID int64) error { - var prevRunDealState market12.DealState - var sectorNumber uint64 - found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) - if err != nil { - return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) - } - if found { - // We have found this deal id in our previous deal state amt, so extract the sector Number from there - sectorNumber = uint64(prevRunDealState.SectorNumber) - } else { - // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error - sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) - if !ok { - return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) + } else { + var oldDealState market11.DealState + err = dealStates.ForEach(&oldDealState, func(dealID int64) error { + var prevRunDealState market12.DealState + var sectorNumber uint64 + found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) + if err != nil { + return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) + } - sectorNumber, ok = sectorNumberAny.(uint64) - if !ok { - return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) + if found { + // we have found this deal id in our previous deal state amt, so extract the sector Number from there + sectorNumber = uint64(prevRunDealState.SectorNumber) + } else { + // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error + sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) + if !ok { + return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) + } + sectorNumber, ok = sectorNumberAny.(uint64) + if !ok { + return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) + } } - } - newDealState := market12.DealState{ - SectorNumber: abi.SectorNumber(sectorNumber), - SectorStartEpoch: oldDealState.SectorStartEpoch, - LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, - SlashEpoch: oldDealState.SlashEpoch, - } - if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { - return xerrors.Errorf("failed to set new deal state: %w", err) - } - return nil - }) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to iterate over deal states: %w", err) + newDealState := market12.DealState{ + SectorNumber: abi.SectorNumber(sectorNumber), + SectorStartEpoch: oldDealState.SectorStartEpoch, + LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, + SlashEpoch: oldDealState.SlashEpoch, + } + if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { + return xerrors.Errorf("failed to set new deal state: %w", err) + } + return nil + }) } newDealStatesCid, err := newDealStates.Root() if err != nil { From f197c104c0343874ac3765bf01af44433fcced1a Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 16:36:16 -0700 Subject: [PATCH 61/63] add err handler --- builtin/v12/migration/top.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 0c58cfdf..3558439a 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -228,6 +228,10 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID } return nil }) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to iterate over deal states: %w", err) + } + } newDealStatesCid, err := newDealStates.Root() if err != nil { From 6ae71c15d6215c24923cef07e3a47cbba0f5e241 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Thu, 14 Sep 2023 21:27:18 -0700 Subject: [PATCH 62/63] prints for debugging --- builtin/v12/migration/top.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 3558439a..26f8d192 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -169,15 +169,20 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID var prevDealStates *adt12.Array prevDealStatesOk, prevDealStatesCid, err := cache.Read(migration.PrevMarketStatesAmtKey()) + fmt.Println("prevDealStatesCid: ", prevDealStatesCid.String()) + if err != nil { return cid.Undef, xerrors.Errorf("failed to read prev market states amt key from cache: %w", err) } if prevDealStatesOk { + fmt.Println("Loading prevDealStatesCid") prevDealStates, err = adt12.AsArray(adtStore, prevDealStatesCid, market.StatesAmtBitwidth) if err != nil { return cid.Undef, xerrors.Errorf("failed to convert prevDealStatesCid to array: %w", err) } } else { + fmt.Println("creating empty array for prevDealStatesCid") + prevDealStates, err = adt12.MakeEmptyArray(adtStore, market.StatesAmtBitwidth) if err != nil { return cid.Undef, xerrors.Errorf("failed to create empty array for prevDealStates: %w", err) From efc66d1af7319b9acea94f2341c194c40df1d382 Mon Sep 17 00:00:00 2001 From: mike seiler Date: Fri, 15 Sep 2023 08:11:42 -0700 Subject: [PATCH 63/63] adding more print statements for debugging and cleaning up an if statement --- builtin/v12/migration/top.go | 76 +++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/builtin/v12/migration/top.go b/builtin/v12/migration/top.go index 26f8d192..bfb6bf4b 100644 --- a/builtin/v12/migration/top.go +++ b/builtin/v12/migration/top.go @@ -195,53 +195,57 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID if err != nil { return cid.Undef, xerrors.Errorf("failed to create new adt empty array newDealStates: %w", err) } - if dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth); err != nil { + dealStates, err := adt11.AsArray(adtStore, oldMarketState.States, market11.StatesAmtBitwidth) + if err != nil { return cid.Undef, xerrors.Errorf("failed to read oldMarketState.States: %w", err) - } else { - var oldDealState market11.DealState - err = dealStates.ForEach(&oldDealState, func(dealID int64) error { - var prevRunDealState market12.DealState - var sectorNumber uint64 - found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) - if err != nil { - return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) - - } - if found { - // we have found this deal id in our previous deal state amt, so extract the sector Number from there - sectorNumber = uint64(prevRunDealState.SectorNumber) - } else { - // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error - sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) - if !ok { - return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) - } - sectorNumber, ok = sectorNumberAny.(uint64) - if !ok { - return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) - } - } + } + fmt.Println("looping through dealStates") + var oldDealState market11.DealState + err = dealStates.ForEach(&oldDealState, func(dealID int64) error { + fmt.Println("dealID:", dealID) + var prevRunDealState market12.DealState + var sectorNumber uint64 + found, err := prevDealStates.Get(uint64(dealID), &prevRunDealState) + if err != nil { + return xerrors.Errorf("failed to load sector number from previous deal states AMT: %d", dealID) - newDealState := market12.DealState{ - SectorNumber: abi.SectorNumber(sectorNumber), - SectorStartEpoch: oldDealState.SectorStartEpoch, - LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, - SlashEpoch: oldDealState.SlashEpoch, + } + if found { + // we have found this deal id in our previous deal state amt, so extract the sector Number from there + sectorNumber = uint64(prevRunDealState.SectorNumber) + } else { + // Deal ID not found in previous run array, so it must be in dealToSectorIndex or error + sectorNumberAny, ok := (*minerMigrator.dealToSectorIndex).Load(uint64(dealID)) + if !ok { + return xerrors.Errorf("failed to load sector number for deal ID: %d", dealID) } - if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { - return xerrors.Errorf("failed to set new deal state: %w", err) + sectorNumber, ok = sectorNumberAny.(uint64) + if !ok { + return xerrors.Errorf("failed to assert sectorNumberUint64 to uint64 for deal ID: %d", dealID) } - return nil - }) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to iterate over deal states: %w", err) } + newDealState := market12.DealState{ + SectorNumber: abi.SectorNumber(sectorNumber), + SectorStartEpoch: oldDealState.SectorStartEpoch, + LastUpdatedEpoch: oldDealState.LastUpdatedEpoch, + SlashEpoch: oldDealState.SlashEpoch, + } + if err := newDealStates.Set(uint64(dealID), &newDealState); err != nil { + return xerrors.Errorf("failed to set new deal state: %w", err) + } + return nil + }) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to iterate over deal states: %w", err) } + newDealStatesCid, err := newDealStates.Root() + fmt.Println("newDealStatesCid is:", newDealStatesCid.String()) if err != nil { return cid.Undef, xerrors.Errorf("failed to get root of newDealStatesCid: %w", err) } + fmt.Println("saving newDealStatesCid:", newDealStatesCid.String()) if err = cache.Write(migration.PrevMarketStatesAmtKey(), newDealStatesCid); err != nil { return cid.Undef, xerrors.Errorf("failed to write prev market states amt key to cache: %w", err)