feat: add method for finding oldest state and computing state (#1038)

* feat: add method for finding oldest state and computing state * chore: Some UX around the `chain state-*` commands (#1042) * chore: Refactor state list commands into one; Output JSON * chore: Add docs and prefixes * chore: fmt the go Co-authored-by: Mike Greenberg <[email protected]> * fixup: remove unused flag and progbar setting Co-authored-by: Mike Greenberg <[email protected]> Co-authored-by: Mike Greenberg <[email protected]>
filecoin-project · Aug 24, 2022 · 593dad6 · 593dad6
1 parent 2c06b9b
commit 593dad6
Show file tree

Hide file tree

Showing 4 changed files with 319 additions and 14 deletions.
diff --git a/commands/chain.go b/commands/chain.go
@@ -17,6 +17,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/urfave/cli/v2"
+	"gopkg.in/cheggaaa/pb.v1"
 
 	"github.com/filecoin-project/lily/chain/actors"
 	init_ "github.com/filecoin-project/lily/chain/actors/builtin/init"
@@ -42,6 +43,9 @@ var ChainCmd = &cli.Command{
 		ChainListCmd,
 		ChainSetHeadCmd,
 		ChainActorCodesCmd,
+		ChainStateInspect,
+		ChainStateCompute,
+		ChainStateComputeRange,
 	},
 }
 
@@ -74,24 +78,188 @@ var ChainActorCodesCmd = &cli.Command{
 	},
 }
 
-func printSortedActorVersions(av map[actors.Version]cid.Cid) error {
-	t := table.NewWriter()
-	t.AppendHeader(table.Row{"Version", "Name", "Family", "Code"})
-	var versions []int
-	for v := range av {
-		versions = append(versions, int(v))
+func marshalReport(reports []*lily.StateReport, verbose bool) ([]byte, error) {
+	type stateHeights struct {
+		Newest int64 `json:"newest"`
+		Oldest int64 `json:"oldest"`
 	}
-	sort.Ints(versions)
-	for _, v := range versions {
-		name, family, err := util.ActorNameAndFamilyFromCode(av[actors.Version(v)])
+	type summarizedHeights struct {
+		Messages   stateHeights `json:"messages"`
+		StateRoots stateHeights `json:"stateroots"`
+	}
+	type hasState struct {
+		Messages  bool `json:"messages"`
+		Receipts  bool `json:"receipts"`
+		StateRoot bool `json:"stateroot"`
+	}
+	type stateReport struct {
+		Summary summarizedHeights  `json:"summary"`
+		Detail  map[int64]hasState `json:"details,omitempty"`
+	}
+
+	var (
+		details         = make(map[int64]hasState)
+		headSet         bool
+		head            = reports[0]
+		oldestMessage   = &lily.StateReport{}
+		oldestStateRoot = &lily.StateReport{}
+	)
+
+	for _, r := range reports {
+		if verbose {
+			details[r.Height] = hasState{
+				Messages:  r.HasMessages,
+				Receipts:  r.HasReceipts,
+				StateRoot: r.HasState,
+			}
+		}
+		if !headSet && (r.HasState && r.HasMessages && r.HasReceipts) {
+			head = r
+			headSet = true
+		}
+		if r.HasState {
+			oldestStateRoot = r
+		}
+		if r.HasMessages {
+			oldestMessage = r
+		}
+	}
+
+	compiledReport := stateReport{
+		Detail: details,
+		Summary: summarizedHeights{
+			Messages:   stateHeights{Newest: head.Height, Oldest: oldestMessage.Height},
+			StateRoots: stateHeights{Newest: head.Height, Oldest: oldestStateRoot.Height},
+		},
+	}
+
+	reportOut, err := json.Marshal(compiledReport)
+	if err != nil {
+		return nil, err
+	}
+
+	return reportOut, nil
+}
+
+var ChainStateInspect = &cli.Command{
+	Name:  "state-inspect",
+	Usage: "Returns details about each epoch's state in the local datastore",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:    "limit",
+			Aliases: []string{"l"},
+			Value:   100,
+			Usage:   "Limit traversal of statetree when searching for oldest state by `N` heights starting from most recent",
+		},
+		&cli.BoolFlag{
+			Name:    "verbose",
+			Aliases: []string{"v"},
+			Usage:   "Include detailed information about the completeness of state for all traversed height(s) starting from most recent",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lotuscli.ReqContext(cctx)
+		lapi, closer, err := GetAPI(ctx)
 		if err != nil {
 			return err
 		}
-		t.AppendRow(table.Row{v, name, family, av[actors.Version(v)]})
-		t.AppendSeparator()
-	}
-	fmt.Println(t.Render())
-	return nil
+		defer closer()
+
+		report, err := lapi.FindOldestState(ctx, cctx.Int64("limit"))
+		if err != nil {
+			return err
+		}
+		sort.Slice(report, func(i, j int) bool {
+			return report[i].Height > report[j].Height
+		})
+
+		out, err := marshalReport(report, cctx.Bool("verbose"))
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(out))
+		return nil
+	},
+}
+
+var ChainStateComputeRange = &cli.Command{
+	Name:  "state-compute",
+	Usage: "Generates the state at epoch `N`",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:     "epoch",
+			Aliases:  []string{"e"},
+			Required: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lotuscli.ReqContext(cctx)
+		lapi, closer, err := GetAPI(ctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		head, err := lapi.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+		ts, err := lapi.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(cctx.Uint64("epoch")), head.Key())
+		if err != nil {
+			return err
+		}
+
+		_, err = lapi.StateCompute(ctx, ts.Key())
+		return err
+
+	},
+}
+
+var ChainStateCompute = &cli.Command{
+	Name:  "state-compute-range",
+	Usage: "Generates the state from epoch `FROM` to epoch `TO`",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:     "from",
+			Required: true,
+		},
+		&cli.Uint64Flag{
+			Name:     "to",
+			Required: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lotuscli.ReqContext(cctx)
+		lapi, closer, err := GetAPI(ctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		head, err := lapi.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+		bar := pb.StartNew(int(cctx.Uint64("to") - cctx.Uint64("from")))
+		bar.ShowTimeLeft = true
+		bar.ShowPercent = true
+		bar.Units = pb.U_NO
+		for i := cctx.Int64("from"); i <= cctx.Int64("to"); i++ {
+			ts, err := lapi.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(i), head.Key())
+			if err != nil {
+				return err
+			}
+
+			_, err = lapi.StateCompute(ctx, ts.Key())
+			if err != nil {
+				return err
+			}
+			bar.Add(1)
+		}
+		bar.Finish()
+		return nil
+
+	},
 }
 
 var ChainHeadCmd = &cli.Command{
@@ -547,3 +715,23 @@ func parseTipSet(ctx context.Context, api lily.LilyAPI, vals []string) (*types.T
 
 	return types.NewTipSet(headers)
 }
+
+func printSortedActorVersions(av map[actors.Version]cid.Cid) error {
+	t := table.NewWriter()
+	t.AppendHeader(table.Row{"Version", "Name", "Family", "Code"})
+	var versions []int
+	for v := range av {
+		versions = append(versions, int(v))
+	}
+	sort.Ints(versions)
+	for _, v := range versions {
+		name, family, err := util.ActorNameAndFamilyFromCode(av[actors.Version(v)])
+		if err != nil {
+			return err
+		}
+		t.AppendRow(table.Row{v, name, family, av[actors.Version(v)]})
+		t.AppendSeparator()
+	}
+	fmt.Println(t.Render())
+	return nil
+}
diff --git a/lens/lily/api.go b/lens/lily/api.go
@@ -71,6 +71,9 @@ type LilyAPI interface {
 	NetPeerInfo(context.Context, peer.ID) (*api.ExtendedPeerInfo, error)
 
 	StartTipSetWorker(ctx context.Context, cfg *LilyTipSetWorkerConfig) (*schedule.JobSubmitResult, error)
+
+	FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error)
+	StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error)
 }
 type LilyJobConfig struct {
 	// Name is the name of the job.

diff --git a/lens/lily/impl.go b/lens/lily/impl.go
@@ -624,6 +624,109 @@ func (m *LilyNodeAPI) LilySurvey(_ context.Context, cfg *LilySurveyConfig) (*sch
 	return res, nil
 }
 
+type StateReport struct {
+	Height      int64
+	TipSet      *types.TipSet
+	HasMessages bool
+	HasReceipts bool
+	HasState    bool
+}
+
+func (m *LilyNodeAPI) StateCompute(ctx context.Context, key types.TipSetKey) (interface{}, error) {
+	ts, err := m.ChainAPI.ChainGetTipSet(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	_, _, err = m.StateManager.TipSetState(ctx, ts)
+	return nil, err
+}
+
+func (m *LilyNodeAPI) FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error) {
+	head, err := m.ChainHead(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var out []*StateReport
+	var oldestEpochLimit = head.Height() - abi.ChainEpoch(limit)
+
+	for i := int64(0); true; i++ {
+		maybeBaseTS, err := m.ChainGetTipSetByHeight(ctx, head.Height()-abi.ChainEpoch(i), head.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		maybeFullTS := TryLoadFullTipSet(ctx, m, maybeBaseTS)
+		out = append(out, &StateReport{
+			Height:      int64(maybeBaseTS.Height()),
+			TipSet:      maybeBaseTS,
+			HasMessages: maybeFullTS.HasMessages,
+			HasReceipts: maybeFullTS.HasReceipts,
+			HasState:    maybeFullTS.HasState,
+		})
+		if (head.Height() - abi.ChainEpoch(i)) <= oldestEpochLimit {
+			break
+		}
+	}
+	return out, nil
+}
+
+type FullBlock struct {
+	Header       *types.BlockHeader
+	BlsMessages  []*types.Message
+	SecpMessages []*types.SignedMessage
+}
+
+type FullTipSet struct {
+	Blocks []*FullBlock
+	TipSet *types.TipSet
+
+	HasMessages bool
+	HasState    bool
+	HasReceipts bool
+}
+
+func TryLoadFullTipSet(ctx context.Context, m *LilyNodeAPI, ts *types.TipSet) *FullTipSet {
+	var (
+		out         []*FullBlock
+		err         error
+		hasState    = true
+		hasMessages = true
+		hasReceipts = true
+	)
+
+	for _, b := range ts.Blocks() {
+		fb := &FullBlock{Header: b}
+
+		fb.BlsMessages, fb.SecpMessages, err = m.ChainAPI.Chain.MessagesForBlock(ctx, b)
+		if err != nil {
+			log.Debugw("failed to load messages", "height", b.Height)
+			hasMessages = false
+		}
+		out = append(out, fb)
+	}
+
+	_, err = adt.AsArray(m.ChainAPI.Chain.ActorStore(ctx), ts.Blocks()[0].ParentMessageReceipts)
+	if err != nil {
+		log.Debugw("failed to load receipts", "height", ts.Blocks()[0].Height)
+		hasReceipts = false
+	}
+
+	_, err = m.StateManager.ParentState(ts)
+	if err != nil {
+		log.Debugw("failed to load statetree", "height", ts.Blocks()[0].Height)
+		hasState = false
+	}
+
+	return &FullTipSet{
+		Blocks:      out,
+		TipSet:      ts,
+		HasMessages: hasMessages,
+		HasState:    hasState,
+		HasReceipts: hasReceipts,
+	}
+}
+
 // used for debugging querries, call ORM.AddHook and this will print all queries.
 type LogQueryHook struct{}
 

diff --git a/lens/lily/struct.go b/lens/lily/struct.go
@@ -77,9 +77,20 @@ type LilyAPIStruct struct {
 		NetPeerInfo      func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error) `perm:"read"`
 
 		StartTipSetWorker func(ctx context.Context, cfg *LilyTipSetWorkerConfig) (*schedule.JobSubmitResult, error) `perm:"read"`
+
+		FindOldestState func(ctx context.Context, limit int64) ([]*StateReport, error)      `perm:"read"`
+		StateCompute    func(ctx context.Context, tsk types.TipSetKey) (interface{}, error) `perm:"read"`
 	}
 }
 
+func (s *LilyAPIStruct) StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error) {
+	return s.Internal.StateCompute(ctx, tsk)
+}
+
+func (s *LilyAPIStruct) FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error) {
+	return s.Internal.FindOldestState(ctx, limit)
+}
+
 func (s *LilyAPIStruct) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
 	return s.Internal.ChainGetTipSetAfterHeight(ctx, epoch, key)
 }