From 369d5be6c513d42619efa4bba4ad8f22beb0c521 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 11 Jan 2025 18:25:04 +1100 Subject: [PATCH 1/3] feat(exec): dump intermediate cache blocks from FVM exec in StateReplay * Plumb through dump_cache from fvm4 to access intermediate blocks: - https://github.com/filecoin-project/filecoin-ffi/pull/512 - https://github.com/filecoin-project/ref-fvm/pull/2101 * Enable cache dumping in StateReplay with LOTUS_REPLAY_DUMP_CACHED_BLOCKS * Add optional "Blocks" field InvocResult * Handle ExecutionEvent::Log's and add "Logs" field to ExecutionTrace * Dump intermediate cache blocks to CAR in /tmp when they appear while using `lotus-shed msg --exec-trace`. --- api/api_full.go | 6 + build/openrpc/full.json | 129 +++++++++++++++++++- build/openrpc/gateway.json | 86 ++++++++++++- chain/consensus/compute_state.go | 27 +++- chain/stmgr/call.go | 4 +- chain/stmgr/execute.go | 4 +- chain/stmgr/stmgr.go | 3 +- chain/types/cbor_gen.go | 65 +++++++++- chain/types/execresult.go | 1 + chain/vm/execution.go | 5 + chain/vm/fvm.go | 8 ++ chain/vm/vm.go | 4 + chain/vm/vmi.go | 4 + cmd/lotus-shed/msg.go | 47 +++++++ conformance/driver.go | 1 + documentation/en/api-v0-methods.md | 54 +++++++- documentation/en/api-v1-unstable-methods.md | 54 +++++++- extern/filecoin-ffi | 2 +- node/impl/full/state.go | 40 +++++- 19 files changed, 507 insertions(+), 37 deletions(-) mode change 160000 => 120000 extern/filecoin-ffi diff --git a/api/api_full.go b/api/api_full.go index 5d751379be2..325d0750db2 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -1262,6 +1262,12 @@ type InvocResult struct { ExecutionTrace types.ExecutionTrace Error string Duration time.Duration + CachedBlocks []Block `json:",omitempty"` +} + +type Block struct { + Cid cid.Cid + Data []byte } type IpldObject struct { diff --git a/build/openrpc/full.json b/build/openrpc/full.json index 1ca1979ff8b..743edfae136 100644 --- a/build/openrpc/full.json +++ b/build/openrpc/full.json @@ -17487,16 +17487,49 @@ "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ], "additionalProperties": false, "properties": { + "CachedBlocks": { + "items": { + "additionalProperties": false, + "properties": { + "Cid": { + "title": "Content Identifier", + "type": "string" + }, + "Data": { + "media": { + "binaryEncoding": "base64" + }, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "Duration": { "title": "number", "type": "number" @@ -17571,6 +17604,12 @@ }, "type": "object" }, + "Logs": { + "items": { + "type": "string" + }, + "type": "array" + }, "Msg": { "additionalProperties": false, "properties": { @@ -18182,12 +18221,26 @@ "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ] } @@ -18202,6 +18255,25 @@ "items": { "additionalProperties": false, "properties": { + "CachedBlocks": { + "items": { + "additionalProperties": false, + "properties": { + "Cid": { + "title": "Content Identifier", + "type": "string" + }, + "Data": { + "media": { + "binaryEncoding": "base64" + }, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "Duration": { "title": "number", "type": "number" @@ -18276,6 +18348,12 @@ }, "type": "object" }, + "Logs": { + "items": { + "type": "string" + }, + "type": "array" + }, "Msg": { "additionalProperties": false, "properties": { @@ -23541,16 +23619,49 @@ "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ], "additionalProperties": false, "properties": { + "CachedBlocks": { + "items": { + "additionalProperties": false, + "properties": { + "Cid": { + "title": "Content Identifier", + "type": "string" + }, + "Data": { + "media": { + "binaryEncoding": "base64" + }, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "Duration": { "title": "number", "type": "number" @@ -23625,6 +23736,12 @@ }, "type": "object" }, + "Logs": { + "items": { + "type": "string" + }, + "type": "array" + }, "Msg": { "additionalProperties": false, "properties": { diff --git a/build/openrpc/gateway.json b/build/openrpc/gateway.json index ed8e4e0f431..5cb0f1e6dea 100644 --- a/build/openrpc/gateway.json +++ b/build/openrpc/gateway.json @@ -8386,16 +8386,49 @@ "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ], "additionalProperties": false, "properties": { + "CachedBlocks": { + "items": { + "additionalProperties": false, + "properties": { + "Cid": { + "title": "Content Identifier", + "type": "string" + }, + "Data": { + "media": { + "binaryEncoding": "base64" + }, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "Duration": { "title": "number", "type": "number" @@ -8470,6 +8503,12 @@ }, "type": "object" }, + "Logs": { + "items": { + "type": "string" + }, + "type": "array" + }, "Msg": { "additionalProperties": false, "properties": { @@ -10872,16 +10911,49 @@ "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ], "additionalProperties": false, "properties": { + "CachedBlocks": { + "items": { + "additionalProperties": false, + "properties": { + "Cid": { + "title": "Content Identifier", + "type": "string" + }, + "Data": { + "media": { + "binaryEncoding": "base64" + }, + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "Duration": { "title": "number", "type": "number" @@ -10956,6 +11028,12 @@ }, "type": "object" }, + "Logs": { + "items": { + "type": "string" + }, + "type": "array" + }, "Msg": { "additionalProperties": false, "properties": { diff --git a/chain/consensus/compute_state.go b/chain/consensus/compute_state.go index 51ebb51371e..46576b47f0b 100644 --- a/chain/consensus/compute_state.go +++ b/chain/consensus/compute_state.go @@ -79,7 +79,8 @@ type FilecoinBlockMessages struct { WinCount int64 } -func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, +func (t *TipSetExecutor) ApplyBlocks( + ctx context.Context, sm *stmgr.StateManager, parentEpoch abi.ChainEpoch, pstate cid.Cid, @@ -88,8 +89,10 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, r rand.Rand, em stmgr.ExecMonitor, vmTracing bool, + cacheStore blockstore.Blockstore, baseFee abi.TokenAmount, - ts *types.TipSet) (cid.Cid, cid.Cid, error) { + ts *types.TipSet, +) (cid.Cid, cid.Cid, error) { done := metrics.Timer(ctx, metrics.VMApplyBlocksTotal) defer done() @@ -240,6 +243,13 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, if em != nil { if err := em.MessageApplied(ctx, ts, cm.Cid(), m, r, false); err != nil { + log.Debugw("ApplyBlocks ExecMonitor#MessageApplied callback failed", "error", err) + if cacheStore != nil { + log.Debug("Dumping vm cache blocks to provided cacheStore") + if err := vmi.DumpCache(cacheStore); err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("dumping vm cache: %w", err) + } + } return cid.Undef, cid.Undef, err } } @@ -296,6 +306,13 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, } } + if cacheStore != nil { + log.Debug("Dumping vm cache blocks to provided cacheStore") + if err := vmi.DumpCache(cacheStore); err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("dumping vm cache: %w", err) + } + } + st, err := vmi.Flush(ctx) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("vm flush failed: %w", err) @@ -316,7 +333,9 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, sm *stmgr.StateManager, ts *types.TipSet, em stmgr.ExecMonitor, - vmTracing bool) (stateroot cid.Cid, rectsroot cid.Cid, err error) { + vmTracing bool, + cacheStore blockstore.Blockstore, +) (stateroot cid.Cid, rectsroot cid.Cid, err error) { ctx, span := trace.StartSpan(ctx, "computeTipSetState") defer span.End() @@ -364,7 +383,7 @@ func (t *TipSetExecutor) ExecuteTipSet(ctx context.Context, } baseFee := blks[0].ParentBaseFee - return t.ApplyBlocks(ctx, sm, parentEpoch, pstate, fbmsgs, blks[0].Height, r, em, vmTracing, baseFee, ts) + return t.ApplyBlocks(ctx, sm, parentEpoch, pstate, fbmsgs, blks[0].Height, r, em, vmTracing, cacheStore, baseFee, ts) } func (t *TipSetExecutor) StoreEventsAMT(ctx context.Context, cs *store.ChainStore, events []types.Event) (cid.Cid, error) { diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 90c193b95f8..7fee85a8b41 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -292,12 +292,12 @@ func (sm *StateManager) callInternal(ctx context.Context, msg *types.Message, pr var errHaltExecution = fmt.Errorf("halt") -func (sm *StateManager) Replay(ctx context.Context, ts *types.TipSet, mcid cid.Cid) (*types.Message, *vm.ApplyRet, error) { +func (sm *StateManager) Replay(ctx context.Context, ts *types.TipSet, mcid cid.Cid, cacheStore blockstore.Blockstore) (*types.Message, *vm.ApplyRet, error) { var finder messageFinder // message to find finder.mcid = mcid - _, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, &finder, true) + _, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, &finder, true, cacheStore) if err != nil && !errors.Is(err, errHaltExecution) { return nil, nil, xerrors.Errorf("unexpected error during execution: %w", err) } diff --git a/chain/stmgr/execute.go b/chain/stmgr/execute.go index 985218ef467..ecf956b117a 100644 --- a/chain/stmgr/execute.go +++ b/chain/stmgr/execute.go @@ -82,7 +82,7 @@ func (sm *StateManager) tipSetState(ctx context.Context, ts *types.TipSet, recom } } - st, rec, err = sm.tsExec.ExecuteTipSet(ctx, sm, ts, sm.tsExecMonitor, false) + st, rec, err = sm.tsExec.ExecuteTipSet(ctx, sm, ts, sm.tsExecMonitor, false, nil) if err != nil { return cid.Undef, cid.Undef, err } @@ -136,7 +136,7 @@ func tryLookupTipsetState(ctx context.Context, cs *store.ChainStore, ts *types.T } func (sm *StateManager) ExecutionTraceWithMonitor(ctx context.Context, ts *types.TipSet, em ExecMonitor) (cid.Cid, error) { - st, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, em, true) + st, _, err := sm.tsExec.ExecuteTipSet(ctx, sm, ts, em, true, nil) return st, err } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 5b227fe922e..95b761f8f29 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/specs-actors/v8/actors/migration/nv16" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain/actors/adt" _init "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -119,7 +120,7 @@ func (m *migrationResultCache) Delete(ctx context.Context, root cid.Cid) { type Executor interface { NewActorRegistry() *vm.ActorRegistry - ExecuteTipSet(ctx context.Context, sm *StateManager, ts *types.TipSet, em ExecMonitor, vmTracing bool) (stateroot cid.Cid, rectsroot cid.Cid, err error) + ExecuteTipSet(ctx context.Context, sm *StateManager, ts *types.TipSet, em ExecMonitor, vmTracing bool, cacheStore blockstore.Blockstore) (stateroot cid.Cid, rectsroot cid.Cid, err error) } type StateManager struct { diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 4c13597d075..632f4309a26 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -2823,7 +2823,7 @@ func (t *ReturnTrace) UnmarshalCBOR(r io.Reader) (err error) { return nil } -var lengthBufExecutionTrace = []byte{133} +var lengthBufExecutionTrace = []byte{134} func (t *ExecutionTrace) MarshalCBOR(w io.Writer) error { if t == nil { @@ -2880,6 +2880,28 @@ func (t *ExecutionTrace) MarshalCBOR(w io.Writer) error { return err } + } + + // t.Logs ([]string) (slice) + if len(t.Logs) > 1000000000 { + return xerrors.Errorf("Slice value in field t.Logs was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Logs))); err != nil { + return err + } + for _, v := range t.Logs { + if len(v) > 8192 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + } return nil } @@ -2903,7 +2925,7 @@ func (t *ExecutionTrace) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -3030,5 +3052,44 @@ func (t *ExecutionTrace) UnmarshalCBOR(r io.Reader) (err error) { } } + // t.Logs ([]string) (slice) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 1000000000 { + return fmt.Errorf("t.Logs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Logs = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.Logs[i] = string(sval) + } + + } + } return nil } diff --git a/chain/types/execresult.go b/chain/types/execresult.go index 99bbb6ece9a..c32a4db6a96 100644 --- a/chain/types/execresult.go +++ b/chain/types/execresult.go @@ -45,6 +45,7 @@ type ExecutionTrace struct { InvokedActor *ActorTrace `json:",omitempty"` GasCharges []*GasTrace `cborgen:"maxlen=1000000000"` Subcalls []ExecutionTrace `cborgen:"maxlen=1000000000"` + Logs []string `cborgen:"maxlen=1000000000" json:",omitempty"` } func (et ExecutionTrace) SumGas() GasTrace { diff --git a/chain/vm/execution.go b/chain/vm/execution.go index 4fb626f4390..fce917427a1 100644 --- a/chain/vm/execution.go +++ b/chain/vm/execution.go @@ -10,6 +10,7 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/tag" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/metrics" ) @@ -58,6 +59,10 @@ func (e *vmExecutor) Flush(ctx context.Context) (cid.Cid, error) { return e.vmi.Flush(ctx) } +func (e *vmExecutor) DumpCache(bs blockstore.Blockstore) error { + return e.vmi.DumpCache(bs) +} + type executionToken struct { lane ExecutionLane reserved int diff --git a/chain/vm/fvm.go b/chain/vm/fvm.go index 77103d31d8d..6190dbbaa00 100644 --- a/chain/vm/fvm.go +++ b/chain/vm/fvm.go @@ -538,6 +538,10 @@ func (vm *FVM) Flush(ctx context.Context) (cid.Cid, error) { return vm.fvm.Flush() } +func (vm *FVM) DumpCache(cacheStore blockstore.Blockstore) error { + return vm.fvm.DumpCache(cacheStore) +} + type dualExecutionFVM struct { main *FVM debug *FVM @@ -608,6 +612,10 @@ func (vm *dualExecutionFVM) Flush(ctx context.Context) (cid.Cid, error) { return vm.main.Flush(ctx) } +func (vm *dualExecutionFVM) DumpCache(cacheStore blockstore.Blockstore) error { + return vm.main.DumpCache(cacheStore) +} + // Passing this as a pointer of structs has proven to be an enormous PiTA; hence this code. type xRedirect struct{ from, to cid.Cid } type xMapping struct{ redirects []xRedirect } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index fabb5e5ebd4..a9ba781dd3f 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -686,6 +686,10 @@ func (vm *LegacyVM) Flush(ctx context.Context) (cid.Cid, error) { return root, nil } +func (vm *LegacyVM) DumpCache(_ blockstore.Blockstore) error { + return fmt.Errorf("not supported") +} + // ActorStore gets the buffered blockstore associated with the LegacyVM. This includes any temporary // blocks produced during this LegacyVM's execution. func (vm *LegacyVM) ActorStore(ctx context.Context) adt.Store { diff --git a/chain/vm/vmi.go b/chain/vm/vmi.go index 042621ca2d4..56ba6f1a5bc 100644 --- a/chain/vm/vmi.go +++ b/chain/vm/vmi.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/network" + bstore "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" ) @@ -35,6 +36,9 @@ type Interface interface { ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) // Flush all buffered objects into the state store provided to the VM at construction. Flush(ctx context.Context) (cid.Cid, error) + // Dump the contents of the caching blockstore to the provided blockstore. This will include the + // final state tree as well as any intermediate objects created during messagae execution. + DumpCache(bs bstore.Blockstore) error } // WARNING: You will not affect your node's execution by misusing this feature, but you will confuse yourself thoroughly! diff --git a/cmd/lotus-shed/msg.go b/cmd/lotus-shed/msg.go index 4de7789a85c..0a8e77c185e 100644 --- a/cmd/lotus-shed/msg.go +++ b/cmd/lotus-shed/msg.go @@ -5,12 +5,17 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "os" + "path" "sort" "github.com/fatih/color" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + carbstore "github.com/ipld/go-car/v2/blockstore" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -79,6 +84,23 @@ var msgCmd = &cli.Command{ return xerrors.Errorf("replay call failed: %w", err) } + /* + var fixSealPrice func(trace types.ExecutionTrace) + fixSealPrice = func(trace types.ExecutionTrace) { + for i := range trace.GasCharges { + if trace.GasCharges[i].Name == "OnVerifySeal" && trace.GasCharges[i].ComputeGas == 2000 { + // should be 42M + trace.GasCharges[i].ComputeGas = 42_000_000 + trace.GasCharges[i].TotalGas += 42_000_000 - 2000 + } + } + for i := range trace.Subcalls { + fixSealPrice(trace.Subcalls[i]) + } + } + fixSealPrice(res.ExecutionTrace) + */ + if cctx.Bool("exec-trace") { // Print the execution trace color.Green("Execution trace:") @@ -89,6 +111,31 @@ var msgCmd = &cli.Command{ fmt.Println(string(trace)) fmt.Println() + if res.CachedBlocks != nil { + cachedBlocksFile := path.Join(os.TempDir(), msg.Cid().String()+".car") + if _, err := os.Stat(cachedBlocksFile); !errors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("cached blocks file %s already exists: %w", cachedBlocksFile, err) + } + bs, err := carbstore.OpenReadWrite(cachedBlocksFile, nil, carbstore.WriteAsCarV1(true)) + if err != nil { + return xerrors.Errorf("opening cached blocks file: %w", err) + } + for _, b := range res.CachedBlocks { + bc, err := blocks.NewBlockWithCid(b.Data, b.Cid) + if err != nil { + return xerrors.Errorf("creating cached block: %w", err) + } + if err := bs.Put(ctx, bc); err != nil { + return xerrors.Errorf("writing cached block: %w", err) + } + } + if err := bs.Close(); err != nil { + return xerrors.Errorf("closing cached blocks file: %w", err) + } + color.Green("Cached blocks written to %s", cachedBlocksFile) + fmt.Println() + } + color.Green("Receipt:") fmt.Printf("Exit code: %d\n", res.MsgRct.ExitCode) fmt.Printf("Return: %x\n", res.MsgRct.Return) diff --git a/conformance/driver.go b/conformance/driver.go index 15ae567063a..21f4628f984 100644 --- a/conformance/driver.go +++ b/conformance/driver.go @@ -175,6 +175,7 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, params params.Rand, recordOutputs, true, + nil, params.BaseFee, nil, ) diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index 789f6948bb4..8f063aa32aa 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -3871,12 +3871,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ``` @@ -4133,12 +4147,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ] } @@ -5604,12 +5632,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ``` diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 7089dcb0663..bdc723c1da6 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -6108,12 +6108,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ``` @@ -6370,12 +6384,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ] } @@ -8054,12 +8082,26 @@ Response: "tt": 60000000000 } ], - "Subcalls": null + "Subcalls": null, + "Logs": [ + "string value" + ] } + ], + "Logs": [ + "string value" ] }, "Error": "string value", - "Duration": 60000000000 + "Duration": 60000000000, + "CachedBlocks": [ + { + "Cid": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Data": "Ynl0ZSBhcnJheQ==" + } + ] } ``` diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi deleted file mode 160000 index 72e100df9db..00000000000 --- a/extern/filecoin-ffi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 72e100df9db736aeda690672ab120500ddbd5e0d diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi new file mode 120000 index 00000000000..8e87f0735b7 --- /dev/null +++ b/extern/filecoin-ffi @@ -0,0 +1 @@ +../../filecoin-ffi/ \ No newline at end of file diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 85f43754da5..16ea594ecca 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "math" + "os" "strconv" + "strings" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" @@ -29,6 +31,7 @@ import ( market5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/build/buildconstants" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin" @@ -105,6 +108,10 @@ type StateAPI struct { TsExec stmgr.Executor } +const ReplayDumpCachedBlocksKey = "LOTUS_REPLAY_DUMP_CACHED_BLOCKS" + +var replayDumpCachedBlocks bool + func (a *StateAPI) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) { return stmgr.GetNetworkName(ctx, a.StateManager, a.Chain.GetHeaviestTipSet().ParentState()) } @@ -451,7 +458,11 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. } } - m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay) + var cacheStore blockstore.Blockstore + if replayDumpCachedBlocks { + cacheStore = blockstore.NewMemory() + } + m, r, err := a.StateManager.Replay(ctx, ts, msgToReplay, cacheStore) if err != nil { return nil, err } @@ -461,7 +472,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. errstr = r.ActorErr.Error() } - return &api.InvocResult{ + result := &api.InvocResult{ MsgCid: msgToReplay, Msg: m, MsgRct: &r.MessageReceipt, @@ -469,7 +480,18 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. ExecutionTrace: r.ExecutionTrace, Error: errstr, Duration: r.Duration, - }, nil + } + if replayDumpCachedBlocks { + bs := cacheStore.(blockstore.MemBlockstore) + result.CachedBlocks = make([]api.Block, 0, len(bs)) + for _, blk := range bs { + result.CachedBlocks = append(result.CachedBlocks, api.Block{ + Cid: blk.Cid(), + Data: blk.RawData(), + }) + } + } + return result, nil } func (m *StateModule) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (a *types.Actor, err error) { @@ -2083,3 +2105,15 @@ func (a *StateAPI) StateGetNetworkParams(ctx context.Context) (*api.NetworkParam }, }, nil } + +func init() { + replayDumpCachedBlocks = (func() bool { + v, _ := os.LookupEnv(ReplayDumpCachedBlocksKey) + switch strings.TrimSpace(strings.ToLower(v)) { + case "", "0", "false", "no": // Consider these values as "do not enable". + return false + default: + return true + } + })() +} From fc5d4b14b29810fce299845ea21edfc60dbd6d4d Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 11 Jan 2025 20:33:32 +1100 Subject: [PATCH 2/3] fixup! feat(exec): dump intermediate cache blocks from FVM exec in StateReplay --- cmd/lotus-shed/block-matcher.go | 278 ++++++++++++++++++++++++++++++++ cmd/lotus-shed/msg.go | 138 +++++++++++++--- 2 files changed, 394 insertions(+), 22 deletions(-) create mode 100644 cmd/lotus-shed/block-matcher.go diff --git a/cmd/lotus-shed/block-matcher.go b/cmd/lotus-shed/block-matcher.go new file mode 100644 index 00000000000..1d8c86776b8 --- /dev/null +++ b/cmd/lotus-shed/block-matcher.go @@ -0,0 +1,278 @@ +package main + +import ( + "bytes" + "context" + "fmt" + + blocks "github.com/ipfs/go-block-format" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/bindnode" + "github.com/ipld/go-ipld-prime/schema" + schemadmt "github.com/ipld/go-ipld-prime/schema/dmt" + schemadsl "github.com/ipld/go-ipld-prime/schema/dsl" + "github.com/ipld/go-ipld-prime/traversal" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-amt-ipld/v4" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-hamt-ipld/v3" + gstbuiltin "github.com/filecoin-project/go-state-types/builtin" + datacap16 "github.com/filecoin-project/go-state-types/builtin/v16/datacap" + market16 "github.com/filecoin-project/go-state-types/builtin/v16/market" + miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner" + power16 "github.com/filecoin-project/go-state-types/builtin/v16/power" + "github.com/filecoin-project/go-state-types/builtin/v16/util/adt" + verifreg16 "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + + "github.com/filecoin-project/lotus/chain/types" +) + +// matchKnownBlockType attempts to determine the type of a block by inspecting its bytes. First we +// attempt to decode it as part of a HAMT or AMT, and if we get one, we inspect the types of the +// values. Otherwise we attempt to decode it as a known type using matchKnownBlockTypeFromBytes. +func matchKnownBlockType(ctx context.Context, nd blocks.Block) (string, error) { + if m, err := matchKnownBlockTypeFromBytes(nd.RawData()); err != nil { + return "", err + } else if m != "" { + return m, nil + } + + // block store with just one block in it, for interacting with the hamt and amt libraries + store := cbor.NewMemCborStore() + if err := store.(*cbor.BasicIpldStore).Blocks.Put(ctx, nd); err != nil { + return "", err + } + + // try to load as a HAMT root/node (they are the same thing) + if _, err := hamt.LoadNode(ctx, store, nd.Cid(), append(adt.DefaultHamtOptions, hamt.UseTreeBitWidth(gstbuiltin.DefaultHamtBitwidth))...); err == nil { + // got a HAMT, now inspect it + hamtNode, err := ipld.DecodeUsingPrototype(nd.RawData(), dagcbor.Decode, bindnode.Prototype(nil, knownTypeSystem.TypeByName("HamtNode"))) + if err != nil { + return "", xerrors.Errorf("failed to decode HamtNode: %w", err) + } + typ, err := matchHamtValues(hamtNode) + if err != nil { + return "", err + } + return fmt.Sprintf("HAMTNode{%d}%s", gstbuiltin.DefaultHamtBitwidth, typ), nil + } + + // try to load as an AMT root, we have to try all bitwidths used in the chain + for _, bitwidth := range []uint{2, 3, 4, 5, 6} { + if _, err := amt.LoadAMT(ctx, store, nd.Cid(), append(adt.DefaultAmtOptions, amt.UseTreeBitWidth(bitwidth))...); err == nil { + // got an AMT root, now inspect it + amtRoot, err := ipld.DecodeUsingPrototype(nd.RawData(), dagcbor.Decode, bindnode.Prototype(nil, knownTypeSystem.TypeByName("AMTRoot"))) + if err != nil { + return "", xerrors.Errorf("failed to decode AMTRoot: %w", err) + } + values, err := traversal.Get(amtRoot, datamodel.ParsePath("Node/Values")) + if err != nil { + return "", xerrors.Errorf("failed to get AMTRoot.Node.Values: %w", err) + } + typ, err := matchAmtValues(values) + if err != nil { + return "", err + } + return fmt.Sprintf("AMTRoot{%d}%s", bitwidth, typ), nil + } + } + + // try to load as an AMT intermediate node, which we can't do using the amt package so we'll + // infer by schema + if amtNode, err := ipld.DecodeUsingPrototype(nd.RawData(), dagcbor.Decode, bindnode.Prototype(nil, knownTypeSystem.TypeByName("AMTNode"))); err == nil { + // got an AMT node, now inspect it + values, err := amtNode.LookupByString("Values") + if err != nil { + return "", xerrors.Errorf("failed to get AMTNode.Values: %w", err) + } + typ, err := matchAmtValues(values) + if err != nil { + return "", err + } + return "AMTNode" + typ, nil + } + + return "", nil +} + +// given a datamodel.Node form of the Values array within an AMT node, attempt to determine the +// type of the values by iterating through them all and checking from their bytes. +func matchAmtValues(values datamodel.Node) (string, error) { + var match string + itr := values.ListIterator() + for !itr.Done() { + _, v, err := itr.Next() + if err != nil { + return "", err + } + enc, err := ipld.Encode(v, dagcbor.Encode) + if err != nil { + return "", err + } + if m, _ := matchKnownBlockTypeFromBytes(enc); m != "" { + if match == "" { + match = m + } else if match != m { + return "", xerrors.Errorf("inconsistent types in AMT values") + } + } + } + if match != "" { + return "[" + match + "]", nil + } + return "", nil +} + +// given a datamodel.Node form of a HAMT node, attempt to determine the type of the values, if there +// are any, by iterating through them all and checking from their bytes. +func matchHamtValues(hamtNode datamodel.Node) (string, error) { + pointers, err := hamtNode.LookupByString("Pointers") + if err != nil { + return "", xerrors.Errorf("failed to get HamtNode.Pointers: %w", err) + } + var match string + itr := pointers.ListIterator() + for !itr.Done() { + _, v, err := itr.Next() + if err != nil { + return "", err + } + b, err := v.LookupByString("Bucket") + if err == nil { + bitr := b.ListIterator() + for !bitr.Done() { + _, kv, err := bitr.Next() + if err != nil { + return "", err + } + bval, err := kv.LookupByString("Value") + if err != nil { + return "", err + } + enc, err := ipld.Encode(bval, dagcbor.Encode) + if err != nil { + return "", err + } + if m, _ := matchKnownBlockTypeFromBytes(enc); m != "" { + if match == "" { + match = m + } else if match != m { + return "", xerrors.Errorf("inconsistent types in HAMT values") + } + } + } + } + } + if match != "" { + return "[" + match + "]", nil + } + return "", nil +} + +var wellKnownBlockBytes = map[string][]byte{ + "EmptyArray": {0x80}, + "EmptyBytes": {0x40}, + "EmptyString": {0x60}, // is this used anywhere in the chain? + "Zero": {0x00}, // is this used anywhere in the chain? + "HAMTNode{5}[empty]": {0x82, 0x40, 0x80}, + "AMTRoot{2/3}[empty]": {0x84, 0x02, 0x00, 0x00, 0x83, 0x41, 0x00, 0x80, 0x80}, + "AMTRoot{4}[empty]": {0x84, 0x04, 0x00, 0x00, 0x83, 0x42, 0x00, 0x00, 0x80, 0x80}, + "AMTRoot{5}[empty]": {0x84, 0x05, 0x00, 0x00, 0x83, 0x44, 0x00, 0x00, 0x00, 0x00, 0x80, 0x80}, + "AMTRoot{6}[empty]": {0x84, 0x06, 0x00, 0x00, 0x83, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x80}, +} + +func matchWellKnownBlockType(b []byte) (string, error) { + for name, wkb := range wellKnownBlockBytes { + if bytes.Equal(b, wkb) { + return name, nil + } + } + return "", nil +} + +// matchKnownBlockTypeFromBytes attempts to determine the type of a block by inspecting its bytes. +// We use a fixed list of known types that have a CBORUnmarshaler that we believe may be possible. +// This list is not exhaustive and should be expanded as unknown types are encountered. +func matchKnownBlockTypeFromBytes(b []byte) (string, error) { + if m, _ := matchWellKnownBlockType(b); m != "" { + return m, nil + } + + if _, err := cbg.ReadCid(bytes.NewReader(b)); err == nil { + return "Cid", nil + } + known := map[string]cbg.CBORUnmarshaler{ + // Fill this out with known types when you see them missing and can identify them + "BlockHeader": &types.BlockHeader{}, + "miner16.State": &miner16.State{}, + "miner16.MinerInfo": &miner16.MinerInfo{}, + "miner16.Deadlines": &miner16.Deadlines{}, + "miner16.Deadline": &miner16.Deadline{}, + "miner16.Partition": &miner16.Partition{}, + "miner16.ExpirationSet": &miner16.ExpirationSet{}, + "miner16.WindowedPoSt": &miner16.WindowedPoSt{}, + "miner16.SectorOnChainInfo": &miner16.SectorOnChainInfo{}, + "miner16.SectorPreCommitOnChainInfo": &miner16.SectorPreCommitOnChainInfo{}, + "power16.State": &power16.State{}, + "market16.State": &market16.State{}, + "verifreg16.State": &verifreg16.State{}, + "datacap16.State": &datacap16.State{}, + "Bitfield": &bitfield.BitField{}, + } + for name, v := range known { + if err := v.UnmarshalCBOR(bytes.NewReader(b)); err == nil { + return name, nil + } + } + return "", nil +} + +const knownTypesSchema = ` +type HamtNode struct { + Bitfield Bytes + Pointers [Pointer] +} representation tuple + +type Pointer union { + | Any link # link to HamtNode + | Bucket list +} representation kinded + +type Bucket [KV] + +type KV struct { + Key Bytes + Value Any +} representation tuple + +type AMTNode struct { + Bmap Bytes + Links [Link] + Values [Any] +} representation tuple + +type AMTRoot struct { + BitWidth Int + Height Int + Count Int + Node AMTNode +} representation tuple +` + +var knownTypeSystem schema.TypeSystem + +func init() { + sch, err := schemadsl.ParseBytes([]byte(knownTypesSchema)) + if err != nil { + panic(err) + } + knownTypeSystem.Init() + if err := schemadmt.Compile(&knownTypeSystem, sch); err != nil { + panic(err) + } +} diff --git a/cmd/lotus-shed/msg.go b/cmd/lotus-shed/msg.go index 0a8e77c185e..2c1f2932ddd 100644 --- a/cmd/lotus-shed/msg.go +++ b/cmd/lotus-shed/msg.go @@ -2,15 +2,16 @@ package main import ( "bytes" + "context" "encoding/base64" "encoding/hex" "encoding/json" - "errors" "fmt" "io" "os" "path" "sort" + "strings" "github.com/fatih/color" blocks "github.com/ipfs/go-block-format" @@ -23,6 +24,7 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -112,28 +114,9 @@ var msgCmd = &cli.Command{ fmt.Println() if res.CachedBlocks != nil { - cachedBlocksFile := path.Join(os.TempDir(), msg.Cid().String()+".car") - if _, err := os.Stat(cachedBlocksFile); !errors.Is(err, os.ErrNotExist) { - return xerrors.Errorf("cached blocks file %s already exists: %w", cachedBlocksFile, err) - } - bs, err := carbstore.OpenReadWrite(cachedBlocksFile, nil, carbstore.WriteAsCarV1(true)) - if err != nil { - return xerrors.Errorf("opening cached blocks file: %w", err) - } - for _, b := range res.CachedBlocks { - bc, err := blocks.NewBlockWithCid(b.Data, b.Cid) - if err != nil { - return xerrors.Errorf("creating cached block: %w", err) - } - if err := bs.Put(ctx, bc); err != nil { - return xerrors.Errorf("writing cached block: %w", err) - } - } - if err := bs.Close(); err != nil { - return xerrors.Errorf("closing cached blocks file: %w", err) + if err := saveAndInspectBlocks(ctx, res, cctx.App.Writer); err != nil { + return err } - color.Green("Cached blocks written to %s", cachedBlocksFile) - fmt.Println() } color.Green("Receipt:") @@ -541,3 +524,114 @@ func gasTracesPerCall(inTrace types.ExecutionTrace) types.ExecutionTrace { accum(inTrace.Msg.To.String(), inTrace) return outTrace } + +func saveAndInspectBlocks(ctx context.Context, res *api.InvocResult, out io.Writer) (err error) { + cachedBlocksFile := path.Join(os.TempDir(), res.MsgCid.String()+".car") + bs, err := carbstore.OpenReadWrite(cachedBlocksFile, nil, carbstore.WriteAsCarV1(true)) + if err != nil { + return xerrors.Errorf("opening cached blocks file: %w", err) + } + + defer func() { + if cerr := bs.Close(); cerr != nil { + err = xerrors.Errorf("closing cached blocks file: %w", cerr) + } + }() + + for _, b := range res.CachedBlocks { + bc, err := blocks.NewBlockWithCid(b.Data, b.Cid) + if err != nil { + return xerrors.Errorf("creating cached block: %w", err) + } + if err := bs.Put(ctx, bc); err != nil { + return xerrors.Errorf("writing cached block: %w", err) + } + } + color.Green("Cached blocks written to %s", cachedBlocksFile) + + type blkStat struct { + cid cid.Cid + knownType string + size int + estimatedGas int + } + + var explainBlocks func(descPfx string, trace types.ExecutionTrace) error + explainBlocks = func(descPfx string, trace types.ExecutionTrace) error { + typ := "Message" + if descPfx != "" { + typ = "Subcall" + } + + blkStats := make([]blkStat, 0, len(res.CachedBlocks)) + var totalBytes, totalGas int + + for _, ll := range trace.Logs { + if strings.HasPrefix(ll, "block_link(") { + c, err := cid.Parse(strings.TrimSuffix(strings.TrimPrefix(ll, "block_link("), ")")) + if err != nil { + return xerrors.Errorf("parsing block cid: %w", err) + } + blk, err := bs.Get(ctx, c) + if err != nil { + return xerrors.Errorf("getting block (%s) from cached blocks: %w", c, err) + } + m, err := matchKnownBlockType(ctx, blk) + if err != nil { + return xerrors.Errorf("matching block type: %w", err) + } + size := len(blk.RawData()) + gas := 172000 + 334000 + 3340*size + blkStats = append(blkStats, blkStat{cid: c, knownType: m, size: size, estimatedGas: gas}) + totalBytes += size + totalGas += gas + } + } + + if len(blkStats) == 0 { + return nil + } + + _, _ = fmt.Fprintln(out, color.New(color.Bold).Sprint(fmt.Sprintf("%s (%s%s) block writes:", typ, descPfx, trace.Msg.To))) + tw := tablewriter.New( + tablewriter.Col("CID"), + tablewriter.Col("Known Type"), + tablewriter.Col("Size", tablewriter.RightAlign()), + tablewriter.Col("S%", tablewriter.RightAlign()), + tablewriter.Col("Estimated Gas", tablewriter.RightAlign()), + tablewriter.Col("G%", tablewriter.RightAlign()), + ) + for _, bs := range blkStats { + tw.Write(map[string]interface{}{ + "CID": bs.cid, + "Known Type": bs.knownType, + "Size": bs.size, + "S%": fmt.Sprintf("%.2f", float64(bs.size)/float64(totalBytes)*100), + "Estimated Gas": bs.estimatedGas, + "G%": fmt.Sprintf("%.2f", float64(bs.estimatedGas)/float64(totalGas)*100), + }) + } + tw.Write(map[string]interface{}{ + "CID": "Total", + "Known Type": "", + "Size": totalBytes, + "S%": "100.00", + "Estimated Gas": totalGas, + "G%": "100.00", + }) + if err := tw.Flush(out, tablewriter.WithBorders()); err != nil { + return xerrors.Errorf("flushing table: %w", err) + } + + for _, subtrace := range trace.Subcalls { + if err := explainBlocks(descPfx+trace.Msg.To.String()+"➜", subtrace); err != nil { + return err + } + } + return nil + } + if err := explainBlocks("", res.ExecutionTrace); err != nil { + return xerrors.Errorf("explaining blocks: %w", err) + } + return +} From 0f4fc442772510ab11a082981f80842e783da7a7 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 11 Jan 2025 20:57:22 +1100 Subject: [PATCH 3/3] fixup! feat(exec): dump intermediate cache blocks from FVM exec in StateReplay --- cmd/lotus-shed/block-matcher.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-shed/block-matcher.go b/cmd/lotus-shed/block-matcher.go index 1d8c86776b8..a7c2c54aab4 100644 --- a/cmd/lotus-shed/block-matcher.go +++ b/cmd/lotus-shed/block-matcher.go @@ -19,8 +19,8 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-amt-ipld/v4" - "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-hamt-ipld/v3" + "github.com/filecoin-project/go-state-types/abi" gstbuiltin "github.com/filecoin-project/go-state-types/builtin" datacap16 "github.com/filecoin-project/go-state-types/builtin/v16/datacap" market16 "github.com/filecoin-project/go-state-types/builtin/v16/market" @@ -120,6 +120,10 @@ func matchAmtValues(values datamodel.Node) (string, error) { } else if match != m { return "", xerrors.Errorf("inconsistent types in AMT values") } + // To debug unknown AMT types, uncomment this block: + // } else { + // enc, _ := ipld.Encode(v, dagjson.Encode) + // return "", xerrors.Errorf("unknown type in AMT values: %s", enc) } } if match != "" { @@ -162,8 +166,12 @@ func matchHamtValues(hamtNode datamodel.Node) (string, error) { if match == "" { match = m } else if match != m { - return "", xerrors.Errorf("inconsistent types in HAMT values") + return "", xerrors.Errorf("inconsistent types in HAMT values: %s != %s", match, m) } + // To debug unknown HAMT types, uncomment this block: + // } else { + // enc, _ := ipld.Encode(bval, dagjson.Encode) + // return "", xerrors.Errorf("unknown type in HAMT values: %s", enc) } } } @@ -202,10 +210,11 @@ func matchKnownBlockTypeFromBytes(b []byte) (string, error) { if m, _ := matchWellKnownBlockType(b); m != "" { return m, nil } - if _, err := cbg.ReadCid(bytes.NewReader(b)); err == nil { return "Cid", nil } + ci := cbg.CborInt(1) + cb := cbg.CborBool(true) known := map[string]cbg.CBORUnmarshaler{ // Fill this out with known types when you see them missing and can identify them "BlockHeader": &types.BlockHeader{}, @@ -220,9 +229,16 @@ func matchKnownBlockTypeFromBytes(b []byte) (string, error) { "miner16.SectorPreCommitOnChainInfo": &miner16.SectorPreCommitOnChainInfo{}, "power16.State": &power16.State{}, "market16.State": &market16.State{}, + "market16.DealProposal": &market16.DealProposal{}, + "market16.DealState": &market16.DealState{}, "verifreg16.State": &verifreg16.State{}, + "verifreg16.Allocation": &verifreg16.Allocation{}, + "verifreg16.Claim": &verifreg16.Claim{}, "datacap16.State": &datacap16.State{}, - "Bitfield": &bitfield.BitField{}, + "[Int]": &market16.SectorDealIDs{}, // verifreg16.RmDcProposalID is one of these too, as are probably others, we can't be certain, it would be context dependent + "Int": &ci, + "Bool": &cb, + "Bytes": &abi.CborBytes{}, // could be TokenAmount, BigInt, etc } for name, v := range known { if err := v.UnmarshalCBOR(bytes.NewReader(b)); err == nil {