Merge branch 'master' into feat/break-treestask

api/api_full.go

@@ -335,7 +335,7 @@ type FullNode interface {
 	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) //perm:read
 	// WalletDefaultAddress returns the address marked as default in the wallet.
 	WalletDefaultAddress(context.Context) (address.Address, error) //perm:write
-	// WalletSetDefault marks the given address as as the default one.
+	// WalletSetDefault marks the given address as the default one.
 	WalletSetDefault(context.Context, address.Address) error //perm:write
 	// WalletExport returns the private key of an address in the wallet.
 	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin

api/v0api/full.go

@@ -293,7 +293,7 @@ type FullNode interface {
 	WalletVerify(context.Context, address.Address, []byte, *crypto.Signature) (bool, error) //perm:read
 	// WalletDefaultAddress returns the address marked as default in the wallet.
 	WalletDefaultAddress(context.Context) (address.Address, error) //perm:write
-	// WalletSetDefault marks the given address as as the default one.
+	// WalletSetDefault marks the given address as the default one.
 	WalletSetDefault(context.Context, address.Address) error //perm:write
 	// WalletExport returns the private key of an address in the wallet.
 	WalletExport(context.Context, address.Address) (*types.KeyInfo, error) //perm:admin

build/builtin_actors.go

@@ -145,10 +145,10 @@ func readEmbeddedBuiltinActorsMetadata(bundle string) ([]*BuiltinActorsMetadata,
 	)
 
 	if !strings.HasPrefix(bundle, "v") {
-		return nil, xerrors.Errorf("bundle bundle '%q' doesn't start with a 'v'", bundle)
+		return nil, xerrors.Errorf("bundle '%q' doesn't start with a 'v'", bundle)
 	}
 	if !strings.HasSuffix(bundle, archiveExt) {
-		return nil, xerrors.Errorf("bundle bundle '%q' doesn't end with '%s'", bundle, archiveExt)
+		return nil, xerrors.Errorf("bundle '%q' doesn't end with '%s'", bundle, archiveExt)
 	}
 	version, err := strconv.ParseInt(bundle[1:len(bundle)-len(archiveExt)], 10, 0)
 	if err != nil {

build/openrpc/full.json

@@ -26382,7 +26382,7 @@
         {
             "name": "Filecoin.WalletSetDefault",
             "description": "```go\nfunc (s *FullNodeStruct) WalletSetDefault(p0 context.Context, p1 address.Address) error {\n\tif s.Internal.WalletSetDefault == nil {\n\t\treturn ErrNotSupported\n\t}\n\treturn s.Internal.WalletSetDefault(p0, p1)\n}\n```",
-            "summary": "WalletSetDefault marks the given address as as the default one.\n",
+            "summary": "WalletSetDefault marks the given address as the default one.\n",
             "paramStructure": "by-position",
             "params": [
                 {

chain/events/filter/index.go

@@ -356,6 +356,10 @@ func (ei *EventIndex) migrateToVersion4(ctx context.Context) error {
 		}
 	}
 
+	if _, err = tx.Exec("INSERT OR IGNORE INTO _meta (version) VALUES (4)"); err != nil {
+		return xerrors.Errorf("increment _meta version: %w", err)
+	}
+
 	err = tx.Commit()
 	if err != nil {
 		return xerrors.Errorf("commit transaction: %w", err)
@@ -478,7 +482,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever
 	// rollback the transaction (a no-op if the transaction was already committed)
 	defer func() { _ = tx.Rollback() }()
 
-	// lets handle the revert case first, since its simpler and we can simply mark all events events in this tipset as reverted and return
+	// lets handle the revert case first, since its simpler and we can simply mark all events in this tipset as reverted and return
 	if revert {
 		_, err = tx.Stmt(ei.stmtRevertEventsInTipset).Exec(te.msgTs.Height(), te.msgTs.Key().Bytes())
 		if err != nil {

chain/messagepool/selection.go

@@ -400,7 +400,7 @@ tailLoop:
 			continue tailLoop
 		}
 
-		// the merge loop ended after processing all the chains and we we probably have still
+		// the merge loop ended after processing all the chains and we probably have still
 		// gas to spare; end the loop.
 		break
 	}

chain/store/checkpoint_test.go

@@ -52,7 +52,7 @@ func TestChainCheckpoint(t *testing.T) {
 	head = cs.GetHeaviestTipSet()
 	require.True(t, head.Equals(checkpoint))
 
-	// Let the second miner miner mine a fork
+	// Let the second miner mine a fork
 	last = checkpointParents
 	for i := 0; i < 4; i++ {
 		ts, err := cg.NextTipSetFromMiners(last, cg.Miners[1:], 0)

curiosrc/market/lmrpc/lmrpc.go

@@ -14,8 +14,8 @@ import (
 
 	"github.com/google/uuid"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/jackc/pgx/v5"
 	manet "github.com/multiformats/go-multiaddr/net"
+	"github.com/yugabyte/pgx/v5"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"

curiosrc/seal/task_porep.go

@@ -3,6 +3,7 @@ package seal
 import (
 	"bytes"
 	"context"
+	"strings"
 
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
@@ -111,6 +112,15 @@ func (p *PoRepTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
 
 	proof, err := p.sc.PoRepSnark(ctx, sr, sealed, unsealed, sectorParams.TicketValue, abi.InteractiveSealRandomness(rand))
 	if err != nil {
+		end, err := p.recoverErrors(ctx, sectorParams.SpID, sectorParams.SectorNumber, err)
+		if err != nil {
+			return false, xerrors.Errorf("recover errors: %w", err)
+		}
+		if end {
+			// done, but the error handling has stored a different than success state
+			return true, nil
+		}
+
 		return false, xerrors.Errorf("failed to compute seal proof: %w", err)
 	}
 
@@ -161,4 +171,46 @@ func (p *PoRepTask) Adder(taskFunc harmonytask.AddTaskFunc) {
 	p.sp.pollers[pollerPoRep].Set(taskFunc)
 }
 
+func (p *PoRepTask) recoverErrors(ctx context.Context, spid, snum int64, cerr error) (end bool, err error) {
+	const (
+		// rust-fil-proofs error strings
+		// https://github.com/filecoin-project/rust-fil-proofs/blob/3f018b51b6327b135830899d237a7ba181942d7e/storage-proofs-porep/src/stacked/vanilla/proof.rs#L454C1-L463
+		errstrInvalidCommD    = "Invalid comm_d detected at challenge_index"
+		errstrInvalidCommR    = "Invalid comm_r detected at challenge_index"
+		errstrInvalidEncoding = "Invalid encoding proof generated at layer"
+	)
+
+	if cerr == nil {
+		return false, xerrors.Errorf("nil error")
+	}
+
+	switch {
+	case strings.Contains(cerr.Error(), errstrInvalidCommD):
+		fallthrough
+	case strings.Contains(cerr.Error(), errstrInvalidCommR):
+		// todo: it might be more optimal to just retry the Trees compute first.
+		//  Invalid CommD/R likely indicates a problem with the data computed in that step
+		//  For now for simplicity just retry the whole thing
+		fallthrough
+	case strings.Contains(cerr.Error(), errstrInvalidEncoding):
+		n, err := p.db.Exec(ctx, `UPDATE sectors_sdr_pipeline
+		SET after_porep = FALSE, after_sdr = FALSE, after_tree_d = FALSE,
+		    after_tree_r = FALSE, after_tree_c = FALSE
+		WHERE sp_id = $1 AND sector_number = $2`,
+			spid, snum)
+		if err != nil {
+			return false, xerrors.Errorf("store sdr success: updating pipeline: %w", err)
+		}
+		if n != 1 {
+			return false, xerrors.Errorf("store sdr success: updated %d rows", n)
+		}
+
+		return true, nil
+
+	default:
+		// if end is false the original error will be returned by the caller
+		return false, nil
+	}
+}
+
 var _ harmonytask.TaskInterface = &PoRepTask{}

documentation/en/api-v0-methods.md

@@ -7594,7 +7594,7 @@ Inputs:
 Response: `"f01234"`
 
 ### WalletSetDefault
-WalletSetDefault marks the given address as as the default one.
+WalletSetDefault marks the given address as the default one.
 
 
 Perms: write

documentation/en/api-v1-unstable-methods.md

@@ -9284,7 +9284,7 @@ Inputs:
 Response: `"f01234"`
 
 ### WalletSetDefault
-WalletSetDefault marks the given address as as the default one.
+WalletSetDefault marks the given address as the default one.
 
 
 Perms: write

go.mod

@@ -107,7 +107,6 @@ require (
 	github.com/ipni/go-libipni v0.0.8
 	github.com/ipni/index-provider v0.12.0
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
-	github.com/jackc/pgx/v5 v5.4.1
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.1.0
@@ -251,6 +250,7 @@ require (
 	github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
+	github.com/jackc/pgx/v5 v5.4.1 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect

go.sum

@@ -305,6 +305,7 @@ github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38a
 github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE=
 github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo=
 github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE=
+github.com/filecoin-project/go-amt-ipld/v4 v4.2.0/go.mod h1:0eDVF7pROvxrsxvLJx+SJZXqRaXXcEPUcgb/rG0zGU4=
 github.com/filecoin-project/go-amt-ipld/v4 v4.3.0 h1:bY42N1gR0DqrLMCKUPzX1VhYVgXaETQm0Um4ohvyEP8=
 github.com/filecoin-project/go-amt-ipld/v4 v4.3.0/go.mod h1:39Ep/yBbF6xN94WevLG9qSbglBJepHa5zeEbAE1pYsc=
 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
@@ -354,7 +355,7 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go
 github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
 github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
-github.com/filecoin-project/go-state-types v0.11.2-0.20230712101859-8f37624fa540/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8=
+github.com/filecoin-project/go-state-types v0.13.1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
 github.com/filecoin-project/go-state-types v0.13.3 h1:9JPkC0E6HDtfHbaOIrFiCDzT/Z0jRTb9En4Y4Ci/b3w=
 github.com/filecoin-project/go-state-types v0.13.3/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
@@ -953,6 +954,7 @@ github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6K
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/koalacxr/quantile v0.0.1 h1:wAW+SQ286Erny9wOjVww96t8ws+x5Zj6AKHDULUK+o0=
@@ -1681,6 +1683,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.1.0/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so=
 github.com/whyrusleeping/cbor-gen v0.1.1 h1:eKfcJIoxivjMtwfCfmJAqSF56MHcWqyIScXwaC1VBgw=
 github.com/whyrusleeping/cbor-gen v0.1.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
@@ -2065,6 +2068,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

journal/alerting/alerts.go

@@ -95,7 +95,7 @@ func (a *Alerting) update(at AlertType, message interface{}, upd func(Alert, jso
 		}{
 			AlertError: err.Error(),
 		})
-		log.Errorw("marshaling marshaling error failed", "type", at, "error", err)
+		log.Errorw("marshaling error failed", "type", at, "error", err)
 	}
 
 	a.alerts[at] = upd(alert, rawMsg)

lib/harmony/harmonydb/sql/20240317-web-summary-index.sql

@@ -1,4 +1,5 @@
 /* Used for webui clusterMachineSummary */
+-- NOTE: This index is changed in 20240420-web-task-indexes.sql
 CREATE INDEX harmony_task_history_work_index
 	ON harmony_task_history (completed_by_host_and_port ASC, name ASC, result ASC, work_end DESC);
 

lib/harmony/harmonydb/sql/20240420-web-task-indexes.sql

@@ -0,0 +1,9 @@
+DROP INDEX harmony_task_history_work_index;
+
+/*
+ This structure improves clusterMachineSummary query better than the old version,
+ while at the same time also being usable by clusterTaskHistorySummary (which wasn't
+ the case with the old index).
+ */
+create index harmony_task_history_work_index
+    on harmony_task_history (work_end desc, completed_by_host_and_port asc, name asc, result asc);

lib/harmony/harmonytask/singleton_task.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/jackc/pgx/v5"
+	"github.com/yugabyte/pgx/v5"
 
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/lib/passcall"

metrics/metrics.go

@@ -190,11 +190,11 @@ var (
 	RcmgrAllowPeer      = stats.Int64("rcmgr/allow_peer", "Number of allowed peer connections", stats.UnitDimensionless)
 	RcmgrBlockPeer      = stats.Int64("rcmgr/block_peer", "Number of blocked peer connections", stats.UnitDimensionless)
 	RcmgrAllowProto     = stats.Int64("rcmgr/allow_proto", "Number of allowed streams attached to a protocol", stats.UnitDimensionless)
-	RcmgrBlockProto     = stats.Int64("rcmgr/block_proto", "Number of blocked blocked streams attached to a protocol", stats.UnitDimensionless)
-	RcmgrBlockProtoPeer = stats.Int64("rcmgr/block_proto", "Number of blocked blocked streams attached to a protocol for a specific peer", stats.UnitDimensionless)
+	RcmgrBlockProto     = stats.Int64("rcmgr/block_proto", "Number of blocked streams attached to a protocol", stats.UnitDimensionless)
+	RcmgrBlockProtoPeer = stats.Int64("rcmgr/block_proto", "Number of blocked streams attached to a protocol for a specific peer", stats.UnitDimensionless)
 	RcmgrAllowSvc       = stats.Int64("rcmgr/allow_svc", "Number of allowed streams attached to a service", stats.UnitDimensionless)
-	RcmgrBlockSvc       = stats.Int64("rcmgr/block_svc", "Number of blocked blocked streams attached to a service", stats.UnitDimensionless)
-	RcmgrBlockSvcPeer   = stats.Int64("rcmgr/block_svc", "Number of blocked blocked streams attached to a service for a specific peer", stats.UnitDimensionless)
+	RcmgrBlockSvc       = stats.Int64("rcmgr/block_svc", "Number of blocked streams attached to a service", stats.UnitDimensionless)
+	RcmgrBlockSvcPeer   = stats.Int64("rcmgr/block_svc", "Number of blocked streams attached to a service for a specific peer", stats.UnitDimensionless)
 	RcmgrAllowMem       = stats.Int64("rcmgr/allow_mem", "Number of allowed memory reservations", stats.UnitDimensionless)
 	RcmgrBlockMem       = stats.Int64("rcmgr/block_mem", "Number of blocked memory reservations", stats.UnitDimensionless)
 

storage/paths/db_index.go

@@ -723,7 +723,7 @@ func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.Sec
 						 FROM storage_path 
 						 WHERE available >= $1
 						 and NOW()-($2 * INTERVAL '1 second') < last_heartbeat
-						 and heartbeat_err is null
+						 and heartbeat_err IS NULL
 						 and (($3 and can_seal = TRUE) or ($4 and can_store = TRUE))
 						order by (available::numeric * weight) desc`,
 		spaceReq,