chore: NV23 release: backport changes from master

.github/workflows/test.yml

@@ -61,6 +61,7 @@ jobs:
           #   to support resource intensive jobs.
           runners: |
             {
+              "itest-niporep_manual": ["self-hosted", "linux", "x64", "4xlarge"],
               "itest-sector_pledge": ["self-hosted", "linux", "x64", "4xlarge"],
               "itest-worker": ["self-hosted", "linux", "x64", "4xlarge"],
               "itest-manual_onboarding": ["self-hosted", "linux", "x64", "4xlarge"],
@@ -118,6 +119,7 @@ jobs:
               "itest-direct_data_onboard_verified",
               "itest-direct_data_onboard",
               "itest-manual_onboarding",
+              "itest-niporep_manual",
               "itest-net",
               "itest-path_detach_redeclare",
               "itest-sealing_resources",

CHANGELOG.md

@@ -2,7 +2,12 @@
 
 # UNRELEASED
 
+## ☢️ Upgrade Warnings ☢️
+
+- This Lotus release includes some correctness improvements to the events subsystem, impacting RPC APIs including `GetActorEventsRaw`, `SubscribeActorEventsRaw`, `eth_getLogs` and the `eth` filter APIs. Part of these improvements involve an events database migration that may take some time to complete on nodes with extensive event databases. See [filecoin-project/lotus#12080](https://github.com/filecoin-project/lotus/pull/12080) for details.
+
 ## New features
+
 - feat: Add trace transaction API supporting RPC method `trace_transaction` ([filecoin-project/lotus#12068](https://github.com/filecoin-project/lotus/pull/12068))
 
 ## Improvements

chain/actors/builtin/miner/utils.go

@@ -8,7 +8,16 @@ import (
 	"github.com/filecoin-project/go-state-types/network"
 )
 
+type SealProofVariant int
+
+const (
+	SealProofVariant_Standard SealProofVariant = iota
+	SealProofVariant_Synthetic
+	SealProofVariant_NonInteractive
+)
+
 var MinSyntheticPoRepVersion = network.Version21
+var MinNonInteractivePoRepVersion = network.Version23
 
 func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error)) (bitfield.BitField, error) {
 	var parts []bitfield.BitField
@@ -33,7 +42,18 @@ func AllPartSectors(mas State, sget func(Partition) (bitfield.BitField, error))
 
 // SealProofTypeFromSectorSize returns preferred seal proof type for creating
 // new miner actors and new sectors
-func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, synthetic bool) (abi.RegisteredSealProof, error) {
+func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, variant SealProofVariant) (abi.RegisteredSealProof, error) {
+	switch variant {
+	case SealProofVariant_Synthetic:
+		if nv < MinSyntheticPoRepVersion {
+			return 0, xerrors.Errorf("synthetic proofs are not supported on network version %d", nv)
+		}
+	case SealProofVariant_NonInteractive:
+		if nv < MinNonInteractivePoRepVersion {
+			return 0, xerrors.Errorf("non-interactive proofs are not supported on network version %d", nv)
+		}
+	}
+
 	switch {
 	case nv < network.Version7:
 		switch ssize {
@@ -67,11 +87,13 @@ func SealProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version, synth
 			return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize)
 		}
 
-		if nv >= MinSyntheticPoRepVersion && synthetic {
+		switch variant {
+		case SealProofVariant_Synthetic:
 			return toSynthetic(v)
-		} else {
-			return v, nil
+		case SealProofVariant_NonInteractive:
+			return toNonInteractive(v)
 		}
+		return v, nil
 	}
 
 	return 0, xerrors.Errorf("unsupported network version")
@@ -94,6 +116,23 @@ func toSynthetic(in abi.RegisteredSealProof) (abi.RegisteredSealProof, error) {
 	}
 }
 
+func toNonInteractive(in abi.RegisteredSealProof) (abi.RegisteredSealProof, error) {
+	switch in {
+	case abi.RegisteredSealProof_StackedDrg2KiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg2KiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg8MiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg8MiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg512MiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg512MiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg32GiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg32GiBV1_2_Feat_NiPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg64GiBV1_1:
+		return abi.RegisteredSealProof_StackedDrg64GiBV1_2_Feat_NiPoRep, nil
+	default:
+		return 0, xerrors.Errorf("unsupported conversion to non-interactive: %v", in)
+	}
+}
+
 // WindowPoStProofTypeFromSectorSize returns preferred post proof type for creating
 // new miner actors and new sectors
 func WindowPoStProofTypeFromSectorSize(ssize abi.SectorSize, nv network.Version) (abi.RegisteredPoStProof, error) {

chain/gen/genesis/miners.go

@@ -18,6 +18,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	builtintypes "github.com/filecoin-project/go-state-types/builtin"
 	power11 "github.com/filecoin-project/go-state-types/builtin/v11/power"
+	miner14 "github.com/filecoin-project/go-state-types/builtin/v14/miner"
 	minertypes "github.com/filecoin-project/go-state-types/builtin/v8/miner"
 	markettypes "github.com/filecoin-project/go-state-types/builtin/v9/market"
 	miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner"
@@ -41,6 +42,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/system"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/consensus"
 	lrand "github.com/filecoin-project/lotus/chain/rand"
@@ -136,7 +138,11 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 		i := i
 		m := m
 
-		spt, err := miner.SealProofTypeFromSectorSize(m.SectorSize, nv, synthetic)
+		variant := miner.SealProofVariant_Standard
+		if synthetic {
+			variant = miner.SealProofVariant_Synthetic
+		}
+		spt, err := miner.SealProofTypeFromSectorSize(m.SectorSize, nv, variant)
 		if err != nil {
 			return cid.Undef, err
 		}
@@ -491,7 +497,12 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 				// Commit one-by-one, otherwise pledge math tends to explode
 				var paramBytes []byte
 
-				if av >= actorstypes.Version6 {
+				if av >= actorstypes.Version14 {
+					confirmParams := &miner14.InternalSectorSetupForPresealParams{
+						Sectors: []abi.SectorNumber{preseal.SectorID},
+					}
+					paramBytes = mustEnc(confirmParams)
+				} else if av >= actorstypes.Version6 {
 					// TODO: fixup
 					confirmParams := &builtin6.ConfirmSectorProofsParams{
 						Sectors: []abi.SectorNumber{preseal.SectorID},
@@ -506,9 +517,17 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sys vm.Syscal
 					paramBytes = mustEnc(confirmParams)
 				}
 
-				_, err = doExecValue(ctx, genesisVm, minerInfos[i].maddr, power.Address, big.Zero(), builtintypes.MethodsMiner.ConfirmSectorProofsValid, paramBytes)
-				if err != nil {
-					return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", err)
+				var csErr error
+				if nv >= network.Version23 {
+					_, csErr = doExecValue(ctx, genesisVm, minerInfos[i].maddr, system.Address, big.Zero(), builtintypes.MethodsMiner.InternalSectorSetupForPreseal,
+						paramBytes)
+				} else {
+					_, csErr = doExecValue(ctx, genesisVm, minerInfos[i].maddr, power.Address, big.Zero(), builtintypes.MethodsMiner.InternalSectorSetupForPreseal,
+						paramBytes)
+				}
+
+				if csErr != nil {
+					return cid.Undef, xerrors.Errorf("failed to confirm presealed sectors: %w", csErr)
 				}
 
 				if av >= actorstypes.Version2 {

cmd/lotus-bench/bench-sectors.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# This is an example of how a full sector lifecycle can be benchmarked using `lotus-bench`. The
+# script generates an unsealed sector, runs PC1, PC2, C1, and C2, and prints the duration of each
+# step. The script also prints the proof length and total duration of the lifecycle.
+#
+# Change `flags` to `--non-interactive` to run NI-PoRep, and switch `sector_size` to the desired
+# sector size. The script assumes that the `lotus-bench` binary is in the same directory as the
+# script.
+#
+# Note that for larger sector sizes, /tmp may not have enough space for the full lifecycle.
+
+set -e
+set -o pipefail
+
+tmpdir=/tmp
+
+flags=""
+# flags="--non-interactive"
+sector_size=2KiB
+# sector_size=8MiB
+# sector_size=512MiB
+# sector_size=32GiB
+# sector_size=64GiB
+
+unsealed_file=${tmpdir}/unsealed${sector_size}
+sealed_file=${tmpdir}/sealed${sector_size}
+cache_dir=${tmpdir}/cache${sector_size}
+c1_file=${tmpdir}/c1_${sector_size}.json
+proof_out=${tmpdir}/proof_${sector_size}.hex
+rm -rf $unsealed_file $sealed_file $cache_dir $c1_file
+
+echo "Generating unsealed sector ..."
+read -r unsealed_cid unsealed_size <<< $(./lotus-bench simple addpiece --sector-size $sector_size /dev/zero $unsealed_file | tail -1)
+if [ $? -ne 0 ]; then exit 1; fi
+echo "Unsealed CID: $unsealed_cid"
+echo "Unsealed Size: $unsealed_size"
+
+start_total=$(date +%s%3N)
+
+echo "Running PC1 ..."
+echo "./lotus-bench simple precommit1 --sector-size $sector_size $flags $unsealed_file $sealed_file $cache_dir $unsealed_cid $unsealed_size"
+start_pc1=$(date +%s%3N)
+pc1_output=$(./lotus-bench simple precommit1 --sector-size $sector_size $flags $unsealed_file $sealed_file $cache_dir $unsealed_cid $unsealed_size | tail -1)
+if [ $? -ne 0 ]; then exit 1; fi
+end_pc1=$(date +%s%3N)
+pc1_duration=$((end_pc1 - start_pc1))
+
+echo "Running PC2 ..."
+echo "./lotus-bench simple precommit2 --sector-size $sector_size $flags $sealed_file $cache_dir $pc1_output"
+start_pc2=$(date +%s%3N)
+read -r commd commr <<< $(./lotus-bench simple precommit2 --sector-size $sector_size $flags $sealed_file $cache_dir $pc1_output | tail -1 | sed -E 's/[dr]://g')
+if [ $? -ne 0 ]; then exit 1; fi
+end_pc2=$(date +%s%3N)
+pc2_duration=$((end_pc2 - start_pc2))
+
+echo "CommD CID: $commd"
+echo "CommR CID: $commr"
+
+echo "Running C1 ..."
+echo "./lotus-bench simple commit1 --sector-size $sector_size $flags $sealed_file $cache_dir ${commd} ${commr} $c1_file"
+start_c1=$(date +%s%3N)
+./lotus-bench simple commit1 --sector-size $sector_size $flags $sealed_file $cache_dir ${commd} ${commr} $c1_file
+end_c1=$(date +%s%3N)
+c1_duration=$((end_c1 - start_c1))
+
+echo "Running C2 ..."
+echo "./lotus-bench simple commit2 $flags $c1_file"
+start_c2=$(date +%s%3N)
+proof=$(./lotus-bench simple commit2 $flags $c1_file | tail -1 | sed 's/^proof: //')
+if [ $? -ne 0 ]; then exit 1; fi
+end_c2=$(date +%s%3N)
+c2_duration=$((end_c2 - start_c2))
+
+echo $proof > $proof_out
+echo "Wrote proof to $proof_out"
+
+# $proof is hex, calculate the length of it in bytes
+proof_len=$(echo "scale=0; ${#proof}/2" | bc)
+echo "Proof length: $proof_len"
+
+end_total=$(date +%s%3N)
+total_duration=$((end_total - start_total))
+
+echo "PC1 duration: $((pc1_duration / 1000)).$((pc1_duration % 1000)) seconds"
+echo "PC2 duration: $((pc2_duration / 1000)).$((pc2_duration % 1000)) seconds"
+echo "C1 duration: $((c1_duration / 1000)).$((c1_duration % 1000)) seconds"
+echo "C2 duration: $((c2_duration / 1000)).$((c2_duration % 1000)) seconds"
+echo "Total duration: $((total_duration / 1000)).$((total_duration % 1000)) seconds"

cmd/lotus-bench/main.go

@@ -338,7 +338,7 @@ var sealBenchCmd = &cli.Command{
 
 		if !skipc2 {
 			log.Info("generating winning post candidates")
-			wipt, err := spt(sectorSize, false).RegisteredWinningPoStProof()
+			wipt, err := spt(sectorSize, miner.SealProofVariant_Standard).RegisteredWinningPoStProof()
 			if err != nil {
 				return err
 			}
@@ -556,7 +556,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 				Miner:  mid,
 				Number: i,
 			},
-			ProofType: spt(sectorSize, false),
+			ProofType: spt(sectorSize, miner.SealProofVariant_Standard),
 		}
 
 		start := time.Now()
@@ -586,7 +586,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 							Miner:  mid,
 							Number: i,
 						},
-						ProofType: spt(sectorSize, false),
+						ProofType: spt(sectorSize, miner.SealProofVariant_Standard),
 					}
 
 					start := time.Now()
@@ -797,7 +797,7 @@ var proveCmd = &cli.Command{
 				Miner:  abi.ActorID(mid),
 				Number: abi.SectorNumber(c2in.SectorNum),
 			},
-			ProofType: spt(abi.SectorSize(c2in.SectorSize), false),
+			ProofType: spt(abi.SectorSize(c2in.SectorSize), miner.SealProofVariant_Standard),
 		}
 
 		fmt.Printf("----\nstart proof computation\n")
@@ -828,8 +828,8 @@ func bps(sectorSize abi.SectorSize, sectorNum int, d time.Duration) string {
 	return types.SizeStr(types.BigInt{Int: bps}) + "/s"
 }
 
-func spt(ssize abi.SectorSize, synth bool) abi.RegisteredSealProof {
-	spt, err := miner.SealProofTypeFromSectorSize(ssize, build.TestNetworkVersion, synth)
+func spt(ssize abi.SectorSize, variant miner.SealProofVariant) abi.RegisteredSealProof {
+	spt, err := miner.SealProofTypeFromSectorSize(ssize, build.TestNetworkVersion, variant)
 	if err != nil {
 		panic(err)
 	}