Using the multi value slice for the beacon state (#12549)

* in progress...

* in progress...

* remove log

* log root

* Revert "Auxiliary commit to revert individual files from f12a609"

This reverts commit 5ae35edb6477d8d0ea4e94b273efc6590484da85.

* cleanup

* remove log

* remove whitespace

* remove logs

* more stuff

* copy

* always rebuild trie

* revert

* add state

* init state

* fix all

* uintptr

* move slice to new package

* lock in `Detach`

* remove constraint

* reorder

* blockroots and stateroots

* fill roots in empty()

* fix hasher

* implement slice for balances and inactivity scores

* detach in setters

* Revert "implement slice for balances and inactivity scores"

This reverts commit 59eb9df.

# Conflicts:
#	beacon-chain/state/state-native/setters_validator.go

* use counter to track states

* typos

* rename interface

* balances

* gauge

* some improvements

* first try with map

* fix

* inactivity scores in progress

* fix test

# Conflicts:
#	beacon-chain/state/state-native/helpers_test.go

* test fixes

* ToProto fix

* copy roots

* validators

* build fixes

* fix bug in `ToProto`

* fix fuzz test

* fix bug in slice getters

* fix state equality checks

* make tests pass

* make tests pass

* more test updates

* Revert "Auxiliary commit to revert individual files from 34e7344"

This reverts commit ecd64efa8917f37ca41460e0356ff007fe55dd9d.

* Revert "make tests pass"

This reverts commit 0cf00f1.

* Revert "make tests pass"

This reverts commit 521b65e.

* pass tests

* deepequal identifiable types

* Deflake `cloners_test.go`

* feature flag for block roots

* feature flag

* remove recursive locks

* reduce complexity of rootSelector

* fix randao mixes root

* some fixes

* revisit tests

* revert change to field trie helpers

* initialize field map for tests

* remove whitespace

* initialize roots with proper length

* more fixes

* out of bounds message fix

* optimize length calculation

* remove call to Len in PubkeyAtIndex

* don't log deposits

* unit tests

* unit tests

* fix

* comments

* test fixes

* id

* remove Enumerator interface

* review feedback

* simplify field trie

* bring back fieldtrie package

* fix bazel file

* use handle32ByteArrays for root computation

* fix locks

* metrics

* bzl

* simplify some things

* use htr in state test

* remove code from require package

* gzl

* more htr

* Fuzzing of the multi-value slice

* assert values

* getter optimizations

* use At when reading from validators

* Nishant's review

* restore safe copy

* remove empty line

* build fix

* restore how we get root at index for deafult mode

* more review comments

* optimize default behavior

* simplify Slice calls

* test fix

* remove unnecessary package

* remove unused setter

* make fieldMap unexported

* some improvements in state package

* call `Slice` instead of manually copying

* unlock in ReadFromEveryValidator

* Potuz's comments

* lock the state when reading from all validators

# Conflicts:
#	beacon-chain/state/state-native/getters_validator.go

* add back preston's changes

* add index

---------

Co-authored-by: Potuz <[email protected]>
Co-authored-by: nisdas <[email protected]>
Co-authored-by: Preston Van Loon <[email protected]>

beacon-chain/blockchain/metrics_test.go

@@ -16,7 +16,7 @@ func TestReportEpochMetrics_BadHeadState(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, h.SetValidators(nil))
 	err = reportEpochMetrics(context.Background(), s, h)
-	require.ErrorContains(t, "failed to initialize precompute: nil validators in state", err)
+	require.ErrorContains(t, "failed to initialize precompute: state has nil validator slice", err)
 }
 
 func TestReportEpochMetrics_BadAttestation(t *testing.T) {

beacon-chain/core/altair/reward_test.go

@@ -31,7 +31,7 @@ func Test_BaseReward(t *testing.T) {
 			valIdx:    2,
 			st:        genState(1),
 			want:      0,
-			errString: "index 2 out of range",
+			errString: "validator index 2 does not exist",
 		},
 		{
 			name:      "active balance is 32eth",
@@ -89,7 +89,7 @@ func Test_BaseRewardWithTotalBalance(t *testing.T) {
 			valIdx:        2,
 			activeBalance: 1,
 			want:          0,
-			errString:     "index 2 out of range",
+			errString:     "validator index 2 does not exist",
 		},
 		{
 			name:          "active balance is 1",

beacon-chain/core/blocks/randao.go

@@ -82,7 +82,7 @@ func ProcessRandaoNoVerify(
 	for i, x := range blockRandaoReveal {
 		latestMixSlice[i] ^= x
 	}
-	if err := beaconState.UpdateRandaoMixesAtIndex(uint64(currentEpoch%latestMixesLength), latestMixSlice); err != nil {
+	if err := beaconState.UpdateRandaoMixesAtIndex(uint64(currentEpoch%latestMixesLength), [32]byte(latestMixSlice)); err != nil {
 		return nil, err
 	}
 	return beaconState, nil

beacon-chain/core/blocks/withdrawals_test.go

@@ -152,7 +152,7 @@ func TestProcessBLSToExecutionChange(t *testing.T) {
 		}
 
 		_, err = blocks.ProcessBLSToExecutionChange(st, signed)
-		require.ErrorContains(t, "out of range", err)
+		require.ErrorContains(t, "out of bounds", err)
 	})
 
 	t.Run("signature does not verify", func(t *testing.T) {

beacon-chain/core/epoch/epoch_processing.go

@@ -349,7 +349,7 @@ func ProcessRandaoMixesReset(state state.BeaconState) (state.BeaconState, error)
 	if err != nil {
 		return nil, err
 	}
-	if err := state.UpdateRandaoMixesAtIndex(uint64(nextEpoch%randaoMixLength), mix); err != nil {
+	if err := state.UpdateRandaoMixesAtIndex(uint64(nextEpoch%randaoMixLength), [32]byte(mix)); err != nil {
 		return nil, err
 	}
 

beacon-chain/core/helpers/sync_committee_test.go

@@ -99,7 +99,7 @@ func TestIsCurrentEpochSyncCommittee_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	ok, err := IsCurrentPeriodSyncCommittee(state, 12390192)
-	require.ErrorContains(t, "index 12390192 out of range", err)
+	require.ErrorContains(t, "validator index 12390192 does not exist", err)
 	require.Equal(t, false, ok)
 }
 
@@ -182,7 +182,7 @@ func TestIsNextEpochSyncCommittee_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	ok, err := IsNextPeriodSyncCommittee(state, 120391029)
-	require.ErrorContains(t, "index 120391029 out of range", err)
+	require.ErrorContains(t, "validator index 120391029 does not exist", err)
 	require.Equal(t, false, ok)
 }
 
@@ -282,7 +282,7 @@ func TestCurrentEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	index, err := CurrentPeriodSyncSubcommitteeIndices(state, 129301923)
-	require.ErrorContains(t, "index 129301923 out of range", err)
+	require.ErrorContains(t, "validator index 129301923 does not exist", err)
 	require.DeepEqual(t, []primitives.CommitteeIndex(nil), index)
 }
 
@@ -367,7 +367,7 @@ func TestNextEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	require.NoError(t, state.SetNextSyncCommittee(syncCommittee))
 
 	index, err := NextPeriodSyncSubcommitteeIndices(state, 21093019)
-	require.ErrorContains(t, "index 21093019 out of range", err)
+	require.ErrorContains(t, "validator index 21093019 does not exist", err)
 	require.DeepEqual(t, []primitives.CommitteeIndex(nil), index)
 }
 

beacon-chain/core/transition/state.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state/stateutil"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/container/trie"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
@@ -221,6 +222,18 @@ func OptimizedGenesisBeaconState(genesisTime uint64, preState state.BeaconState,
 
 // EmptyGenesisState returns an empty beacon state object.
 func EmptyGenesisState() (state.BeaconState, error) {
+	blockRoots := make([][]byte, fieldparams.BlockRootsLength)
+	for i := range blockRoots {
+		blockRoots[i] = make([]byte, fieldparams.RootLength)
+	}
+	stateRoots := make([][]byte, fieldparams.StateRootsLength)
+	for i := range stateRoots {
+		stateRoots[i] = make([]byte, fieldparams.RootLength)
+	}
+	mixes := make([][]byte, fieldparams.RandaoMixesLength)
+	for i := range mixes {
+		mixes[i] = make([]byte, fieldparams.RootLength)
+	}
 	st := &ethpb.BeaconState{
 		// Misc fields.
 		Slot: 0,
@@ -229,6 +242,9 @@ func EmptyGenesisState() (state.BeaconState, error) {
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 			Epoch:           0,
 		},
+		BlockRoots:  blockRoots,
+		StateRoots:  stateRoots,
+		RandaoMixes: mixes,
 		// Validator registry fields.
 		Validators: []*ethpb.Validator{},
 		Balances:   []uint64{},

beacon-chain/rpc/eth/beacon/handlers_pool.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -15,7 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/core"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
+	consensus_types "github.com/prysmaticlabs/prysm/v4/consensus-types"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/crypto/bls"
 	http2 "github.com/prysmaticlabs/prysm/v4/network/http"
@@ -228,8 +229,8 @@ func (s *Server) SubmitVoluntaryExit(w http.ResponseWriter, r *http.Request) {
 	}
 	val, err := headState.ValidatorAtIndexReadOnly(exit.Exit.ValidatorIndex)
 	if err != nil {
-		if outOfRangeErr, ok := err.(*state_native.ValidatorIndexOutOfRangeError); ok {
-			http2.HandleError(w, "Could not get exiting validator: "+outOfRangeErr.Error(), http.StatusBadRequest)
+		if errors.Is(err, consensus_types.ErrOutOfBounds) {
+			http2.HandleError(w, "Could not get validator: "+err.Error(), http.StatusBadRequest)
 			return
 		}
 		http2.HandleError(w, "Could not get validator: "+err.Error(), http.StatusInternalServerError)

beacon-chain/rpc/eth/beacon/handlers_pool_test.go

@@ -485,7 +485,7 @@ func TestSubmitVoluntaryExit(t *testing.T) {
 		e := &http2.DefaultErrorJson{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), e))
 		assert.Equal(t, http.StatusBadRequest, e.Code)
-		assert.Equal(t, true, strings.Contains(e.Message, "Could not get exiting validator"))
+		assert.Equal(t, true, strings.Contains(e.Message, "Could not get validator"))
 	})
 }
 

beacon-chain/rpc/eth/beacon/state_test.go

@@ -108,8 +108,8 @@ func TestGetStateRoot(t *testing.T) {
 }
 
 func TestGetRandao(t *testing.T) {
-	mixCurrent := bytesutil.PadTo([]byte("current"), 32)
-	mixOld := bytesutil.PadTo([]byte("old"), 32)
+	mixCurrent := bytesutil.ToBytes32([]byte("current"))
+	mixOld := bytesutil.ToBytes32([]byte("old"))
 	epochCurrent := primitives.Epoch(100000)
 	epochOld := 100000 - params.BeaconConfig().EpochsPerHistoricalVector + 1
 
@@ -125,7 +125,7 @@ func TestGetRandao(t *testing.T) {
 	headSt, err := util.NewBeaconState()
 	require.NoError(t, err)
 	require.NoError(t, headSt.SetSlot(params.BeaconConfig().SlotsPerEpoch))
-	headRandao := bytesutil.PadTo([]byte("head"), 32)
+	headRandao := bytesutil.ToBytes32([]byte("head"))
 	require.NoError(t, headSt.UpdateRandaoMixesAtIndex(uint64(headEpoch), headRandao))
 
 	db := dbTest.SetupDB(t)
@@ -143,25 +143,25 @@ func TestGetRandao(t *testing.T) {
 	t.Run("no epoch requested", func(t *testing.T) {
 		resp, err := server.GetRandao(ctx, &eth2.RandaoRequest{StateId: []byte("head")})
 		require.NoError(t, err)
-		assert.DeepEqual(t, mixCurrent, resp.Data.Randao)
+		assert.DeepEqual(t, mixCurrent[:], resp.Data.Randao)
 	})
 	t.Run("current epoch requested", func(t *testing.T) {
 		resp, err := server.GetRandao(ctx, &eth2.RandaoRequest{StateId: []byte("head"), Epoch: &epochCurrent})
 		require.NoError(t, err)
-		assert.DeepEqual(t, mixCurrent, resp.Data.Randao)
+		assert.DeepEqual(t, mixCurrent[:], resp.Data.Randao)
 	})
 	t.Run("old epoch requested", func(t *testing.T) {
 		resp, err := server.GetRandao(ctx, &eth2.RandaoRequest{StateId: []byte("head"), Epoch: &epochOld})
 		require.NoError(t, err)
-		assert.DeepEqual(t, mixOld, resp.Data.Randao)
+		assert.DeepEqual(t, mixOld[:], resp.Data.Randao)
 	})
 	t.Run("head state below `EpochsPerHistoricalVector`", func(t *testing.T) {
 		server.Stater = &testutil.MockStater{
 			BeaconState: headSt,
 		}
 		resp, err := server.GetRandao(ctx, &eth2.RandaoRequest{StateId: []byte("head")})
 		require.NoError(t, err)
-		assert.DeepEqual(t, headRandao, resp.Data.Randao)
+		assert.DeepEqual(t, headRandao[:], resp.Data.Randao)
 	})
 	t.Run("epoch too old", func(t *testing.T) {
 		epochTooOld := primitives.Epoch(100000 - st.RandaoMixesLength())

beacon-chain/rpc/eth/validator/BUILD.bazel

@@ -29,10 +29,10 @@ go_library(
         "//beacon-chain/rpc/eth/shared:go_default_library",
         "//beacon-chain/rpc/lookup:go_default_library",
         "//beacon-chain/state:go_default_library",
-        "//beacon-chain/state/state-native:go_default_library",
         "//beacon-chain/sync:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//consensus-types:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//consensus-types/validator:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/rpc/eth/validator/handlers.go

@@ -24,9 +24,9 @@ import (
 	rpchelpers "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
-	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
+	consensus_types "github.com/prysmaticlabs/prysm/v4/consensus-types"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	validator2 "github.com/prysmaticlabs/prysm/v4/consensus-types/validator"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -339,8 +339,8 @@ func (s *Server) SubmitBeaconCommitteeSubscription(w http.ResponseWriter, r *htt
 		subscriptions[i] = consensusItem
 		val, err := st.ValidatorAtIndexReadOnly(consensusItem.ValidatorIndex)
 		if err != nil {
-			if outOfRangeErr, ok := err.(*state_native.ValidatorIndexOutOfRangeError); ok {
-				http2.HandleError(w, "Could not get validator: "+outOfRangeErr.Error(), http.StatusBadRequest)
+			if errors.Is(err, consensus_types.ErrOutOfBounds) {
+				http2.HandleError(w, "Could not get validator: "+err.Error(), http.StatusBadRequest)
 				return
 			}
 			http2.HandleError(w, "Could not get validator: "+err.Error(), http.StatusInternalServerError)

beacon-chain/state/fieldtrie/BUILD.bazel

@@ -7,12 +7,11 @@ go_library(
         "field_trie_helpers.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/fieldtrie",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/state/state-native/custom-types:go_default_library",
         "//beacon-chain/state/state-native/types:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
-        "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

beacon-chain/state/fieldtrie/field_trie.go

@@ -31,11 +31,11 @@ type FieldTrie struct {
 // NewFieldTrie is the constructor for the field trie data structure. It creates the corresponding
 // trie according to the given parameters. Depending on whether the field is a basic/composite array
 // which is either fixed/variable length, it will appropriately determine the trie.
-func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements interface{}, length uint64) (*FieldTrie, error) {
+func NewFieldTrie(field types.FieldIndex, fieldInfo types.DataType, elements interface{}, length uint64) (*FieldTrie, error) {
 	if elements == nil {
 		return &FieldTrie{
 			field:      field,
-			dataType:   dataType,
+			dataType:   fieldInfo,
 			reference:  stateutil.NewRef(1),
 			RWMutex:    new(sync.RWMutex),
 			length:     length,
@@ -48,10 +48,10 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 		return nil, err
 	}
 
-	if err := validateElements(field, dataType, elements, length); err != nil {
+	if err := validateElements(field, fieldInfo, elements, length); err != nil {
 		return nil, err
 	}
-	switch dataType {
+	switch fieldInfo {
 	case types.BasicArray:
 		fl, err := stateutil.ReturnTrieLayer(fieldRoots, length)
 		if err != nil {
@@ -60,7 +60,7 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 		return &FieldTrie{
 			fieldLayers: fl,
 			field:       field,
-			dataType:    dataType,
+			dataType:    fieldInfo,
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
@@ -70,14 +70,14 @@ func NewFieldTrie(field types.FieldIndex, dataType types.DataType, elements inte
 		return &FieldTrie{
 			fieldLayers: stateutil.ReturnTrieLayerVariable(fieldRoots, length),
 			field:       field,
-			dataType:    dataType,
+			dataType:    fieldInfo,
 			reference:   stateutil.NewRef(1),
 			RWMutex:     new(sync.RWMutex),
 			length:      length,
 			numOfElems:  reflect.Indirect(reflect.ValueOf(elements)).Len(),
 		}, nil
 	default:
-		return nil, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(dataType).Name())
+		return nil, errors.Errorf("unrecognized data type in field map: %v", reflect.TypeOf(fieldInfo).Name())
 	}
 }