Implemented wait if heimdall is not synced to the chain (#13807)

cmd/devnet/services/polygon/heimdall.go

@@ -221,6 +221,10 @@ func (h *Heimdall) getSpanOverrideHeight() uint64 {
 	//MainChain: 8664000
 }
 
+func (h *Heimdall) FetchStatus(ctx context.Context) (*heimdall.Status, error) {
+	return nil, errors.New("TODO")
+}
+
 func (h *Heimdall) FetchCheckpoint(ctx context.Context, number int64) (*heimdall.Checkpoint, error) {
 	return nil, errors.New("TODO")
 }

cmd/devnet/services/polygon/heimdallsim/heimdall_simulator.go

@@ -228,6 +228,10 @@ func (h *HeimdallSimulator) FetchStateSyncEvent(ctx context.Context, id uint64)
 	return nil, errors.New("method FetchStateSyncEvent not implemented")
 }
 
+func (h *HeimdallSimulator) FetchStatus(ctx context.Context) (*heimdall.Status, error) {
+	return nil, errors.New("method FetchStatus not implemented")
+}
+
 func (h *HeimdallSimulator) FetchCheckpoint(ctx context.Context, number int64) (*heimdall.Checkpoint, error) {
 	return nil, errors.New("method FetchCheckpoint not implemented")
 }

polygon/bor/bor_test.go

@@ -79,6 +79,10 @@ func (h *test_heimdall) FetchStateSyncEvent(ctx context.Context, id uint64) (*he
 	return nil, nil
 }
 
+func (h *test_heimdall) FetchStatus(ctx context.Context) (*heimdall.Status, error) {
+	return nil, nil
+}
+
 func (h *test_heimdall) FetchSpan(ctx context.Context, spanID uint64) (*heimdall.Span, error) {
 
 	if span, ok := h.spans[heimdall.SpanId(spanID)]; ok {

polygon/heimdall/client.go

@@ -30,6 +30,8 @@ type Client interface {
 	FetchSpan(ctx context.Context, spanID uint64) (*Span, error)
 	FetchSpans(ctx context.Context, page uint64, limit uint64) ([]*Span, error)
 
+	FetchStatus(ctx context.Context) (*Status, error)
+
 	FetchCheckpoint(ctx context.Context, number int64) (*Checkpoint, error)
 	FetchCheckpointCount(ctx context.Context) (int64, error)
 	FetchCheckpoints(ctx context.Context, page uint64, limit uint64) ([]*Checkpoint, error)

polygon/heimdall/client_http.go

@@ -125,6 +125,8 @@ const (
 	fetchStateSyncEventsPath   = "clerk/event-record/list"
 	fetchStateSyncEvent        = "clerk/event-record/%s"
 
+	fetchStatus = "/status"
+
 	fetchCheckpoint                = "/checkpoints/%s"
 	fetchCheckpointCount           = "/checkpoints/count"
 	fetchCheckpointList            = "/checkpoints/list"
@@ -349,6 +351,22 @@ func (c *HttpClient) FetchMilestone(ctx context.Context, number int64) (*Milesto
 	return &response.Result, nil
 }
 
+func (c *HttpClient) FetchStatus(ctx context.Context) (*Status, error) {
+	url, err := statusURL(c.urlString)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx = withRequestType(ctx, statusRequest)
+
+	response, err := FetchWithRetry[StatusResponse](ctx, c, url, c.logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &response.Result, nil
+}
+
 // FetchCheckpointCount fetches the checkpoint count from heimdall
 func (c *HttpClient) FetchCheckpointCount(ctx context.Context) (int64, error) {
 	url, err := checkpointCountURL(c.urlString)
@@ -587,6 +605,10 @@ func checkpointCountURL(urlString string) (*url.URL, error) {
 	return makeURL(urlString, fetchCheckpointCount, "")
 }
 
+func statusURL(urlString string) (*url.URL, error) {
+	return makeURL(urlString, fetchStatus, "")
+}
+
 func checkpointListURL(urlString string, page uint64, limit uint64) (*url.URL, error) {
 	return makeURL(urlString, fetchCheckpointList, fmt.Sprintf(fetchCheckpointListQueryFormat, page, limit))
 }

polygon/heimdall/metrics.go

@@ -34,6 +34,7 @@ type (
 )
 
 const (
+	statusRequest             requestType = "status"
 	stateSyncRequest          requestType = "state-sync"
 	spanRequest               requestType = "span"
 	checkpointRequest         requestType = "checkpoint"

polygon/heimdall/service.go

@@ -32,6 +32,10 @@ import (
 	"github.com/erigontech/erigon/polygon/bor/valset"
 )
 
+const (
+	isCatchingDelaySec = 600
+)
+
 type ServiceConfig struct {
 	Store     Store
 	BorConfig *borcfg.BorConfig
@@ -47,6 +51,7 @@ type Service struct {
 	milestoneScraper          *Scraper[*Milestone]
 	spanScraper               *Scraper[*Span]
 	spanBlockProducersTracker *spanBlockProducersTracker
+	client                    Client
 	ready                     ready
 }
 
@@ -100,6 +105,7 @@ func NewService(config ServiceConfig) *Service {
 		milestoneScraper:          milestoneScraper,
 		spanScraper:               spanScraper,
 		spanBlockProducersTracker: newSpanBlockProducersTracker(logger, borConfig, store.SpanBlockProducerSelections()),
+		client:                    client,
 	}
 }
 
@@ -397,3 +403,25 @@ func (s *Service) replayUntrackedSpans(ctx context.Context) error {
 
 	return nil
 }
+
+func (s *Service) IsCatchingUp(ctx context.Context) (bool, error) {
+	status, err := s.client.FetchStatus(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	if status.CatchingUp {
+		return true, nil
+	}
+
+	parsed, err := time.Parse(time.RFC3339, status.LatestBlockTime)
+	if err != nil {
+		return false, err
+	}
+
+	if parsed.Unix() < time.Now().Unix()-isCatchingDelaySec {
+		return true, nil
+	}
+
+	return false, nil
+}

polygon/heimdall/service_test.go

@@ -489,3 +489,47 @@ type difficultiesKV struct {
 	Signer     common.Address
 	Difficulty uint64
 }
+
+func TestIsCatchingUp(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	mockClient := NewMockClient(ctrl)
+
+	s := Service{
+		client: mockClient,
+	}
+
+	mockClient.EXPECT().
+		FetchStatus(gomock.Any()).
+		DoAndReturn(func(ctx context.Context) (*Status, error) {
+			return &Status{
+				LatestBlockTime: "",
+				CatchingUp:      true,
+			}, nil
+		})
+
+	isCatchingUp, err := s.IsCatchingUp(context.TODO())
+	require.NoError(t, err)
+	require.True(t, isCatchingUp)
+}
+
+func TestIsCatchingUpLateBlock(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	mockClient := NewMockClient(ctrl)
+
+	s := Service{
+		client: mockClient,
+	}
+
+	mockClient.EXPECT().
+		FetchStatus(gomock.Any()).
+		DoAndReturn(func(ctx context.Context) (*Status, error) {
+			return &Status{
+				LatestBlockTime: "2025-02-14T11:45:00.764588Z",
+				CatchingUp:      false,
+			}, nil
+		})
+
+	isCatchingUp, err := s.IsCatchingUp(context.TODO())
+	require.NoError(t, err)
+	require.True(t, isCatchingUp)
+}

polygon/heimdall/status.go

@@ -0,0 +1,30 @@
+// Copyright 2024 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
+
+package heimdall
+
+type Status struct {
+	LatestBlockHash   string `json:"latest_block_hash"`
+	LatestAppHash     string `json:"latest_app_hash"`
+	LatestBlockHeight string `json:"latest_block_height"`
+	LatestBlockTime   string `json:"latest_block_time"`
+	CatchingUp        bool   `json:"catching_up"`
+}
+
+type StatusResponse struct {
+	Height string `json:"height"`
+	Result Status `json:"result"`
+}

polygon/sync/sync.go

@@ -25,6 +25,7 @@ import (
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 
 	"github.com/erigontech/erigon-lib/common"
+	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon/core/types"
 	"github.com/erigontech/erigon/eth/ethconfig"
@@ -34,6 +35,7 @@ import (
 )
 
 type heimdallSynchronizer interface {
+	IsCatchingUp(ctx context.Context) (bool, error)
 	SynchronizeCheckpoints(ctx context.Context) (latest *heimdall.Checkpoint, err error)
 	SynchronizeMilestones(ctx context.Context) (latest *heimdall.Milestone, err error)
 	SynchronizeSpans(ctx context.Context, blockNum uint64) error
@@ -668,6 +670,26 @@ func (s *Sync) Run(ctx context.Context) error {
 
 	s.logger.Info(syncLogPrefix("running sync component"))
 
+	for {
+		// we have to check if the heimdall we are connected to is synchonised with the chain
+		// to prevent getting empty list of checkpoints/milestones during the sync
+
+		catchingUp, err := s.heimdallSync.IsCatchingUp(ctx)
+		if err != nil {
+			return err
+		}
+
+		if !catchingUp {
+			break
+		}
+
+		s.logger.Warn(syncLogPrefix("your heimdalld process is behind, please check its logs and <HEIMDALL_HOST>:1317/status api"))
+
+		if err := libcommon.Sleep(ctx, 30*time.Second); err != nil {
+			return err
+		}
+	}
+
 	result, err := s.syncToTip(ctx)
 	if err != nil {
 		return err