From 8cf0fed2f6fc13c24ec7d6860e5f297372a88b30 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Wed, 8 Oct 2025 18:27:40 +0200 Subject: [PATCH 01/14] feat: add Reth Backup Helper script for MDBX database snapshots --- scripts/reth-backup/README.md | 40 +++++++++ scripts/reth-backup/backup.sh | 162 ++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 scripts/reth-backup/README.md create mode 100755 scripts/reth-backup/backup.sh diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md new file mode 100644 index 000000000..3fb06fb56 --- /dev/null +++ b/scripts/reth-backup/README.md @@ -0,0 +1,40 @@ +# Reth Backup Helper + +Script to snapshot the `ev-reth` MDBX database while the node keeps running and +record the block height contained in the snapshot. + +## Prerequisites + +- Docker access to the container running `ev-reth` (defaults to the service name + `ev-reth` from `docker-compose`). +- The `mdbx_copy` binary available inside that container. If it is not provided + by the image, compile it once inside the container (see [libmdbx + documentation](https://libmdbx.dqdkfa.ru/)). +- `jq` installed on the host to parse the JSON output. + +## Usage + +```bash +./scripts/reth-backup/backup.sh \ + --container ev-reth \ + --datadir /home/reth/eth-home \ + --mdbx-copy /tmp/libmdbx/build/mdbx_copy \ + /path/to/backups +``` + +This creates a timestamped folder under `/path/to/backups` with: + +- `db/mdbx.dat` – consistent MDBX snapshot. +- `db/mdbx.lck` – placeholder lock file (empty). +- `static_files/` – static files copied from the node. +- `stage_checkpoints.json` – raw StageCheckpoints table. +- `height.txt` – extracted block height (from the `Finish` stage). + +Additional flags: + +- `--tag LABEL` to override the timestamped folder name. +- `--keep-remote` to leave the temporary snapshot inside the container (useful + for debugging). + +The script outputs the height at the end so you can coordinate other backups +with the same block number. diff --git a/scripts/reth-backup/backup.sh b/scripts/reth-backup/backup.sh new file mode 100755 index 000000000..956939f6e --- /dev/null +++ b/scripts/reth-backup/backup.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: backup.sh [OPTIONS] + +Create a consistent backup of the ev-reth database using mdbx_copy and record +the block height captured in the snapshot. + +Options: + --container NAME Docker container name running ev-reth (default: ev-reth) + --datadir PATH Path to the reth datadir inside the container + (default: /home/reth/eth-home) + --mdbx-copy CMD Path to the mdbx_copy binary inside the container + (default: mdbx_copy; override if you compiled it elsewhere) + --tag LABEL Custom label for the backup directory (default: timestamp) + --keep-remote Leave the temporary snapshot inside the container + -h, --help Show this help message + +Requirements: + - Docker access to the container running ev-reth. + - mdbx_copy available inside that container (compile it once if necessary). + - jq installed on the host (used to parse StageCheckpoints JSON). + +The destination directory will receive: + //db/mdbx.dat MDBX snapshot + //db/mdbx.lck Empty lock file placeholder + //static_files/... Static files copied from the node + //stage_checkpoints.json + //height.txt Height extracted from StageCheckpoints +EOF +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command '$1' not found in PATH" >&2 + exit 1 + } +} + +DEST="" +CONTAINER="ev-reth" +DATADIR="/home/reth/eth-home" +MDBX_COPY="mdbx_copy" +BACKUP_TAG="" +KEEP_REMOTE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --container) + CONTAINER="$2" + shift 2 + ;; + --datadir) + DATADIR="$2" + shift 2 + ;; + --mdbx-copy) + MDBX_COPY="$2" + shift 2 + ;; + --tag) + BACKUP_TAG="$2" + shift 2 + ;; + --keep-remote) + KEEP_REMOTE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + if [[ -z "$DEST" ]]; then + DEST="$1" + shift + else + echo "unexpected argument: $1" >&2 + usage >&2 + exit 1 + fi + ;; + esac +done + +if [[ -z "$DEST" ]]; then + echo "error: destination directory is required" >&2 + usage >&2 + exit 1 +fi + +require_cmd docker +require_cmd jq + +if [[ -z "$BACKUP_TAG" ]]; then + BACKUP_TAG="$(date +'%Y%m%d-%H%M%S')" +fi + +REMOTE_TMP="/tmp/reth-backup-${BACKUP_TAG}" +HOST_DEST="$(mkdir -p "$DEST" && cd "$DEST" && pwd)/${BACKUP_TAG}" + +echo "Creating backup tag '$BACKUP_TAG' into ${HOST_DEST}" + +# Prepare temporary workspace inside the container. +docker exec "$CONTAINER" bash -c "rm -rf '$REMOTE_TMP' && mkdir -p '$REMOTE_TMP/db' '$REMOTE_TMP/static_files'" + +# Verify mdbx_copy availability. +if ! docker exec "$CONTAINER" bash -lc "command -v '$MDBX_COPY' >/dev/null 2>&1 || [ -x '$MDBX_COPY' ]"; then + echo "error: unable to find executable '$MDBX_COPY' inside container '$CONTAINER'" >&2 + exit 1 +fi + +echo "Running mdbx_copy inside container..." +docker exec "$CONTAINER" bash -lc "'$MDBX_COPY' --compact '${DATADIR}/db' '$REMOTE_TMP/db/mdbx.dat'" +docker exec "$CONTAINER" bash -lc "touch '$REMOTE_TMP/db/mdbx.lck'" + +echo "Copying static_files..." +docker exec "$CONTAINER" bash -lc "if [ -d '${DATADIR}/static_files' ]; then cp -a '${DATADIR}/static_files/.' '$REMOTE_TMP/static_files/' 2>/dev/null || true; fi" + +echo "Querying StageCheckpoints height..." +STAGE_JSON=$(docker exec "$CONTAINER" ev-reth db --datadir "$REMOTE_TMP" list StageCheckpoints --len 20 --json) +HEIGHT=$(echo "$STAGE_JSON" | jq -r '.[] | select(.[0]=="Finish") | .[1].block_number' | tr -d '\r\n') + +if [[ -z "$HEIGHT" || "$HEIGHT" == "null" ]]; then + echo "warning: could not determine height from StageCheckpoints" >&2 +fi + +echo "Copying snapshot to host..." +mkdir -p "$HOST_DEST/db" +docker cp "${CONTAINER}:${REMOTE_TMP}/db/mdbx.dat" "$HOST_DEST/db/mdbx.dat" +docker cp "${CONTAINER}:${REMOTE_TMP}/db/mdbx.lck" "$HOST_DEST/db/mdbx.lck" + +if docker exec "$CONTAINER" test -d "${REMOTE_TMP}/static_files"; then + mkdir -p "$HOST_DEST/static_files" + docker cp "${CONTAINER}:${REMOTE_TMP}/static_files/." "$HOST_DEST/static_files/" +fi + +echo "$STAGE_JSON" > "$HOST_DEST/stage_checkpoints.json" +if [[ -n "$HEIGHT" && "$HEIGHT" != "null" ]]; then + echo "$HEIGHT" > "$HOST_DEST/height.txt" + echo "Backup height: $HEIGHT" +else + echo "Height not captured (see stage_checkpoints.json for details)" +fi + +if [[ "$KEEP_REMOTE" -ne 1 ]]; then + docker exec "$CONTAINER" rm -rf "$REMOTE_TMP" +fi + +echo "Backup completed: $HOST_DEST" From 8abdd83ae1c099fbf8d3787a72bd40e3130802b8 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Wed, 8 Oct 2025 18:47:36 +0200 Subject: [PATCH 02/14] feat: add Dockerfile for mdbx_copy setup and update backup script for compact mode --- scripts/reth-backup/Dockerfile | 24 ++++++++++++++++++++++++ scripts/reth-backup/backup.sh | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 scripts/reth-backup/Dockerfile diff --git a/scripts/reth-backup/Dockerfile b/scripts/reth-backup/Dockerfile new file mode 100644 index 000000000..e328bae19 --- /dev/null +++ b/scripts/reth-backup/Dockerfile @@ -0,0 +1,24 @@ +FROM ghcr.io/evstack/ev-reth:latest + +ARG LIBMDBX_REPO=https://github.com/erthink/libmdbx.git +ARG LIBMDBX_REF=master + +RUN set -eux; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + git \ + jq \ + ; \ + rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + git clone --depth 1 --branch "${LIBMDBX_REF}" "${LIBMDBX_REPO}" /tmp/libmdbx; \ + cmake -S /tmp/libmdbx -B /tmp/libmdbx/build -DCMAKE_BUILD_TYPE=Release; \ + cmake --build /tmp/libmdbx/build --target mdbx_copy mdbx_dump mdbx_chk; \ + install -m 0755 /tmp/libmdbx/build/mdbx_copy /usr/local/bin/mdbx_copy; \ + install -m 0755 /tmp/libmdbx/build/mdbx_dump /usr/local/bin/mdbx_dump; \ + install -m 0755 /tmp/libmdbx/build/mdbx_chk /usr/local/bin/mdbx_chk; \ + rm -rf /tmp/libmdbx diff --git a/scripts/reth-backup/backup.sh b/scripts/reth-backup/backup.sh index 956939f6e..7983ba4bd 100755 --- a/scripts/reth-backup/backup.sh +++ b/scripts/reth-backup/backup.sh @@ -37,7 +37,7 @@ require_cmd() { if ! command -v "$1" >/dev/null 2>&1; then echo "error: required command '$1' not found in PATH" >&2 exit 1 - } + fi } DEST="" @@ -123,14 +123,14 @@ if ! docker exec "$CONTAINER" bash -lc "command -v '$MDBX_COPY' >/dev/null 2>&1 fi echo "Running mdbx_copy inside container..." -docker exec "$CONTAINER" bash -lc "'$MDBX_COPY' --compact '${DATADIR}/db' '$REMOTE_TMP/db/mdbx.dat'" +docker exec "$CONTAINER" bash -lc "'$MDBX_COPY' -c '${DATADIR}/db' '$REMOTE_TMP/db/mdbx.dat'" docker exec "$CONTAINER" bash -lc "touch '$REMOTE_TMP/db/mdbx.lck'" echo "Copying static_files..." docker exec "$CONTAINER" bash -lc "if [ -d '${DATADIR}/static_files' ]; then cp -a '${DATADIR}/static_files/.' '$REMOTE_TMP/static_files/' 2>/dev/null || true; fi" echo "Querying StageCheckpoints height..." -STAGE_JSON=$(docker exec "$CONTAINER" ev-reth db --datadir "$REMOTE_TMP" list StageCheckpoints --len 20 --json) +STAGE_JSON=$(docker exec "$CONTAINER" ev-reth db --datadir "$REMOTE_TMP" list StageCheckpoints --len 20 --json | sed -n '/^\[/,$p') HEIGHT=$(echo "$STAGE_JSON" | jq -r '.[] | select(.[0]=="Finish") | .[1].block_number' | tr -d '\r\n') if [[ -z "$HEIGHT" || "$HEIGHT" == "null" ]]; then From 9663554a3fc279ba16233cfc462d532e8ce50e9a Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 9 Oct 2025 16:42:15 +0200 Subject: [PATCH 03/14] feat: implement Backup functionality for datastore - Added Backup method to Store interface and DefaultStore implementation to stream a Badger backup of the datastore. - Introduced BackupRequest and BackupResponse messages in the state_rpc.proto file to handle backup requests and responses. - Implemented backup streaming logic in StoreServer, including metadata handling for current and target heights. - Created a backupStreamWriter to manage chunked writing of backup data. - Updated client tests to validate the Backup functionality. - Enhanced mock store to support Backup method for testing. - Added unit tests for Backup functionality in the store package. --- pkg/rpc/client/client.go | 50 +++ pkg/rpc/client/client_test.go | 31 ++ pkg/rpc/server/server.go | 161 ++++++++++ pkg/store/backup.go | 43 +++ pkg/store/store_test.go | 22 ++ pkg/store/types.go | 5 + proto/evnode/v1/state_rpc.proto | 41 ++- test/mocks/store.go | 73 +++++ types/pb/evnode/v1/batch.pb.go | 2 +- types/pb/evnode/v1/config.pb.go | 2 +- types/pb/evnode/v1/evnode.pb.go | 2 +- types/pb/evnode/v1/execution.pb.go | 2 +- types/pb/evnode/v1/health.pb.go | 2 +- types/pb/evnode/v1/p2p_rpc.pb.go | 2 +- types/pb/evnode/v1/signer.pb.go | 2 +- types/pb/evnode/v1/state.pb.go | 2 +- types/pb/evnode/v1/state_rpc.pb.go | 290 ++++++++++++++++-- .../pb/evnode/v1/v1connect/config.connect.go | 4 +- .../evnode/v1/v1connect/state_rpc.connect.go | 30 ++ 19 files changed, 724 insertions(+), 42 deletions(-) create mode 100644 pkg/store/backup.go diff --git a/pkg/rpc/client/client.go b/pkg/rpc/client/client.go index 316b028f6..34cce4614 100644 --- a/pkg/rpc/client/client.go +++ b/pkg/rpc/client/client.go @@ -2,6 +2,8 @@ package client import ( "context" + "fmt" + "io" "net/http" "connectrpc.com/connect" @@ -92,6 +94,54 @@ func (c *Client) GetMetadata(ctx context.Context, key string) ([]byte, error) { return resp.Msg.Value, nil } +// Backup streams a datastore backup into the provided writer and returns the final metadata emitted by the server. +// The writer is not closed by this method. +func (c *Client) Backup(ctx context.Context, params *pb.BackupRequest, dst io.Writer) (*pb.BackupMetadata, error) { + if dst == nil { + return nil, fmt.Errorf("backup destination writer cannot be nil") + } + + if params == nil { + params = &pb.BackupRequest{} + } + + stream, err := c.storeClient.Backup(ctx, connect.NewRequest(params)) + if err != nil { + return nil, err + } + defer stream.Close() // Best effort; ignore close error to preserve primary result. + + var lastMetadata *pb.BackupMetadata + for stream.Receive() { + msg := stream.Msg() + if metadata := msg.GetMetadata(); metadata != nil { + lastMetadata = metadata + continue + } + + if chunk := msg.GetChunk(); chunk != nil { + if _, err := dst.Write(chunk); err != nil { + _ = stream.Close() + return lastMetadata, fmt.Errorf("failed to write backup chunk: %w", err) + } + } + } + + if err := stream.Err(); err != nil { + return lastMetadata, err + } + + if lastMetadata == nil { + return nil, fmt.Errorf("backup stream completed without metadata") + } + + if !lastMetadata.GetCompleted() { + return lastMetadata, fmt.Errorf("backup stream ended without completion metadata") + } + + return lastMetadata, nil +} + // GetPeerInfo returns information about the connected peers func (c *Client) GetPeerInfo(ctx context.Context) ([]*pb.PeerInfo, error) { req := connect.NewRequest(&emptypb.Empty{}) diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index 97c247513..fa7080420 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -1,7 +1,9 @@ package client import ( + "bytes" "context" + "io" "net/http" "net/http/httptest" "testing" @@ -20,6 +22,7 @@ import ( "github.com/evstack/ev-node/pkg/rpc/server" "github.com/evstack/ev-node/test/mocks" "github.com/evstack/ev-node/types" + pb "github.com/evstack/ev-node/types/pb/evnode/v1" rpc "github.com/evstack/ev-node/types/pb/evnode/v1/v1connect" ) @@ -173,6 +176,34 @@ func TestClientGetBlockByHash(t *testing.T) { mockStore.AssertExpectations(t) } +func TestClientBackup(t *testing.T) { + mockStore := mocks.NewMockStore(t) + mockP2P := mocks.NewMockP2PRPC(t) + + mockStore.On("Height", mock.Anything).Return(uint64(15), nil) + mockStore.On("Backup", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + writer := args.Get(1).(io.Writer) + _, _ = writer.Write([]byte("chunk-1")) + _, _ = writer.Write([]byte("chunk-2")) + }).Return(uint64(42), nil) + + testServer, client := setupTestServer(t, mockStore, mockP2P) + defer testServer.Close() + + var buf bytes.Buffer + metadata, err := client.Backup(context.Background(), &pb.BackupRequest{TargetHeight: 10}, &buf) + require.NoError(t, err) + require.NotNil(t, metadata) + require.Equal(t, "chunk-1chunk-2", buf.String()) + require.True(t, metadata.GetCompleted()) + require.Equal(t, uint64(15), metadata.GetCurrentHeight()) + require.Equal(t, uint64(10), metadata.GetTargetHeight()) + require.Equal(t, uint64(0), metadata.GetSinceVersion()) + require.Equal(t, uint64(42), metadata.GetLastVersion()) + + mockStore.AssertExpectations(t) +} + func TestClientGetPeerInfo(t *testing.T) { // Create mocks mockStore := mocks.NewMockStore(t) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 290994be3..109568e0c 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "io" "net/http" "time" @@ -188,6 +189,166 @@ func (s *StoreServer) GetMetadata( }), nil } +// Backup streams a Badger backup of the datastore so it can be persisted externally. +func (s *StoreServer) Backup( + ctx context.Context, + req *connect.Request[pb.BackupRequest], + stream *connect.ServerStream[pb.BackupResponse], +) error { + since := req.Msg.GetSinceVersion() + targetHeight := req.Msg.GetTargetHeight() + + currentHeight, err := s.store.Height(ctx) + if err != nil { + return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to get current height: %w", err)) + } + + if targetHeight != 0 && targetHeight > currentHeight { + return connect.NewError( + connect.CodeFailedPrecondition, + fmt.Errorf("requested target height %d exceeds current height %d", targetHeight, currentHeight), + ) + } + + initialMetadata := &pb.BackupMetadata{ + CurrentHeight: currentHeight, + TargetHeight: targetHeight, + SinceVersion: since, + Completed: false, + LastVersion: 0, + } + + if err := stream.Send(&pb.BackupResponse{ + Response: &pb.BackupResponse_Metadata{ + Metadata: initialMetadata, + }, + }); err != nil { + return err + } + + writer := newBackupStreamWriter(stream, defaultBackupChunkSize) + version, err := s.store.Backup(ctx, writer, since) + if err != nil { + var connectErr *connect.Error + if errors.As(err, &connectErr) { + return connectErr + } + if errors.Is(err, context.Canceled) { + return connect.NewError(connect.CodeCanceled, err) + } + if errors.Is(err, context.DeadlineExceeded) { + return connect.NewError(connect.CodeDeadlineExceeded, err) + } + return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to execute backup: %w", err)) + } + + if err := writer.Flush(); err != nil { + var connectErr *connect.Error + if errors.As(err, &connectErr) { + return connectErr + } + if errors.Is(err, context.Canceled) { + return connect.NewError(connect.CodeCanceled, err) + } + if errors.Is(err, context.DeadlineExceeded) { + return connect.NewError(connect.CodeDeadlineExceeded, err) + } + return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to flush backup stream: %w", err)) + } + + completedMetadata := &pb.BackupMetadata{ + CurrentHeight: currentHeight, + TargetHeight: targetHeight, + SinceVersion: since, + LastVersion: version, + Completed: true, + } + + if err := stream.Send(&pb.BackupResponse{ + Response: &pb.BackupResponse_Metadata{ + Metadata: completedMetadata, + }, + }); err != nil { + return err + } + + return nil +} + +const defaultBackupChunkSize = 128 * 1024 + +var _ io.Writer = (*backupStreamWriter)(nil) + +type backupStreamWriter struct { + stream *connect.ServerStream[pb.BackupResponse] + buf []byte + chunkSize int +} + +func newBackupStreamWriter(stream *connect.ServerStream[pb.BackupResponse], chunkSize int) *backupStreamWriter { + if chunkSize <= 0 { + chunkSize = defaultBackupChunkSize + } + + return &backupStreamWriter{ + stream: stream, + buf: make([]byte, 0, chunkSize), + chunkSize: chunkSize, + } +} + +func (w *backupStreamWriter) Write(p []byte) (int, error) { + written := 0 + for len(p) > 0 { + space := w.chunkSize - len(w.buf) + if space == 0 { + if err := w.flush(); err != nil { + return written, err + } + space = w.chunkSize - len(w.buf) + } + + if space > len(p) { + space = len(p) + } + + w.buf = append(w.buf, p[:space]...) + p = p[space:] + written += space + + if len(w.buf) == w.chunkSize { + if err := w.flush(); err != nil { + return written, err + } + } + } + return written, nil +} + +func (w *backupStreamWriter) Flush() error { + return w.flush() +} + +func (w *backupStreamWriter) flush() error { + if len(w.buf) == 0 { + return nil + } + + chunk := make([]byte, len(w.buf)) + copy(chunk, w.buf) + + if err := w.stream.Send(&pb.BackupResponse{ + Response: &pb.BackupResponse_Chunk{ + Chunk: chunk, + }, + }); err != nil { + return err + } + + w.buf = w.buf[:0] + return nil +} + type ConfigServer struct { config config.Config signer []byte diff --git a/pkg/store/backup.go b/pkg/store/backup.go new file mode 100644 index 000000000..6e7a9fbc9 --- /dev/null +++ b/pkg/store/backup.go @@ -0,0 +1,43 @@ +package store + +import ( + "context" + "fmt" + "io" + + badger4 "github.com/ipfs/go-ds-badger4" +) + +// Backup streams the underlying Badger datastore snapshot into the provided writer. +// The returned uint64 corresponds to the last version contained in the backup stream, +// which can be re-used to generate incremental backups via the since parameter. +func (s *DefaultStore) Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + + // Try to leverage a native backup implementation if the underlying datastore exposes one. + type backupable interface { + Backup(io.Writer, uint64) (uint64, error) + } + if dsBackup, ok := s.db.(backupable); ok { + version, err := dsBackup.Backup(writer, since) + if err != nil { + return 0, fmt.Errorf("datastore backup failed: %w", err) + } + return version, nil + } + + // Default Badger datastore used across ev-node. + badgerDatastore, ok := s.db.(*badger4.Datastore) + if !ok { + return 0, fmt.Errorf("backup is not supported by the configured datastore") + } + + // `badger.DB.Backup` internally orchestrates a consistent snapshot without pausing writes. + version, err := badgerDatastore.DB.Backup(writer, since) + if err != nil { + return 0, fmt.Errorf("badger backup failed: %w", err) + } + return version, nil +} diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index dd3c32fb0..bd7d0ba79 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "encoding/binary" "errors" @@ -1127,3 +1128,24 @@ func TestRollbackDAIncludedHeightGetMetadataError(t *testing.T) { require.Contains(err.Error(), "failed to get DA included height") require.Contains(err.Error(), "metadata retrieval failed") } + +func TestDefaultStoreBackup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kv, err := NewDefaultInMemoryKVStore() + require.NoError(t, err) + + s := New(kv) + t.Cleanup(func() { + require.NoError(t, s.Close()) + }) + + require.NoError(t, s.SetMetadata(ctx, "backup-test", []byte("value"))) + + var buf bytes.Buffer + version, err := s.Backup(ctx, &buf, 0) + require.NoError(t, err) + require.NotZero(t, buf.Len()) + require.NotZero(t, version) +} diff --git a/pkg/store/types.go b/pkg/store/types.go index a50d9f375..c2f214779 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -2,6 +2,7 @@ package store import ( "context" + "io" "github.com/evstack/ev-node/types" ) @@ -50,6 +51,10 @@ type Store interface { // Aggregator is used to determine if the rollback is performed on the aggregator node. Rollback(ctx context.Context, height uint64, aggregator bool) error + // Backup writes a consistent backup stream to writer. The returned version can be used + // as the starting point for incremental backups. + Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) + // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error } diff --git a/proto/evnode/v1/state_rpc.proto b/proto/evnode/v1/state_rpc.proto index 1b468b6ef..3f71a6f1e 100644 --- a/proto/evnode/v1/state_rpc.proto +++ b/proto/evnode/v1/state_rpc.proto @@ -1,10 +1,9 @@ syntax = "proto3"; package evnode.v1; -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; import "evnode/v1/evnode.proto"; import "evnode/v1/state.proto"; +import "google/protobuf/empty.proto"; option go_package = "github.com/evstack/ev-node/types/pb/evnode/v1"; @@ -21,12 +20,15 @@ service StoreService { // GetGenesisDaHeight returns the DA height at which the first Evolve block was included. rpc GetGenesisDaHeight(google.protobuf.Empty) returns (GetGenesisDaHeightResponse) {} + + // Backup streams a Badger backup of the datastore so it can be persisted externally. + rpc Backup(BackupRequest) returns (stream BackupResponse) {} } // Block contains all the components of a complete block message Block { SignedHeader header = 1; - Data data = 2; + Data data = 2; } // GetBlockRequest defines the request for retrieving a block @@ -34,15 +36,15 @@ message GetBlockRequest { // The height or hash of the block to retrieve oneof identifier { uint64 height = 1; - bytes hash = 2; + bytes hash = 2; } } // GetBlockResponse defines the response for retrieving a block message GetBlockResponse { - Block block = 1; + Block block = 1; uint64 header_da_height = 2; - uint64 data_da_height = 3; + uint64 data_da_height = 3; } // GetStateResponse defines the response for retrieving the current state @@ -62,5 +64,30 @@ message GetMetadataResponse { // GetGenesisDaHeightResponse defines the DA height at which the first Evolve block was included. message GetGenesisDaHeightResponse { - uint64 height = 3; + uint64 height = 3; +} + +// BackupRequest defines the parameters for requesting a datastore backup. +message BackupRequest { + // target_height is the Evolve height the client wants to align with. 0 skips the check. + uint64 target_height = 1; + // since_version allows incremental backups. 0 produces a full backup. + uint64 since_version = 2; +} + +// BackupMetadata contains progress or completion details emitted during a backup stream. +message BackupMetadata { + uint64 current_height = 1; + uint64 target_height = 2; + uint64 since_version = 3; + uint64 last_version = 4; + bool completed = 5; +} + +// BackupResponse multiplexes metadata and raw backup data chunks in the stream. +message BackupResponse { + oneof response { + BackupMetadata metadata = 1; + bytes chunk = 2; + } } diff --git a/test/mocks/store.go b/test/mocks/store.go index 278872f19..0156ae605 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -6,6 +6,7 @@ package mocks import ( "context" + "io" "github.com/evstack/ev-node/types" mock "github.com/stretchr/testify/mock" @@ -82,6 +83,78 @@ func (_c *MockStore_Close_Call) RunAndReturn(run func() error) *MockStore_Close_ return _c } +// Backup provides a mock function for the type MockStore +func (_mock *MockStore) Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) { + ret := _mock.Called(ctx, writer, since) + + if len(ret) == 0 { + panic("no return value specified for Backup") + } + + var r0 uint64 + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, io.Writer, uint64) (uint64, error)); ok { + return returnFunc(ctx, writer, since) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, io.Writer, uint64) uint64); ok { + r0 = returnFunc(ctx, writer, since) + } else { + r0 = ret.Get(0).(uint64) + } + if returnFunc, ok := ret.Get(1).(func(context.Context, io.Writer, uint64) error); ok { + r1 = returnFunc(ctx, writer, since) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockStore_Backup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Backup' +type MockStore_Backup_Call struct { + *mock.Call +} + +// Backup is a helper method to define mock.On call +// - ctx context.Context +// - writer io.Writer +// - since uint64 +func (_e *MockStore_Expecter) Backup(ctx interface{}, writer interface{}, since interface{}) *MockStore_Backup_Call { + return &MockStore_Backup_Call{Call: _e.mock.On("Backup", ctx, writer, since)} +} + +func (_c *MockStore_Backup_Call) Run(run func(ctx context.Context, writer io.Writer, since uint64)) *MockStore_Backup_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 io.Writer + if args[1] != nil { + arg1 = args[1].(io.Writer) + } + var arg2 uint64 + if args[2] != nil { + arg2 = args[2].(uint64) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockStore_Backup_Call) Return(version uint64, err error) *MockStore_Backup_Call { + _c.Call.Return(version, err) + return _c +} + +func (_c *MockStore_Backup_Call) RunAndReturn(run func(ctx context.Context, writer io.Writer, since uint64) (uint64, error)) *MockStore_Backup_Call { + _c.Call.Return(run) + return _c +} + // GetBlockByHash provides a mock function for the type MockStore func (_mock *MockStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { ret := _mock.Called(ctx, hash) diff --git a/types/pb/evnode/v1/batch.pb.go b/types/pb/evnode/v1/batch.pb.go index 77b26c00a..576f1edfc 100644 --- a/types/pb/evnode/v1/batch.pb.go +++ b/types/pb/evnode/v1/batch.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/batch.proto diff --git a/types/pb/evnode/v1/config.pb.go b/types/pb/evnode/v1/config.pb.go index 1cab00fd1..23eb58d56 100644 --- a/types/pb/evnode/v1/config.pb.go +++ b/types/pb/evnode/v1/config.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/config.proto diff --git a/types/pb/evnode/v1/evnode.pb.go b/types/pb/evnode/v1/evnode.pb.go index 700b2182e..acfdeccaf 100644 --- a/types/pb/evnode/v1/evnode.pb.go +++ b/types/pb/evnode/v1/evnode.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/evnode.proto diff --git a/types/pb/evnode/v1/execution.pb.go b/types/pb/evnode/v1/execution.pb.go index 0fc0f125b..cd243512e 100644 --- a/types/pb/evnode/v1/execution.pb.go +++ b/types/pb/evnode/v1/execution.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/execution.proto diff --git a/types/pb/evnode/v1/health.pb.go b/types/pb/evnode/v1/health.pb.go index aafe1640f..1a76c46c7 100644 --- a/types/pb/evnode/v1/health.pb.go +++ b/types/pb/evnode/v1/health.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/health.proto diff --git a/types/pb/evnode/v1/p2p_rpc.pb.go b/types/pb/evnode/v1/p2p_rpc.pb.go index c1c45e17a..9325f8492 100644 --- a/types/pb/evnode/v1/p2p_rpc.pb.go +++ b/types/pb/evnode/v1/p2p_rpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/p2p_rpc.proto diff --git a/types/pb/evnode/v1/signer.pb.go b/types/pb/evnode/v1/signer.pb.go index b1f6ec321..1b72b62a9 100644 --- a/types/pb/evnode/v1/signer.pb.go +++ b/types/pb/evnode/v1/signer.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/signer.proto diff --git a/types/pb/evnode/v1/state.pb.go b/types/pb/evnode/v1/state.pb.go index 18386afe7..301666ee8 100644 --- a/types/pb/evnode/v1/state.pb.go +++ b/types/pb/evnode/v1/state.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/state.proto diff --git a/types/pb/evnode/v1/state_rpc.pb.go b/types/pb/evnode/v1/state_rpc.pb.go index 0250f1921..b90cdb0c1 100644 --- a/types/pb/evnode/v1/state_rpc.pb.go +++ b/types/pb/evnode/v1/state_rpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.10 // protoc (unknown) // source: evnode/v1/state_rpc.proto @@ -402,6 +402,221 @@ func (x *GetGenesisDaHeightResponse) GetHeight() uint64 { return 0 } +// BackupRequest defines the parameters for requesting a datastore backup. +type BackupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // target_height is the Evolve height the client wants to align with. 0 skips the check. + TargetHeight uint64 `protobuf:"varint,1,opt,name=target_height,json=targetHeight,proto3" json:"target_height,omitempty"` + // since_version allows incremental backups. 0 produces a full backup. + SinceVersion uint64 `protobuf:"varint,2,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupRequest) Reset() { + *x = BackupRequest{} + mi := &file_evnode_v1_state_rpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupRequest) ProtoMessage() {} + +func (x *BackupRequest) ProtoReflect() protoreflect.Message { + mi := &file_evnode_v1_state_rpc_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupRequest.ProtoReflect.Descriptor instead. +func (*BackupRequest) Descriptor() ([]byte, []int) { + return file_evnode_v1_state_rpc_proto_rawDescGZIP(), []int{7} +} + +func (x *BackupRequest) GetTargetHeight() uint64 { + if x != nil { + return x.TargetHeight + } + return 0 +} + +func (x *BackupRequest) GetSinceVersion() uint64 { + if x != nil { + return x.SinceVersion + } + return 0 +} + +// BackupMetadata contains progress or completion details emitted during a backup stream. +type BackupMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + CurrentHeight uint64 `protobuf:"varint,1,opt,name=current_height,json=currentHeight,proto3" json:"current_height,omitempty"` + TargetHeight uint64 `protobuf:"varint,2,opt,name=target_height,json=targetHeight,proto3" json:"target_height,omitempty"` + SinceVersion uint64 `protobuf:"varint,3,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` + LastVersion uint64 `protobuf:"varint,4,opt,name=last_version,json=lastVersion,proto3" json:"last_version,omitempty"` + Completed bool `protobuf:"varint,5,opt,name=completed,proto3" json:"completed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupMetadata) Reset() { + *x = BackupMetadata{} + mi := &file_evnode_v1_state_rpc_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupMetadata) ProtoMessage() {} + +func (x *BackupMetadata) ProtoReflect() protoreflect.Message { + mi := &file_evnode_v1_state_rpc_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupMetadata.ProtoReflect.Descriptor instead. +func (*BackupMetadata) Descriptor() ([]byte, []int) { + return file_evnode_v1_state_rpc_proto_rawDescGZIP(), []int{8} +} + +func (x *BackupMetadata) GetCurrentHeight() uint64 { + if x != nil { + return x.CurrentHeight + } + return 0 +} + +func (x *BackupMetadata) GetTargetHeight() uint64 { + if x != nil { + return x.TargetHeight + } + return 0 +} + +func (x *BackupMetadata) GetSinceVersion() uint64 { + if x != nil { + return x.SinceVersion + } + return 0 +} + +func (x *BackupMetadata) GetLastVersion() uint64 { + if x != nil { + return x.LastVersion + } + return 0 +} + +func (x *BackupMetadata) GetCompleted() bool { + if x != nil { + return x.Completed + } + return false +} + +// BackupResponse multiplexes metadata and raw backup data chunks in the stream. +type BackupResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Response: + // + // *BackupResponse_Metadata + // *BackupResponse_Chunk + Response isBackupResponse_Response `protobuf_oneof:"response"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupResponse) Reset() { + *x = BackupResponse{} + mi := &file_evnode_v1_state_rpc_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupResponse) ProtoMessage() {} + +func (x *BackupResponse) ProtoReflect() protoreflect.Message { + mi := &file_evnode_v1_state_rpc_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupResponse.ProtoReflect.Descriptor instead. +func (*BackupResponse) Descriptor() ([]byte, []int) { + return file_evnode_v1_state_rpc_proto_rawDescGZIP(), []int{9} +} + +func (x *BackupResponse) GetResponse() isBackupResponse_Response { + if x != nil { + return x.Response + } + return nil +} + +func (x *BackupResponse) GetMetadata() *BackupMetadata { + if x != nil { + if x, ok := x.Response.(*BackupResponse_Metadata); ok { + return x.Metadata + } + } + return nil +} + +func (x *BackupResponse) GetChunk() []byte { + if x != nil { + if x, ok := x.Response.(*BackupResponse_Chunk); ok { + return x.Chunk + } + } + return nil +} + +type isBackupResponse_Response interface { + isBackupResponse_Response() +} + +type BackupResponse_Metadata struct { + Metadata *BackupMetadata `protobuf:"bytes,1,opt,name=metadata,proto3,oneof"` +} + +type BackupResponse_Chunk struct { + Chunk []byte `protobuf:"bytes,2,opt,name=chunk,proto3,oneof"` +} + +func (*BackupResponse_Metadata) isBackupResponse_Response() {} + +func (*BackupResponse_Chunk) isBackupResponse_Response() {} + var File_evnode_v1_state_rpc_proto protoreflect.FileDescriptor const file_evnode_v1_state_rpc_proto_rawDesc = "" + @@ -426,12 +641,27 @@ const file_evnode_v1_state_rpc_proto_rawDesc = "" + "\x13GetMetadataResponse\x12\x14\n" + "\x05value\x18\x01 \x01(\fR\x05value\"4\n" + "\x1aGetGenesisDaHeightResponse\x12\x16\n" + - "\x06height\x18\x03 \x01(\x04R\x06height2\xbf\x02\n" + + "\x06height\x18\x03 \x01(\x04R\x06height\"Y\n" + + "\rBackupRequest\x12#\n" + + "\rtarget_height\x18\x01 \x01(\x04R\ftargetHeight\x12#\n" + + "\rsince_version\x18\x02 \x01(\x04R\fsinceVersion\"\xc2\x01\n" + + "\x0eBackupMetadata\x12%\n" + + "\x0ecurrent_height\x18\x01 \x01(\x04R\rcurrentHeight\x12#\n" + + "\rtarget_height\x18\x02 \x01(\x04R\ftargetHeight\x12#\n" + + "\rsince_version\x18\x03 \x01(\x04R\fsinceVersion\x12!\n" + + "\flast_version\x18\x04 \x01(\x04R\vlastVersion\x12\x1c\n" + + "\tcompleted\x18\x05 \x01(\bR\tcompleted\"m\n" + + "\x0eBackupResponse\x127\n" + + "\bmetadata\x18\x01 \x01(\v2\x19.evnode.v1.BackupMetadataH\x00R\bmetadata\x12\x16\n" + + "\x05chunk\x18\x02 \x01(\fH\x00R\x05chunkB\n" + + "\n" + + "\bresponse2\x82\x03\n" + "\fStoreService\x12E\n" + "\bGetBlock\x12\x1a.evnode.v1.GetBlockRequest\x1a\x1b.evnode.v1.GetBlockResponse\"\x00\x12A\n" + "\bGetState\x12\x16.google.protobuf.Empty\x1a\x1b.evnode.v1.GetStateResponse\"\x00\x12N\n" + "\vGetMetadata\x12\x1d.evnode.v1.GetMetadataRequest\x1a\x1e.evnode.v1.GetMetadataResponse\"\x00\x12U\n" + - "\x12GetGenesisDaHeight\x12\x16.google.protobuf.Empty\x1a%.evnode.v1.GetGenesisDaHeightResponse\"\x00B/Z-github.com/evstack/ev-node/types/pb/evnode/v1b\x06proto3" + "\x12GetGenesisDaHeight\x12\x16.google.protobuf.Empty\x1a%.evnode.v1.GetGenesisDaHeightResponse\"\x00\x12A\n" + + "\x06Backup\x12\x18.evnode.v1.BackupRequest\x1a\x19.evnode.v1.BackupResponse\"\x000\x01B/Z-github.com/evstack/ev-node/types/pb/evnode/v1b\x06proto3" var ( file_evnode_v1_state_rpc_proto_rawDescOnce sync.Once @@ -445,7 +675,7 @@ func file_evnode_v1_state_rpc_proto_rawDescGZIP() []byte { return file_evnode_v1_state_rpc_proto_rawDescData } -var file_evnode_v1_state_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_evnode_v1_state_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_evnode_v1_state_rpc_proto_goTypes = []any{ (*Block)(nil), // 0: evnode.v1.Block (*GetBlockRequest)(nil), // 1: evnode.v1.GetBlockRequest @@ -454,29 +684,35 @@ var file_evnode_v1_state_rpc_proto_goTypes = []any{ (*GetMetadataRequest)(nil), // 4: evnode.v1.GetMetadataRequest (*GetMetadataResponse)(nil), // 5: evnode.v1.GetMetadataResponse (*GetGenesisDaHeightResponse)(nil), // 6: evnode.v1.GetGenesisDaHeightResponse - (*SignedHeader)(nil), // 7: evnode.v1.SignedHeader - (*Data)(nil), // 8: evnode.v1.Data - (*State)(nil), // 9: evnode.v1.State - (*emptypb.Empty)(nil), // 10: google.protobuf.Empty + (*BackupRequest)(nil), // 7: evnode.v1.BackupRequest + (*BackupMetadata)(nil), // 8: evnode.v1.BackupMetadata + (*BackupResponse)(nil), // 9: evnode.v1.BackupResponse + (*SignedHeader)(nil), // 10: evnode.v1.SignedHeader + (*Data)(nil), // 11: evnode.v1.Data + (*State)(nil), // 12: evnode.v1.State + (*emptypb.Empty)(nil), // 13: google.protobuf.Empty } var file_evnode_v1_state_rpc_proto_depIdxs = []int32{ - 7, // 0: evnode.v1.Block.header:type_name -> evnode.v1.SignedHeader - 8, // 1: evnode.v1.Block.data:type_name -> evnode.v1.Data + 10, // 0: evnode.v1.Block.header:type_name -> evnode.v1.SignedHeader + 11, // 1: evnode.v1.Block.data:type_name -> evnode.v1.Data 0, // 2: evnode.v1.GetBlockResponse.block:type_name -> evnode.v1.Block - 9, // 3: evnode.v1.GetStateResponse.state:type_name -> evnode.v1.State - 1, // 4: evnode.v1.StoreService.GetBlock:input_type -> evnode.v1.GetBlockRequest - 10, // 5: evnode.v1.StoreService.GetState:input_type -> google.protobuf.Empty - 4, // 6: evnode.v1.StoreService.GetMetadata:input_type -> evnode.v1.GetMetadataRequest - 10, // 7: evnode.v1.StoreService.GetGenesisDaHeight:input_type -> google.protobuf.Empty - 2, // 8: evnode.v1.StoreService.GetBlock:output_type -> evnode.v1.GetBlockResponse - 3, // 9: evnode.v1.StoreService.GetState:output_type -> evnode.v1.GetStateResponse - 5, // 10: evnode.v1.StoreService.GetMetadata:output_type -> evnode.v1.GetMetadataResponse - 6, // 11: evnode.v1.StoreService.GetGenesisDaHeight:output_type -> evnode.v1.GetGenesisDaHeightResponse - 8, // [8:12] is the sub-list for method output_type - 4, // [4:8] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 12, // 3: evnode.v1.GetStateResponse.state:type_name -> evnode.v1.State + 8, // 4: evnode.v1.BackupResponse.metadata:type_name -> evnode.v1.BackupMetadata + 1, // 5: evnode.v1.StoreService.GetBlock:input_type -> evnode.v1.GetBlockRequest + 13, // 6: evnode.v1.StoreService.GetState:input_type -> google.protobuf.Empty + 4, // 7: evnode.v1.StoreService.GetMetadata:input_type -> evnode.v1.GetMetadataRequest + 13, // 8: evnode.v1.StoreService.GetGenesisDaHeight:input_type -> google.protobuf.Empty + 7, // 9: evnode.v1.StoreService.Backup:input_type -> evnode.v1.BackupRequest + 2, // 10: evnode.v1.StoreService.GetBlock:output_type -> evnode.v1.GetBlockResponse + 3, // 11: evnode.v1.StoreService.GetState:output_type -> evnode.v1.GetStateResponse + 5, // 12: evnode.v1.StoreService.GetMetadata:output_type -> evnode.v1.GetMetadataResponse + 6, // 13: evnode.v1.StoreService.GetGenesisDaHeight:output_type -> evnode.v1.GetGenesisDaHeightResponse + 9, // 14: evnode.v1.StoreService.Backup:output_type -> evnode.v1.BackupResponse + 10, // [10:15] is the sub-list for method output_type + 5, // [5:10] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_evnode_v1_state_rpc_proto_init() } @@ -490,13 +726,17 @@ func file_evnode_v1_state_rpc_proto_init() { (*GetBlockRequest_Height)(nil), (*GetBlockRequest_Hash)(nil), } + file_evnode_v1_state_rpc_proto_msgTypes[9].OneofWrappers = []any{ + (*BackupResponse_Metadata)(nil), + (*BackupResponse_Chunk)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_evnode_v1_state_rpc_proto_rawDesc), len(file_evnode_v1_state_rpc_proto_rawDesc)), NumEnums: 0, - NumMessages: 7, + NumMessages: 10, NumExtensions: 0, NumServices: 1, }, diff --git a/types/pb/evnode/v1/v1connect/config.connect.go b/types/pb/evnode/v1/v1connect/config.connect.go index 0677dd1ca..e7fb47c1d 100644 --- a/types/pb/evnode/v1/v1connect/config.connect.go +++ b/types/pb/evnode/v1/v1connect/config.connect.go @@ -46,7 +46,7 @@ const ( type ConfigServiceClient interface { // GetNamespace returns the namespace for this network GetNamespace(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetNamespaceResponse], error) - // GetSequencerInfo returns information about the sequencer + // GetSignerInfo returns information about the signer GetSignerInfo(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetSignerInfoResponse], error) } @@ -96,7 +96,7 @@ func (c *configServiceClient) GetSignerInfo(ctx context.Context, req *connect.Re type ConfigServiceHandler interface { // GetNamespace returns the namespace for this network GetNamespace(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetNamespaceResponse], error) - // GetSequencerInfo returns information about the sequencer + // GetSignerInfo returns information about the signer GetSignerInfo(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetSignerInfoResponse], error) } diff --git a/types/pb/evnode/v1/v1connect/state_rpc.connect.go b/types/pb/evnode/v1/v1connect/state_rpc.connect.go index 1fe049fa1..0e18b9f5c 100644 --- a/types/pb/evnode/v1/v1connect/state_rpc.connect.go +++ b/types/pb/evnode/v1/v1connect/state_rpc.connect.go @@ -44,6 +44,8 @@ const ( // StoreServiceGetGenesisDaHeightProcedure is the fully-qualified name of the StoreService's // GetGenesisDaHeight RPC. StoreServiceGetGenesisDaHeightProcedure = "/evnode.v1.StoreService/GetGenesisDaHeight" + // StoreServiceBackupProcedure is the fully-qualified name of the StoreService's Backup RPC. + StoreServiceBackupProcedure = "/evnode.v1.StoreService/Backup" ) // StoreServiceClient is a client for the evnode.v1.StoreService service. @@ -56,6 +58,8 @@ type StoreServiceClient interface { GetMetadata(context.Context, *connect.Request[v1.GetMetadataRequest]) (*connect.Response[v1.GetMetadataResponse], error) // GetGenesisDaHeight returns the DA height at which the first Evolve block was included. GetGenesisDaHeight(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetGenesisDaHeightResponse], error) + // Backup streams a Badger backup of the datastore so it can be persisted externally. + Backup(context.Context, *connect.Request[v1.BackupRequest]) (*connect.ServerStreamForClient[v1.BackupResponse], error) } // NewStoreServiceClient constructs a client for the evnode.v1.StoreService service. By default, it @@ -93,6 +97,12 @@ func NewStoreServiceClient(httpClient connect.HTTPClient, baseURL string, opts . connect.WithSchema(storeServiceMethods.ByName("GetGenesisDaHeight")), connect.WithClientOptions(opts...), ), + backup: connect.NewClient[v1.BackupRequest, v1.BackupResponse]( + httpClient, + baseURL+StoreServiceBackupProcedure, + connect.WithSchema(storeServiceMethods.ByName("Backup")), + connect.WithClientOptions(opts...), + ), } } @@ -102,6 +112,7 @@ type storeServiceClient struct { getState *connect.Client[emptypb.Empty, v1.GetStateResponse] getMetadata *connect.Client[v1.GetMetadataRequest, v1.GetMetadataResponse] getGenesisDaHeight *connect.Client[emptypb.Empty, v1.GetGenesisDaHeightResponse] + backup *connect.Client[v1.BackupRequest, v1.BackupResponse] } // GetBlock calls evnode.v1.StoreService.GetBlock. @@ -124,6 +135,11 @@ func (c *storeServiceClient) GetGenesisDaHeight(ctx context.Context, req *connec return c.getGenesisDaHeight.CallUnary(ctx, req) } +// Backup calls evnode.v1.StoreService.Backup. +func (c *storeServiceClient) Backup(ctx context.Context, req *connect.Request[v1.BackupRequest]) (*connect.ServerStreamForClient[v1.BackupResponse], error) { + return c.backup.CallServerStream(ctx, req) +} + // StoreServiceHandler is an implementation of the evnode.v1.StoreService service. type StoreServiceHandler interface { // GetBlock returns a block by height or hash @@ -134,6 +150,8 @@ type StoreServiceHandler interface { GetMetadata(context.Context, *connect.Request[v1.GetMetadataRequest]) (*connect.Response[v1.GetMetadataResponse], error) // GetGenesisDaHeight returns the DA height at which the first Evolve block was included. GetGenesisDaHeight(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetGenesisDaHeightResponse], error) + // Backup streams a Badger backup of the datastore so it can be persisted externally. + Backup(context.Context, *connect.Request[v1.BackupRequest], *connect.ServerStream[v1.BackupResponse]) error } // NewStoreServiceHandler builds an HTTP handler from the service implementation. It returns the @@ -167,6 +185,12 @@ func NewStoreServiceHandler(svc StoreServiceHandler, opts ...connect.HandlerOpti connect.WithSchema(storeServiceMethods.ByName("GetGenesisDaHeight")), connect.WithHandlerOptions(opts...), ) + storeServiceBackupHandler := connect.NewServerStreamHandler( + StoreServiceBackupProcedure, + svc.Backup, + connect.WithSchema(storeServiceMethods.ByName("Backup")), + connect.WithHandlerOptions(opts...), + ) return "/evnode.v1.StoreService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case StoreServiceGetBlockProcedure: @@ -177,6 +201,8 @@ func NewStoreServiceHandler(svc StoreServiceHandler, opts ...connect.HandlerOpti storeServiceGetMetadataHandler.ServeHTTP(w, r) case StoreServiceGetGenesisDaHeightProcedure: storeServiceGetGenesisDaHeightHandler.ServeHTTP(w, r) + case StoreServiceBackupProcedure: + storeServiceBackupHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -201,3 +227,7 @@ func (UnimplementedStoreServiceHandler) GetMetadata(context.Context, *connect.Re func (UnimplementedStoreServiceHandler) GetGenesisDaHeight(context.Context, *connect.Request[emptypb.Empty]) (*connect.Response[v1.GetGenesisDaHeightResponse], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("evnode.v1.StoreService.GetGenesisDaHeight is not implemented")) } + +func (UnimplementedStoreServiceHandler) Backup(context.Context, *connect.Request[v1.BackupRequest], *connect.ServerStream[v1.BackupResponse]) error { + return connect.NewError(connect.CodeUnimplemented, errors.New("evnode.v1.StoreService.Backup is not implemented")) +} From 05fe34a2d3eed04e4b71b6c39430230a3c041531 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 9 Oct 2025 17:05:53 +0200 Subject: [PATCH 04/14] feat: add Backup command for datastore with streaming functionality --- apps/evm/single/main.go | 3 + pkg/cmd/backup.go | 154 ++++++++++++++++++++++++++++++++++++++++ pkg/cmd/backup_test.go | 128 +++++++++++++++++++++++++++++++++ 3 files changed, 285 insertions(+) create mode 100644 pkg/cmd/backup.go create mode 100644 pkg/cmd/backup_test.go diff --git a/apps/evm/single/main.go b/apps/evm/single/main.go index a562ceff7..d02872a05 100644 --- a/apps/evm/single/main.go +++ b/apps/evm/single/main.go @@ -23,11 +23,14 @@ func main() { // Add configuration flags to NetInfoCmd so it can read RPC address config.AddFlags(rollcmd.NetInfoCmd) + backupCmd := rollcmd.NewBackupCmd() + config.AddFlags(backupCmd) rootCmd.AddCommand( cmd.InitCmd(), cmd.RunCmd, cmd.NewRollbackCmd(), + backupCmd, rollcmd.VersionCmd, rollcmd.NetInfoCmd, rollcmd.StoreUnsafeCleanCmd, diff --git a/pkg/cmd/backup.go b/pkg/cmd/backup.go new file mode 100644 index 000000000..0cbf65c0d --- /dev/null +++ b/pkg/cmd/backup.go @@ -0,0 +1,154 @@ +package cmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + clientrpc "github.com/evstack/ev-node/pkg/rpc/client" + pb "github.com/evstack/ev-node/types/pb/evnode/v1" +) + +// NewBackupCmd creates a cobra command that streams a datastore backup via the RPC client. +func NewBackupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "backup", + Short: "Stream a datastore backup to a local file via RPC", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + nodeConfig, err := ParseConfig(cmd) + if err != nil { + return fmt.Errorf("error parsing config: %w", err) + } + + rpcAddress := strings.TrimSpace(nodeConfig.RPC.Address) + if rpcAddress == "" { + return fmt.Errorf("RPC address not found in node configuration") + } + + baseURL := rpcAddress + if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") { + baseURL = fmt.Sprintf("http://%s", baseURL) + } + + outputPath, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + + if outputPath == "" { + timestamp := time.Now().UTC().Format("20060102-150405") + outputPath = fmt.Sprintf("evnode-backup-%s.badger", timestamp) + } + + outputPath, err = filepath.Abs(outputPath) + if err != nil { + return fmt.Errorf("failed to resolve output path: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + + force, err := cmd.Flags().GetBool("force") + if err != nil { + return err + } + + if !force { + if _, statErr := os.Stat(outputPath); statErr == nil { + return fmt.Errorf("output file %s already exists (use --force to overwrite)", outputPath) + } else if !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("failed to inspect output file: %w", statErr) + } + } + + file, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + defer file.Close() + + writer := bufio.NewWriterSize(file, 1<<20) // 1 MiB buffer for fewer syscalls. + bytesCount := &countingWriter{} + streamWriter := io.MultiWriter(writer, bytesCount) + + targetHeight, err := cmd.Flags().GetUint64("target-height") + if err != nil { + return err + } + + sinceVersion, err := cmd.Flags().GetUint64("since-version") + if err != nil { + return err + } + + ctx := cmd.Context() + if ctx == nil { + ctx = context.Background() + } + + client := clientrpc.NewClient(baseURL) + + metadata, backupErr := client.Backup(ctx, &pb.BackupRequest{ + TargetHeight: targetHeight, + SinceVersion: sinceVersion, + }, streamWriter) + if backupErr != nil { + // Remove the partial file on failure to avoid keeping corrupt snapshots. + _ = writer.Flush() + _ = file.Close() + _ = os.Remove(outputPath) + return fmt.Errorf("backup failed: %w", backupErr) + } + + if err := writer.Flush(); err != nil { + _ = file.Close() + _ = os.Remove(outputPath) + return fmt.Errorf("failed to flush backup data: %w", err) + } + + if !metadata.GetCompleted() { + _ = file.Close() + _ = os.Remove(outputPath) + return fmt.Errorf("backup stream ended without completion metadata") + } + + cmd.Printf("Backup saved to %s (%d bytes)\n", outputPath, bytesCount.Bytes()) + cmd.Printf("Current height: %d\n", metadata.GetCurrentHeight()) + cmd.Printf("Target height: %d\n", metadata.GetTargetHeight()) + cmd.Printf("Since version: %d\n", metadata.GetSinceVersion()) + cmd.Printf("Last version: %d\n", metadata.GetLastVersion()) + + return nil + }, + } + + cmd.Flags().String("output", "", "Path to the backup file (defaults to ./evnode-backup-.badger)") + cmd.Flags().Uint64("target-height", 0, "Target chain height to align the backup with (0 disables the check)") + cmd.Flags().Uint64("since-version", 0, "Generate an incremental backup starting from the provided version") + cmd.Flags().Bool("force", false, "Overwrite the output file if it already exists") + + return cmd +} + +type countingWriter struct { + total int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + c.total += int64(len(p)) + return len(p), nil +} + +func (c *countingWriter) Bytes() int64 { + return c.total +} diff --git a/pkg/cmd/backup_test.go b/pkg/cmd/backup_test.go new file mode 100644 index 000000000..d74bf7efd --- /dev/null +++ b/pkg/cmd/backup_test.go @@ -0,0 +1,128 @@ +package cmd + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/rs/zerolog" + "github.com/spf13/cobra" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + + "github.com/evstack/ev-node/pkg/config" + "github.com/evstack/ev-node/pkg/rpc/server" + "github.com/evstack/ev-node/test/mocks" + "github.com/evstack/ev-node/types/pb/evnode/v1/v1connect" +) + +func TestBackupCmd_Success(t *testing.T) { + t.Parallel() + + mockStore := mocks.NewMockStore(t) + + mockStore.On("Height", mock.Anything).Return(uint64(15), nil) + mockStore.On("Backup", mock.Anything, mock.Anything, uint64(9)).Run(func(args mock.Arguments) { + writer := args.Get(1).(io.Writer) + _, _ = writer.Write([]byte("chunk-1")) + _, _ = writer.Write([]byte("chunk-2")) + }).Return(uint64(21), nil) + + logger := zerolog.Nop() + storeServer := server.NewStoreServer(mockStore, logger) + mux := http.NewServeMux() + storePath, storeHandler := v1connect.NewStoreServiceHandler(storeServer) + mux.Handle(storePath, storeHandler) + + httpServer := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{})) + defer httpServer.Close() + + tempDir, err := os.MkdirTemp("", "evnode-backup-*") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + backupCmd := NewBackupCmd() + config.AddFlags(backupCmd) + + rootCmd := &cobra.Command{Use: "root"} + config.AddGlobalFlags(rootCmd, "test") + rootCmd.AddCommand(backupCmd) + + outPath := filepath.Join(tempDir, "snapshot.badger") + rpcAddr := strings.TrimPrefix(httpServer.URL, "http://") + + output, err := executeCommandC( + rootCmd, + "backup", + "--home="+tempDir, + "--evnode.rpc.address="+rpcAddr, + "--output", outPath, + "--target-height", "12", + "--since-version", "9", + ) + + require.NoError(t, err, "command failed: %s", output) + + data, readErr := os.ReadFile(outPath) + require.NoError(t, readErr) + require.Equal(t, "chunk-1chunk-2", string(data)) + + require.Contains(t, output, "Backup saved to") + require.Contains(t, output, "Current height: 15") + require.Contains(t, output, "Target height: 12") + require.Contains(t, output, "Since version: 9") + require.Contains(t, output, "Last version: 21") + + mockStore.AssertExpectations(t) +} + +func TestBackupCmd_ExistingFileWithoutForce(t *testing.T) { + t.Parallel() + + mockStore := mocks.NewMockStore(t) + logger := zerolog.Nop() + storeServer := server.NewStoreServer(mockStore, logger) + mux := http.NewServeMux() + storePath, storeHandler := v1connect.NewStoreServiceHandler(storeServer) + mux.Handle(storePath, storeHandler) + + httpServer := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{})) + defer httpServer.Close() + + tempDir, err := os.MkdirTemp("", "evnode-backup-existing-*") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + outPath := filepath.Join(tempDir, "snapshot.badger") + require.NoError(t, os.WriteFile(outPath, []byte("existing"), 0o600)) + + backupCmd := NewBackupCmd() + config.AddFlags(backupCmd) + + rootCmd := &cobra.Command{Use: "root"} + config.AddGlobalFlags(rootCmd, "test") + rootCmd.AddCommand(backupCmd) + + rpcAddr := strings.TrimPrefix(httpServer.URL, "http://") + + output, err := executeCommandC( + rootCmd, + "backup", + "--home="+tempDir, + "--evnode.rpc.address="+rpcAddr, + "--output", outPath, + ) + + require.Error(t, err) + require.Contains(t, output, "already exists (use --force to overwrite)") +} From bf29d4e50860137b7b78ba0d7e963f4b071c39d6 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 9 Oct 2025 18:29:24 +0200 Subject: [PATCH 05/14] feat: enhance Backup functionality to support datastore unwrapping and improve error handling --- pkg/store/backup.go | 62 ++++++++++++++++++++++++----------- scripts/reth-backup/README.md | 37 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/pkg/store/backup.go b/pkg/store/backup.go index 6e7a9fbc9..26600435e 100644 --- a/pkg/store/backup.go +++ b/pkg/store/backup.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + ds "github.com/ipfs/go-datastore" badger4 "github.com/ipfs/go-ds-badger4" ) @@ -16,28 +17,51 @@ func (s *DefaultStore) Backup(ctx context.Context, writer io.Writer, since uint6 return 0, err } - // Try to leverage a native backup implementation if the underlying datastore exposes one. - type backupable interface { - Backup(io.Writer, uint64) (uint64, error) - } - if dsBackup, ok := s.db.(backupable); ok { - version, err := dsBackup.Backup(writer, since) - if err != nil { - return 0, fmt.Errorf("datastore backup failed: %w", err) - } - return version, nil - } - - // Default Badger datastore used across ev-node. - badgerDatastore, ok := s.db.(*badger4.Datastore) + visited := make(map[ds.Datastore]struct{}) + current, ok := any(s.db).(ds.Datastore) if !ok { return 0, fmt.Errorf("backup is not supported by the configured datastore") } - // `badger.DB.Backup` internally orchestrates a consistent snapshot without pausing writes. - version, err := badgerDatastore.DB.Backup(writer, since) - if err != nil { - return 0, fmt.Errorf("badger backup failed: %w", err) + for { + // Try to leverage a native backup implementation if the underlying datastore exposes one. + type backupable interface { + Backup(io.Writer, uint64) (uint64, error) + } + if dsBackup, ok := current.(backupable); ok { + version, err := dsBackup.Backup(writer, since) + if err != nil { + return 0, fmt.Errorf("datastore backup failed: %w", err) + } + return version, nil + } + + // Default Badger datastore used across ev-node. + if badgerDatastore, ok := current.(*badger4.Datastore); ok { + // `badger.DB.Backup` internally orchestrates a consistent snapshot without pausing writes. + version, err := badgerDatastore.DB.Backup(writer, since) + if err != nil { + return 0, fmt.Errorf("badger backup failed: %w", err) + } + return version, nil + } + + // Attempt to unwrap shimmed datastores (e.g., prefix or mutex wrappers) to reach the backing store. + if _, seen := visited[current]; seen { + break + } + visited[current] = struct{}{} + + shim, ok := current.(ds.Shim) + if !ok { + break + } + children := shim.Children() + if len(children) == 0 { + break + } + current = children[0] } - return version, nil + + return 0, fmt.Errorf("backup is not supported by the configured datastore") } diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md index 3fb06fb56..497c5ee16 100644 --- a/scripts/reth-backup/README.md +++ b/scripts/reth-backup/README.md @@ -38,3 +38,40 @@ Additional flags: The script outputs the height at the end so you can coordinate other backups with the same block number. + +## End-to-end workflow with `apps/evm/single` + +The `evm/single` docker-compose setup expects the backup image to reuse the +`ghcr.io/evstack/ev-reth:latest` tag. To capture both the MDBX and ev-node +Badger backups end-to-end: + +1. Build the helper image so `ev-reth` has the MDBX tooling preinstalled. + ```bash + docker build -t ghcr.io/evstack/ev-reth:latest scripts/reth-backup + ``` +2. Build the `ev-node-evm-single` image so it includes the latest CLI backup + command (optional if you already pushed the binary and re-tagged it). + ```bash + docker build -t ghcr.io/evstack/ev-node-evm-single:main -f apps/evm/single/Dockerfile . + ``` +3. Start the stack. + ```bash + (cd apps/evm/single && docker compose up -d) + ``` +4. Run the MDBX snapshot script (adjust the destination as needed). + ```bash + ./scripts/reth-backup/backup.sh backups/full-run/reth + ``` +5. Stream a Badger backup from the ev-node into the container filesystem and + copy it to the host. + ```bash + docker compose exec ev-node-evm-single \ + evm-single backup --output /tmp/evnode-backup.badger --force + docker cp ev-node-evm-single:/tmp/evnode-backup.badger backups/full-run/ev-node/ + ``` +6. When finished, tear the stack down. + ```bash + (cd apps/evm/single && docker compose down) + ``` + +Both backups can then be found under `backups/full-run/`. From c76b16d6e255e9e66572ea46d82543fa542ed7d6 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Mon, 13 Oct 2025 15:42:03 +0200 Subject: [PATCH 06/14] update changelog --- scripts/reth-backup/README.md | 138 ++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 5 deletions(-) diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md index 497c5ee16..055e17880 100644 --- a/scripts/reth-backup/README.md +++ b/scripts/reth-backup/README.md @@ -62,16 +62,144 @@ Badger backups end-to-end: ```bash ./scripts/reth-backup/backup.sh backups/full-run/reth ``` -5. Stream a Badger backup from the ev-node into the container filesystem and - copy it to the host. + The script prints the generated tag (for example `20251013-104816`) and the + captured height (stored under + `backups/full-run/reth//height.txt`). +5. Align the ev-node datastore to that height and take the Badger backup: ```bash - docker compose exec ev-node-evm-single \ - evm-single backup --output /tmp/evnode-backup.badger --force - docker cp ev-node-evm-single:/tmp/evnode-backup.badger backups/full-run/ev-node/ + HEIGHT=$(cat backups/full-run/reth//height.txt) + BACKUP_ROOT="$(pwd)/backups/full-run" + + # Stop the managed container so it cannot advance. + (cd apps/evm/single && docker compose stop ev-node-evm-single) + + # Roll the datastore back to the captured height. The --sync-node and + # --skip-p2p-stores flags avoid DA-finality and header-store checks. + (cd apps/evm/single && docker compose run --rm \ + ev-node-evm-single rollback \ + --height "${HEIGHT}" \ + --home /root/.evm-single \ + --sync-node \ + --skip-p2p-stores) + + # Stream the Badger backup without producing new blocks. + cat <<'EOF' > /tmp/evnode_backup.sh +set -euo pipefail +OUT="$1" +TARGET="$2" +START_CMD="evm-single start \ + --home=/root/.evm-single \ + --evm.jwt-secret $EVM_JWT_SECRET \ + --evm.genesis-hash $EVM_GENESIS_HASH \ + --evm.engine-url $EVM_ENGINE_URL \ + --evm.eth-url $EVM_ETH_URL \ + --rollkit.node.block_time 1h \ + --rollkit.node.aggregator=true \ + --rollkit.signer.passphrase $EVM_SIGNER_PASSPHRASE \ + --rollkit.da.address $DA_ADDRESS \ + --evnode.clear_cache" +rm -f "$OUT" +sh -c "$START_CMD" & +PID=$! +trap "kill $PID 2>/dev/null || true; wait $PID 2>/dev/null || true" EXIT +for i in $(seq 1 30); do + sleep 2 + if evm-single backup --output "$OUT" --force --target-height "$TARGET" >/tmp/backup.log 2>&1; then + cat /tmp/backup.log + exit 0 + fi + cat /tmp/backup.log +done +echo "backup did not succeed within timeout" >&2 +exit 1 +EOF + chmod +x /tmp/evnode_backup.sh + + (cd apps/evm/single && docker compose run --rm \ + --entrypoint sh \ + -v /tmp/evnode_backup.sh:/tmp/evnode_backup.sh \ + -v "${BACKUP_ROOT}/ev-node:/host-backup" \ + ev-node-evm-single \ + -c "/tmp/evnode_backup.sh /host-backup/evnode-backup-aligned.badger ${HEIGHT}") + + rm /tmp/evnode_backup.sh + # Bring the managed container back with its usual supervisor. + (cd apps/evm/single && docker compose start ev-node-evm-single) ``` + The CLI will report the streamed metadata, and the backup lands at + `backups/full-run/ev-node/evnode-backup-aligned.badger`. 6. When finished, tear the stack down. ```bash (cd apps/evm/single && docker compose down) ``` Both backups can then be found under `backups/full-run/`. + +## Restoring and validating the backups + +To verify that both snapshots can be replayed, you can shut everything down, mutate the live data, and then restore from the artifacts collected above. + +1. **Let the stack advance after the backup (optional but recommended).** Keep `docker compose` running for a few more blocks or submit a dev transaction so the live height diverges from the one recorded in `backups/full-run/reth/height.txt`. +2. **Stop the services.** + ```bash + (cd apps/evm/single && docker compose down) + ``` +3. **Recreate the containers without starting them.** This gives you stopped containers that already own the right named volumes. + ```bash + (cd apps/evm/single && docker compose up --no-start) + ``` +4. **Restore the `ev-reth` MDBX volume from the snapshot.** Run the following from the repository root (adjust `BACKUP_ROOT` if you saved the files elsewhere): + ```bash + BACKUP_ROOT="$(pwd)/backups/full-run" + docker run --rm \ + --volumes-from ev-reth \ + -v "${BACKUP_ROOT}/reth:/backup:ro" \ + alpine:3.18 \ + sh -ec 'rm -rf /home/reth/eth-home/db /home/reth/eth-home/static_files && \ + mkdir -p /home/reth/eth-home/db /home/reth/eth-home/static_files && \ + cp /backup/db/mdbx.dat /home/reth/eth-home/db/mdbx.dat && \ + cp /backup/db/mdbx.lck /home/reth/eth-home/db/mdbx.lck && \ + cp -a /backup/static_files/. /home/reth/eth-home/static_files/ || true' + ``` + > `docker run --volumes-from ev-reth` reuses the stopped container's volumes; adjust `alpine:3.18` if you prefer another image that provides `cp`. +5. **Restore the `ev-node` Badger datastore.** + ```bash + TEMP_RESTORE="$(mktemp -d backups/full-run/ev-node/restore-XXXXXX)" + badger restore --dir "${TEMP_RESTORE}" --files "${BACKUP_ROOT}/ev-node/evnode-backup-aligned.badger" + docker run --rm \ + --volumes-from ev-node-evm-single \ + -v "${TEMP_RESTORE}:/restore:ro" \ + alpine:3.18 \ + sh -ec 'rm -rf /root/.evm-single/data && mkdir -p /root/.evm-single/data/evm-single && \ + cp -a /restore/. /root/.evm-single/data/evm-single/' + rm -rf "${TEMP_RESTORE}" + ``` + > Install the Badger CLI once via `go install github.com/dgraph-io/badger/v4/cmd/badger@latest` if it is not already on your `$PATH`. +6. **Start the services back up.** + ```bash + (cd apps/evm/single && docker compose up -d ev-reth local-da) + (cd apps/evm/single && docker compose up -d ev-node-evm-single) + ``` + If you prefer to launch the sequencer manually first, you can run: + ```bash + (cd apps/evm/single && docker compose run --rm \ + ev-node-evm-single start \ + --home /root/.evm-single \ + --evm.jwt-secret "$EVM_JWT_SECRET" \ + --evm.genesis-hash "$EVM_GENESIS_HASH" \ + --evm.engine-url "$EVM_ENGINE_URL" \ + --evm.eth-url "$EVM_ETH_URL" \ + --rollkit.node.block_time 1s \ + --rollkit.node.aggregator=true \ + --rollkit.signer.passphrase "$EVM_SIGNER_PASSPHRASE" \ + --rollkit.da.address "$DA_ADDRESS" \ + --evnode.clear_cache) + ``` + and once it exits cleanly, start the managed container with `docker compose up -d ev-node-evm-single`. +7. **Confirm the node resumes from the backed-up height.** Compare the logged chain height with `backups/full-run/reth/height.txt` and run: + ```bash + (cd apps/evm/single && docker compose exec ev-node-evm-single evm-single net-info --home /root/.evm-single) + ``` + The reported height should match the snapshot before new blocks are produced. + +With this round-trip check you can be confident that the snapshot pair (MDBX + Badger) fully recreates the state captured during the backup. From 078015f8815d52fd90b52655fbe6ee898c56e3dc Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Mon, 13 Oct 2025 18:10:59 +0200 Subject: [PATCH 07/14] feat: implement backup library for local and Docker execution modes --- scripts/reth-backup/README.md | 80 +++++++++++++-- scripts/reth-backup/backup-lib.sh | 165 ++++++++++++++++++++++++++++++ scripts/reth-backup/backup.sh | 104 ++++++++++++++----- 3 files changed, 317 insertions(+), 32 deletions(-) create mode 100644 scripts/reth-backup/backup-lib.sh diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md index 055e17880..643fb0d1c 100644 --- a/scripts/reth-backup/README.md +++ b/scripts/reth-backup/README.md @@ -3,26 +3,59 @@ Script to snapshot the `ev-reth` MDBX database while the node keeps running and record the block height contained in the snapshot. +The script supports two execution modes: + +- **local**: Backup a reth instance running directly on the host machine +- **docker**: Backup a reth instance running in a Docker container + ## Prerequisites -- Docker access to the container running `ev-reth` (defaults to the service name - `ev-reth` from `docker-compose`). -- The `mdbx_copy` binary available inside that container. If it is not provided - by the image, compile it once inside the container (see [libmdbx +### Common requirements + +- The `mdbx_copy` binary available in the target environment (see [libmdbx documentation](https://libmdbx.dqdkfa.ru/)). - `jq` installed on the host to parse the JSON output. +### Docker mode + +- Docker access to the container running `ev-reth` (defaults to the service name + `ev-reth` from `docker-compose`). + +### Local mode + +- Direct filesystem access to the reth datadir. +- Sufficient permissions to read the database files. + ## Usage +### Local mode + +When reth is running directly on your machine: + +```bash +./scripts/reth-backup/backup.sh \ + --mode local \ + --datadir /var/lib/reth \ + --mdbx-copy /usr/local/bin/mdbx_copy \ + /path/to/backups +``` + +### Docker mode + +When reth is running in a Docker container: + ```bash ./scripts/reth-backup/backup.sh \ + --mode docker \ --container ev-reth \ --datadir /home/reth/eth-home \ --mdbx-copy /tmp/libmdbx/build/mdbx_copy \ /path/to/backups ``` -This creates a timestamped folder under `/path/to/backups` with: +### Output structure + +Both modes create a timestamped folder under `/path/to/backups` with: - `db/mdbx.dat` – consistent MDBX snapshot. - `db/mdbx.lck` – placeholder lock file (empty). @@ -33,39 +66,64 @@ This creates a timestamped folder under `/path/to/backups` with: Additional flags: - `--tag LABEL` to override the timestamped folder name. -- `--keep-remote` to leave the temporary snapshot inside the container (useful - for debugging). +- `--keep-remote` to leave the temporary snapshot in the target environment + (useful for debugging). The script outputs the height at the end so you can coordinate other backups with the same block number. -## End-to-end workflow with `apps/evm/single` +## Architecture + +The backup script is split into two components: + +- **`backup-lib.sh`**: Abstract execution layer providing a common interface for + different execution modes (local, docker). This library defines functions like + `exec_remote`, `copy_from_remote`, `copy_to_remote`, and `cleanup_remote` + that are implemented differently for each backend. +- **`backup.sh`**: Main script that uses the library and orchestrates the backup + workflow. It's mode-agnostic and works with any backend that implements the + required interface. + +This separation allows easy extension to support additional execution +environments (SSH, Kubernetes, etc.) without modifying the core backup logic. + +## End-to-end workflow with `apps/evm/single` (Docker mode) The `evm/single` docker-compose setup expects the backup image to reuse the `ghcr.io/evstack/ev-reth:latest` tag. To capture both the MDBX and ev-node Badger backups end-to-end: 1. Build the helper image so `ev-reth` has the MDBX tooling preinstalled. + ```bash docker build -t ghcr.io/evstack/ev-reth:latest scripts/reth-backup ``` + 2. Build the `ev-node-evm-single` image so it includes the latest CLI backup command (optional if you already pushed the binary and re-tagged it). + ```bash docker build -t ghcr.io/evstack/ev-node-evm-single:main -f apps/evm/single/Dockerfile . ``` + 3. Start the stack. + ```bash (cd apps/evm/single && docker compose up -d) ``` + 4. Run the MDBX snapshot script (adjust the destination as needed). + ```bash - ./scripts/reth-backup/backup.sh backups/full-run/reth + ./scripts/reth-backup/backup.sh --mode docker backups/full-run/reth ``` + The script prints the generated tag (for example `20251013-104816`) and the captured height (stored under `backups/full-run/reth//height.txt`). + 5. Align the ev-node datastore to that height and take the Badger backup: + ```bash HEIGHT=$(cat backups/full-run/reth//height.txt) BACKUP_ROOT="$(pwd)/backups/full-run" @@ -123,12 +181,16 @@ EOF -c "/tmp/evnode_backup.sh /host-backup/evnode-backup-aligned.badger ${HEIGHT}") rm /tmp/evnode_backup.sh + # Bring the managed container back with its usual supervisor. (cd apps/evm/single && docker compose start ev-node-evm-single) ``` + The CLI will report the streamed metadata, and the backup lands at `backups/full-run/ev-node/evnode-backup-aligned.badger`. + 6. When finished, tear the stack down. + ```bash (cd apps/evm/single && docker compose down) ``` diff --git a/scripts/reth-backup/backup-lib.sh b/scripts/reth-backup/backup-lib.sh new file mode 100644 index 000000000..c445f1459 --- /dev/null +++ b/scripts/reth-backup/backup-lib.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +# backup-lib.sh - Abstract execution layer for reth backup operations +# Provides a common interface for local and Docker-based executions. + +# Backend interface that must be implemented: +# - exec_remote Execute a command in the target environment +# - copy_from_remote Copy a file/directory from target to local +# - copy_to_remote Copy a file/directory from local to target +# - cleanup_remote Remove a path in the target environment + +# ============================================================================ +# LOCAL BACKEND +# ============================================================================ + +local_exec_remote() { + bash -c "$1" +} + +local_copy_from_remote() { + local src="$1" + local dst="$2" + cp -a "$src" "$dst" +} + +local_copy_to_remote() { + local src="$1" + local dst="$2" + cp -a "$src" "$dst" +} + +local_cleanup_remote() { + local path="$1" + rm -rf "$path" +} + +local_check_available() { + # Always available + return 0 +} + +# ============================================================================ +# DOCKER BACKEND +# ============================================================================ + +docker_exec_remote() { + local container="$BACKEND_CONTAINER" + docker exec "$container" bash -lc "$1" +} + +docker_copy_from_remote() { + local container="$BACKEND_CONTAINER" + local src="$1" + local dst="$2" + docker cp "${container}:${src}" "$dst" +} + +docker_copy_to_remote() { + local container="$BACKEND_CONTAINER" + local src="$1" + local dst="$2" + docker cp "$src" "${container}:${dst}" +} + +docker_cleanup_remote() { + local container="$BACKEND_CONTAINER" + local path="$1" + docker exec "$container" rm -rf "$path" +} + +docker_check_available() { + if ! command -v docker >/dev/null 2>&1; then + echo "error: docker command not found" >&2 + return 1 + fi + + local container="$BACKEND_CONTAINER" + if [[ -z "$container" ]]; then + echo "error: container name is required for docker mode" >&2 + return 1 + fi + + if ! docker ps --format '{{.Names}}' | grep -q "^${container}$"; then + echo "error: container '$container' is not running" >&2 + return 1 + fi + + return 0 +} + +# ============================================================================ +# BACKEND INITIALIZATION +# ============================================================================ + +# Set the backend mode and initialize function pointers +init_backend() { + local mode="$1" + + case "$mode" in + local) + exec_remote=local_exec_remote + copy_from_remote=local_copy_from_remote + copy_to_remote=local_copy_to_remote + cleanup_remote=local_cleanup_remote + check_backend_available=local_check_available + ;; + docker) + exec_remote=docker_exec_remote + copy_from_remote=docker_copy_from_remote + copy_to_remote=docker_copy_to_remote + cleanup_remote=docker_cleanup_remote + check_backend_available=docker_check_available + ;; + *) + echo "error: unknown backend mode '$mode'" >&2 + echo "supported modes: local, docker" >&2 + return 1 + ;; + esac + + BACKEND_MODE="$mode" + return 0 +} + +# ============================================================================ +# HIGH-LEVEL BACKUP OPERATIONS +# ============================================================================ + +# Verify that a command is available in the target environment +verify_remote_command() { + local cmd="$1" + if ! $exec_remote "command -v '$cmd' >/dev/null 2>&1 || [ -x '$cmd' ]"; then + echo "error: command '$cmd' not found in target environment" >&2 + return 1 + fi + return 0 +} + +# Create a directory in the target environment +create_remote_dir() { + local path="$1" + $exec_remote "mkdir -p '$path'" +} + +# Check if a path exists in the target environment +remote_path_exists() { + local path="$1" + $exec_remote "test -e '$path'" +} + +# Run mdbx_copy in the target environment +run_mdbx_copy() { + local mdbx_copy="$1" + local source_db="$2" + local dest_file="$3" + + echo "Running mdbx_copy..." + $exec_remote "'$mdbx_copy' -c '$source_db' '$dest_file'" +} + +# Query ev-reth for stage checkpoints +query_stage_checkpoints() { + local datadir="$1" + $exec_remote "ev-reth db --datadir '$datadir' list StageCheckpoints --len 20 --json" | sed -n '/^\[/,$p' +} diff --git a/scripts/reth-backup/backup.sh b/scripts/reth-backup/backup.sh index 7983ba4bd..c254fecb1 100755 --- a/scripts/reth-backup/backup.sh +++ b/scripts/reth-backup/backup.sh @@ -2,6 +2,10 @@ set -euo pipefail +# Load the backend library +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/backup-lib.sh" + usage() { cat <<'EOF' Usage: backup.sh [OPTIONS] @@ -10,19 +14,27 @@ Create a consistent backup of the ev-reth database using mdbx_copy and record the block height captured in the snapshot. Options: + --mode MODE Execution mode: 'local' or 'docker' (default: docker) --container NAME Docker container name running ev-reth (default: ev-reth) - --datadir PATH Path to the reth datadir inside the container - (default: /home/reth/eth-home) - --mdbx-copy CMD Path to the mdbx_copy binary inside the container + Only used in docker mode. + --datadir PATH Path to the reth datadir in the target environment + (default docker: /home/reth/eth-home) + (default local: /var/lib/reth) + --mdbx-copy CMD Path to the mdbx_copy binary in the target environment (default: mdbx_copy; override if you compiled it elsewhere) --tag LABEL Custom label for the backup directory (default: timestamp) - --keep-remote Leave the temporary snapshot inside the container + --keep-remote Leave the temporary snapshot in the target environment -h, --help Show this help message +Modes: + local Run backup on the local machine (reth running locally) + docker Run backup on a Docker container (default) + Requirements: - - Docker access to the container running ev-reth. - - mdbx_copy available inside that container (compile it once if necessary). + - mdbx_copy available in the target environment (compile it once if necessary). - jq installed on the host (used to parse StageCheckpoints JSON). + - For docker mode: Docker access to the container running ev-reth. + - For local mode: Direct filesystem access to reth datadir. The destination directory will receive: //db/mdbx.dat MDBX snapshot @@ -30,6 +42,13 @@ The destination directory will receive: //static_files/... Static files copied from the node //stage_checkpoints.json //height.txt Height extracted from StageCheckpoints + +Examples: + # Backup from local reth instance + ./backup.sh --mode local --datadir /var/lib/reth /path/to/backups + + # Backup from Docker container + ./backup.sh --mode docker --container ev-reth /path/to/backups EOF } @@ -41,14 +60,19 @@ require_cmd() { } DEST="" +MODE="docker" CONTAINER="ev-reth" -DATADIR="/home/reth/eth-home" +DATADIR="" MDBX_COPY="mdbx_copy" BACKUP_TAG="" KEEP_REMOTE=0 while [[ $# -gt 0 ]]; do case "$1" in + --mode) + MODE="$2" + shift 2 + ;; --container) CONTAINER="$2" shift 2 @@ -101,7 +125,39 @@ if [[ -z "$DEST" ]]; then exit 1 fi -require_cmd docker +# Validate and set defaults based on mode +case "$MODE" in + local) + if [[ -z "$DATADIR" ]]; then + DATADIR="/var/lib/reth" + fi + ;; + docker) + if [[ -z "$DATADIR" ]]; then + DATADIR="/home/reth/eth-home" + fi + ;; + *) + echo "error: invalid mode '$MODE'. Use 'local' or 'docker'." >&2 + exit 1 + ;; +esac + +# Initialize the backend +if ! init_backend "$MODE"; then + exit 1 +fi + +# Set container for docker mode +if [[ "$MODE" == "docker" ]]; then + BACKEND_CONTAINER="$CONTAINER" +fi + +# Check backend availability +if ! $check_backend_available; then + exit 1 +fi + require_cmd jq if [[ -z "$BACKUP_TAG" ]]; then @@ -111,26 +167,27 @@ fi REMOTE_TMP="/tmp/reth-backup-${BACKUP_TAG}" HOST_DEST="$(mkdir -p "$DEST" && cd "$DEST" && pwd)/${BACKUP_TAG}" +echo "Mode: $MODE" echo "Creating backup tag '$BACKUP_TAG' into ${HOST_DEST}" -# Prepare temporary workspace inside the container. -docker exec "$CONTAINER" bash -c "rm -rf '$REMOTE_TMP' && mkdir -p '$REMOTE_TMP/db' '$REMOTE_TMP/static_files'" +# Prepare temporary workspace in target environment +echo "Preparing temporary workspace..." +$exec_remote "rm -rf '$REMOTE_TMP' && mkdir -p '$REMOTE_TMP/db' '$REMOTE_TMP/static_files'" -# Verify mdbx_copy availability. -if ! docker exec "$CONTAINER" bash -lc "command -v '$MDBX_COPY' >/dev/null 2>&1 || [ -x '$MDBX_COPY' ]"; then - echo "error: unable to find executable '$MDBX_COPY' inside container '$CONTAINER'" >&2 +# Verify mdbx_copy availability +if ! verify_remote_command "$MDBX_COPY"; then exit 1 fi -echo "Running mdbx_copy inside container..." -docker exec "$CONTAINER" bash -lc "'$MDBX_COPY' -c '${DATADIR}/db' '$REMOTE_TMP/db/mdbx.dat'" -docker exec "$CONTAINER" bash -lc "touch '$REMOTE_TMP/db/mdbx.lck'" +echo "Running mdbx_copy in target environment..." +run_mdbx_copy "$MDBX_COPY" "${DATADIR}/db" "$REMOTE_TMP/db/mdbx.dat" +$exec_remote "touch '$REMOTE_TMP/db/mdbx.lck'" echo "Copying static_files..." -docker exec "$CONTAINER" bash -lc "if [ -d '${DATADIR}/static_files' ]; then cp -a '${DATADIR}/static_files/.' '$REMOTE_TMP/static_files/' 2>/dev/null || true; fi" +$exec_remote "if [ -d '${DATADIR}/static_files' ]; then cp -a '${DATADIR}/static_files/.' '$REMOTE_TMP/static_files/' 2>/dev/null || true; fi" echo "Querying StageCheckpoints height..." -STAGE_JSON=$(docker exec "$CONTAINER" ev-reth db --datadir "$REMOTE_TMP" list StageCheckpoints --len 20 --json | sed -n '/^\[/,$p') +STAGE_JSON=$(query_stage_checkpoints "$REMOTE_TMP") HEIGHT=$(echo "$STAGE_JSON" | jq -r '.[] | select(.[0]=="Finish") | .[1].block_number' | tr -d '\r\n') if [[ -z "$HEIGHT" || "$HEIGHT" == "null" ]]; then @@ -139,12 +196,12 @@ fi echo "Copying snapshot to host..." mkdir -p "$HOST_DEST/db" -docker cp "${CONTAINER}:${REMOTE_TMP}/db/mdbx.dat" "$HOST_DEST/db/mdbx.dat" -docker cp "${CONTAINER}:${REMOTE_TMP}/db/mdbx.lck" "$HOST_DEST/db/mdbx.lck" +$copy_from_remote "${REMOTE_TMP}/db/mdbx.dat" "$HOST_DEST/db/mdbx.dat" +$copy_from_remote "${REMOTE_TMP}/db/mdbx.lck" "$HOST_DEST/db/mdbx.lck" -if docker exec "$CONTAINER" test -d "${REMOTE_TMP}/static_files"; then +if remote_path_exists "${REMOTE_TMP}/static_files"; then mkdir -p "$HOST_DEST/static_files" - docker cp "${CONTAINER}:${REMOTE_TMP}/static_files/." "$HOST_DEST/static_files/" + $copy_from_remote "${REMOTE_TMP}/static_files/." "$HOST_DEST/static_files/" || true fi echo "$STAGE_JSON" > "$HOST_DEST/stage_checkpoints.json" @@ -156,7 +213,8 @@ else fi if [[ "$KEEP_REMOTE" -ne 1 ]]; then - docker exec "$CONTAINER" rm -rf "$REMOTE_TMP" + echo "Cleaning up temporary files..." + $cleanup_remote "$REMOTE_TMP" fi echo "Backup completed: $HOST_DEST" From aa2845e35cceda0fa0b066c6fc96efe47c2a83c9 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Tue, 14 Oct 2025 14:58:07 +0200 Subject: [PATCH 08/14] feat: add restore command and update backup functionality for datastore --- apps/evm/single/main.go | 3 + pkg/cmd/backup.go | 8 - pkg/cmd/backup_test.go | 2 - pkg/cmd/restore.go | 131 +++++++++++++++ pkg/rpc/client/client_test.go | 5 +- pkg/rpc/server/server.go | 10 -- pkg/store/restore.go | 66 ++++++++ pkg/store/types.go | 3 + proto/evnode/v1/state_rpc.proto | 11 +- scripts/reth-backup/README.md | 249 ++++++++++++++--------------- test/mocks/store.go | 149 +++++++++++------ types/pb/evnode/v1/state_rpc.pb.go | 40 ++--- 12 files changed, 441 insertions(+), 236 deletions(-) create mode 100644 pkg/cmd/restore.go create mode 100644 pkg/store/restore.go diff --git a/apps/evm/single/main.go b/apps/evm/single/main.go index d02872a05..a27872ea2 100644 --- a/apps/evm/single/main.go +++ b/apps/evm/single/main.go @@ -25,12 +25,15 @@ func main() { config.AddFlags(rollcmd.NetInfoCmd) backupCmd := rollcmd.NewBackupCmd() config.AddFlags(backupCmd) + restoreCmd := rollcmd.NewRestoreCmd() + config.AddFlags(restoreCmd) rootCmd.AddCommand( cmd.InitCmd(), cmd.RunCmd, cmd.NewRollbackCmd(), backupCmd, + restoreCmd, rollcmd.VersionCmd, rollcmd.NetInfoCmd, rollcmd.StoreUnsafeCleanCmd, diff --git a/pkg/cmd/backup.go b/pkg/cmd/backup.go index 0cbf65c0d..cfc9eb62a 100644 --- a/pkg/cmd/backup.go +++ b/pkg/cmd/backup.go @@ -81,11 +81,6 @@ func NewBackupCmd() *cobra.Command { bytesCount := &countingWriter{} streamWriter := io.MultiWriter(writer, bytesCount) - targetHeight, err := cmd.Flags().GetUint64("target-height") - if err != nil { - return err - } - sinceVersion, err := cmd.Flags().GetUint64("since-version") if err != nil { return err @@ -99,7 +94,6 @@ func NewBackupCmd() *cobra.Command { client := clientrpc.NewClient(baseURL) metadata, backupErr := client.Backup(ctx, &pb.BackupRequest{ - TargetHeight: targetHeight, SinceVersion: sinceVersion, }, streamWriter) if backupErr != nil { @@ -124,7 +118,6 @@ func NewBackupCmd() *cobra.Command { cmd.Printf("Backup saved to %s (%d bytes)\n", outputPath, bytesCount.Bytes()) cmd.Printf("Current height: %d\n", metadata.GetCurrentHeight()) - cmd.Printf("Target height: %d\n", metadata.GetTargetHeight()) cmd.Printf("Since version: %d\n", metadata.GetSinceVersion()) cmd.Printf("Last version: %d\n", metadata.GetLastVersion()) @@ -133,7 +126,6 @@ func NewBackupCmd() *cobra.Command { } cmd.Flags().String("output", "", "Path to the backup file (defaults to ./evnode-backup-.badger)") - cmd.Flags().Uint64("target-height", 0, "Target chain height to align the backup with (0 disables the check)") cmd.Flags().Uint64("since-version", 0, "Generate an incremental backup starting from the provided version") cmd.Flags().Bool("force", false, "Overwrite the output file if it already exists") diff --git a/pkg/cmd/backup_test.go b/pkg/cmd/backup_test.go index d74bf7efd..3875b3f64 100644 --- a/pkg/cmd/backup_test.go +++ b/pkg/cmd/backup_test.go @@ -65,7 +65,6 @@ func TestBackupCmd_Success(t *testing.T) { "--home="+tempDir, "--evnode.rpc.address="+rpcAddr, "--output", outPath, - "--target-height", "12", "--since-version", "9", ) @@ -77,7 +76,6 @@ func TestBackupCmd_Success(t *testing.T) { require.Contains(t, output, "Backup saved to") require.Contains(t, output, "Current height: 15") - require.Contains(t, output, "Target height: 12") require.Contains(t, output, "Since version: 9") require.Contains(t, output, "Last version: 21") diff --git a/pkg/cmd/restore.go b/pkg/cmd/restore.go new file mode 100644 index 000000000..0cd70e31c --- /dev/null +++ b/pkg/cmd/restore.go @@ -0,0 +1,131 @@ +package cmd + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/evstack/ev-node/pkg/store" +) + +// NewRestoreCmd creates a cobra command that restores a datastore from a Badger backup file. +func NewRestoreCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "restore", + Short: "Restore a datastore from a Badger backup file", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + nodeConfig, err := ParseConfig(cmd) + if err != nil { + return fmt.Errorf("error parsing config: %w", err) + } + + inputPath, err := cmd.Flags().GetString("input") + if err != nil { + return err + } + + if inputPath == "" { + return fmt.Errorf("--input flag is required") + } + + inputPath, err = filepath.Abs(inputPath) + if err != nil { + return fmt.Errorf("failed to resolve input path: %w", err) + } + + // Check if input file exists + if _, err := os.Stat(inputPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("backup file not found: %s", inputPath) + } + return fmt.Errorf("failed to access backup file: %w", err) + } + + // Check if datastore already exists + force, err := cmd.Flags().GetBool("force") + if err != nil { + return err + } + + dbPath := filepath.Join(nodeConfig.RootDir, nodeConfig.DBPath) + if _, err := os.Stat(dbPath); err == nil && !force { + return fmt.Errorf("datastore already exists at %s (use --force to overwrite)", dbPath) + } + + // Remove existing datastore if force is enabled + if force { + if err := os.RemoveAll(dbPath); err != nil { + return fmt.Errorf("failed to remove existing datastore: %w", err) + } + } + + // Create the datastore directory + if err := os.MkdirAll(dbPath, 0o755); err != nil { + return fmt.Errorf("failed to create datastore directory: %w", err) + } + + appName, err := cmd.Flags().GetString("app-name") + if err != nil { + return err + } + + if appName == "" { + appName = "ev-node" + } + + // Open the datastore + kvStore, err := store.NewDefaultKVStore(nodeConfig.RootDir, nodeConfig.DBPath, appName) + if err != nil { + return fmt.Errorf("failed to open datastore: %w", err) + } + defer kvStore.Close() + + evStore := store.New(kvStore) + + // Open the backup file + file, err := os.Open(inputPath) + if err != nil { + return fmt.Errorf("failed to open backup file: %w", err) + } + defer file.Close() + + reader := bufio.NewReaderSize(file, 1<<20) // 1 MiB buffer + + ctx := cmd.Context() + if ctx == nil { + ctx = context.Background() + } + + cmd.Println("Restoring datastore from backup...") + + // Perform the restore + if err := evStore.Restore(ctx, reader); err != nil { + return fmt.Errorf("restore failed: %w", err) + } + + // Get the final height + height, err := evStore.Height(ctx) + if err != nil { + cmd.Printf("Warning: could not determine restored height: %v\n", err) + } else { + cmd.Printf("Restore completed successfully\n") + cmd.Printf("Restored height: %d\n", height) + } + + cmd.Printf("Datastore restored to: %s\n", dbPath) + + return nil + }, + } + + cmd.Flags().String("input", "", "Path to the backup file (required)") + cmd.Flags().String("app-name", "", "Application name for the datastore (default: ev-node)") + cmd.Flags().Bool("force", false, "Overwrite existing datastore if it exists") + + return cmd +} diff --git a/pkg/rpc/client/client_test.go b/pkg/rpc/client/client_test.go index fa7080420..c5a49ca46 100644 --- a/pkg/rpc/client/client_test.go +++ b/pkg/rpc/client/client_test.go @@ -191,14 +191,13 @@ func TestClientBackup(t *testing.T) { defer testServer.Close() var buf bytes.Buffer - metadata, err := client.Backup(context.Background(), &pb.BackupRequest{TargetHeight: 10}, &buf) + metadata, err := client.Backup(context.Background(), &pb.BackupRequest{SinceVersion: 5}, &buf) require.NoError(t, err) require.NotNil(t, metadata) require.Equal(t, "chunk-1chunk-2", buf.String()) require.True(t, metadata.GetCompleted()) require.Equal(t, uint64(15), metadata.GetCurrentHeight()) - require.Equal(t, uint64(10), metadata.GetTargetHeight()) - require.Equal(t, uint64(0), metadata.GetSinceVersion()) + require.Equal(t, uint64(5), metadata.GetSinceVersion()) require.Equal(t, uint64(42), metadata.GetLastVersion()) mockStore.AssertExpectations(t) diff --git a/pkg/rpc/server/server.go b/pkg/rpc/server/server.go index 109568e0c..7f0ae5fff 100644 --- a/pkg/rpc/server/server.go +++ b/pkg/rpc/server/server.go @@ -196,23 +196,14 @@ func (s *StoreServer) Backup( stream *connect.ServerStream[pb.BackupResponse], ) error { since := req.Msg.GetSinceVersion() - targetHeight := req.Msg.GetTargetHeight() currentHeight, err := s.store.Height(ctx) if err != nil { return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to get current height: %w", err)) } - if targetHeight != 0 && targetHeight > currentHeight { - return connect.NewError( - connect.CodeFailedPrecondition, - fmt.Errorf("requested target height %d exceeds current height %d", targetHeight, currentHeight), - ) - } - initialMetadata := &pb.BackupMetadata{ CurrentHeight: currentHeight, - TargetHeight: targetHeight, SinceVersion: since, Completed: false, LastVersion: 0, @@ -258,7 +249,6 @@ func (s *StoreServer) Backup( completedMetadata := &pb.BackupMetadata{ CurrentHeight: currentHeight, - TargetHeight: targetHeight, SinceVersion: since, LastVersion: version, Completed: true, diff --git a/pkg/store/restore.go b/pkg/store/restore.go new file mode 100644 index 000000000..96b8e4366 --- /dev/null +++ b/pkg/store/restore.go @@ -0,0 +1,66 @@ +package store + +import ( + "context" + "fmt" + "io" + + ds "github.com/ipfs/go-datastore" + badger4 "github.com/ipfs/go-ds-badger4" +) + +// Restore loads a Badger backup from the provided reader into the datastore. +// This operation will fail if the datastore already contains data, unless the datastore +// supports merging backups. The restore process is atomic and will either complete +// fully or leave the datastore unchanged. +func (s *DefaultStore) Restore(ctx context.Context, reader io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + + visited := make(map[ds.Datastore]struct{}) + current, ok := any(s.db).(ds.Datastore) + if !ok { + return fmt.Errorf("restore is not supported by the configured datastore") + } + + for { + // Try to leverage a native restore implementation if the underlying datastore exposes one. + type restorable interface { + Load(io.Reader) error + } + if dsRestore, ok := current.(restorable); ok { + if err := dsRestore.Load(reader); err != nil { + return fmt.Errorf("datastore restore failed: %w", err) + } + return nil + } + + // Default Badger datastore used across ev-node. + if badgerDatastore, ok := current.(*badger4.Datastore); ok { + // `badger.DB.Load` internally restores the backup atomically. + if err := badgerDatastore.DB.Load(reader, 16); err != nil { + return fmt.Errorf("badger restore failed: %w", err) + } + return nil + } + + // Attempt to unwrap shimmed datastores (e.g., prefix or mutex wrappers) to reach the backing store. + if _, seen := visited[current]; seen { + break + } + visited[current] = struct{}{} + + shim, ok := current.(ds.Shim) + if !ok { + break + } + children := shim.Children() + if len(children) == 0 { + break + } + current = children[0] + } + + return fmt.Errorf("restore is not supported by the configured datastore") +} diff --git a/pkg/store/types.go b/pkg/store/types.go index 844174833..b5f6c9df4 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -80,6 +80,9 @@ type Rollback interface { // as the starting point for incremental backups. Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) + // Restore loads a backup stream from reader into the datastore. + Restore(ctx context.Context, reader io.Reader) error + // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error } diff --git a/proto/evnode/v1/state_rpc.proto b/proto/evnode/v1/state_rpc.proto index 3f71a6f1e..5ec2eb22d 100644 --- a/proto/evnode/v1/state_rpc.proto +++ b/proto/evnode/v1/state_rpc.proto @@ -69,19 +69,16 @@ message GetGenesisDaHeightResponse { // BackupRequest defines the parameters for requesting a datastore backup. message BackupRequest { - // target_height is the Evolve height the client wants to align with. 0 skips the check. - uint64 target_height = 1; // since_version allows incremental backups. 0 produces a full backup. - uint64 since_version = 2; + uint64 since_version = 1; } // BackupMetadata contains progress or completion details emitted during a backup stream. message BackupMetadata { uint64 current_height = 1; - uint64 target_height = 2; - uint64 since_version = 3; - uint64 last_version = 4; - bool completed = 5; + uint64 since_version = 2; + uint64 last_version = 3; + bool completed = 4; } // BackupResponse multiplexes metadata and raw backup data chunks in the stream. diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md index 643fb0d1c..f8cb363a0 100644 --- a/scripts/reth-backup/README.md +++ b/scripts/reth-backup/README.md @@ -89,179 +89,168 @@ environments (SSH, Kubernetes, etc.) without modifying the core backup logic. ## End-to-end workflow with `apps/evm/single` (Docker mode) -The `evm/single` docker-compose setup expects the backup image to reuse the -`ghcr.io/evstack/ev-reth:latest` tag. To capture both the MDBX and ev-node -Badger backups end-to-end: +### Prerequisites -1. Build the helper image so `ev-reth` has the MDBX tooling preinstalled. +1. Build the reth image with MDBX tooling: ```bash docker build -t ghcr.io/evstack/ev-reth:latest scripts/reth-backup ``` -2. Build the `ev-node-evm-single` image so it includes the latest CLI backup - command (optional if you already pushed the binary and re-tagged it). +2. Build the ev-node image with backup/restore commands: ```bash docker build -t ghcr.io/evstack/ev-node-evm-single:main -f apps/evm/single/Dockerfile . ``` -3. Start the stack. +3. Start the stack: ```bash - (cd apps/evm/single && docker compose up -d) + cd apps/evm/single && docker compose up -d ``` -4. Run the MDBX snapshot script (adjust the destination as needed). +### Backup + +1. Backup reth (captures MDBX snapshot at current height): ```bash ./scripts/reth-backup/backup.sh --mode docker backups/full-run/reth ``` - The script prints the generated tag (for example `20251013-104816`) and the - captured height (stored under - `backups/full-run/reth//height.txt`). + Note the printed TAG (e.g., `20251013-104816`) and height. -5. Align the ev-node datastore to that height and take the Badger backup: +2. Backup ev-node (captures complete Badger datastore): ```bash - HEIGHT=$(cat backups/full-run/reth//height.txt) - BACKUP_ROOT="$(pwd)/backups/full-run" - - # Stop the managed container so it cannot advance. - (cd apps/evm/single && docker compose stop ev-node-evm-single) - - # Roll the datastore back to the captured height. The --sync-node and - # --skip-p2p-stores flags avoid DA-finality and header-store checks. - (cd apps/evm/single && docker compose run --rm \ - ev-node-evm-single rollback \ - --height "${HEIGHT}" \ - --home /root/.evm-single \ - --sync-node \ - --skip-p2p-stores) - - # Stream the Badger backup without producing new blocks. - cat <<'EOF' > /tmp/evnode_backup.sh -set -euo pipefail -OUT="$1" -TARGET="$2" -START_CMD="evm-single start \ - --home=/root/.evm-single \ - --evm.jwt-secret $EVM_JWT_SECRET \ - --evm.genesis-hash $EVM_GENESIS_HASH \ - --evm.engine-url $EVM_ENGINE_URL \ - --evm.eth-url $EVM_ETH_URL \ - --rollkit.node.block_time 1h \ - --rollkit.node.aggregator=true \ - --rollkit.signer.passphrase $EVM_SIGNER_PASSPHRASE \ - --rollkit.da.address $DA_ADDRESS \ - --evnode.clear_cache" -rm -f "$OUT" -sh -c "$START_CMD" & -PID=$! -trap "kill $PID 2>/dev/null || true; wait $PID 2>/dev/null || true" EXIT -for i in $(seq 1 30); do - sleep 2 - if evm-single backup --output "$OUT" --force --target-height "$TARGET" >/tmp/backup.log 2>&1; then - cat /tmp/backup.log - exit 0 - fi - cat /tmp/backup.log -done -echo "backup did not succeed within timeout" >&2 -exit 1 -EOF - chmod +x /tmp/evnode_backup.sh - - (cd apps/evm/single && docker compose run --rm \ - --entrypoint sh \ - -v /tmp/evnode_backup.sh:/tmp/evnode_backup.sh \ - -v "${BACKUP_ROOT}/ev-node:/host-backup" \ - ev-node-evm-single \ - -c "/tmp/evnode_backup.sh /host-backup/evnode-backup-aligned.badger ${HEIGHT}") - - rm /tmp/evnode_backup.sh - - # Bring the managed container back with its usual supervisor. - (cd apps/evm/single && docker compose start ev-node-evm-single) + TAG= # from previous step + HEIGHT=$(cat backups/full-run/reth/${TAG}/height.txt) + + mkdir -p backups/full-run/ev-node + + docker exec evolveevm-ev-node-evm-single-1 \ + evm-single backup \ + --output /tmp/backup-${TAG}.badger \ + --force + + docker cp evolveevm-ev-node-evm-single-1:/tmp/backup-${TAG}.badger \ + backups/full-run/ev-node/ + + echo ${HEIGHT} > backups/full-run/ev-node/target-height.txt ``` - The CLI will report the streamed metadata, and the backup lands at - `backups/full-run/ev-node/evnode-backup-aligned.badger`. +### Restore -6. When finished, tear the stack down. +1. Stop services and recreate containers: ```bash - (cd apps/evm/single && docker compose down) + cd apps/evm/single + docker compose down + docker compose up --no-start ``` -Both backups can then be found under `backups/full-run/`. - -## Restoring and validating the backups +2. Restore reth volume: -To verify that both snapshots can be replayed, you can shut everything down, mutate the live data, and then restore from the artifacts collected above. - -1. **Let the stack advance after the backup (optional but recommended).** Keep `docker compose` running for a few more blocks or submit a dev transaction so the live height diverges from the one recorded in `backups/full-run/reth/height.txt`. -2. **Stop the services.** - ```bash - (cd apps/evm/single && docker compose down) - ``` -3. **Recreate the containers without starting them.** This gives you stopped containers that already own the right named volumes. - ```bash - (cd apps/evm/single && docker compose up --no-start) - ``` -4. **Restore the `ev-reth` MDBX volume from the snapshot.** Run the following from the repository root (adjust `BACKUP_ROOT` if you saved the files elsewhere): ```bash - BACKUP_ROOT="$(pwd)/backups/full-run" + TAG= docker run --rm \ --volumes-from ev-reth \ - -v "${BACKUP_ROOT}/reth:/backup:ro" \ + -v "$(pwd)/backups/full-run/reth/${TAG}:/backup:ro" \ alpine:3.18 \ - sh -ec 'rm -rf /home/reth/eth-home/db /home/reth/eth-home/static_files && \ - mkdir -p /home/reth/eth-home/db /home/reth/eth-home/static_files && \ - cp /backup/db/mdbx.dat /home/reth/eth-home/db/mdbx.dat && \ - cp /backup/db/mdbx.lck /home/reth/eth-home/db/mdbx.lck && \ - cp -a /backup/static_files/. /home/reth/eth-home/static_files/ || true' + sh -c 'rm -rf /home/reth/eth-home/db /home/reth/eth-home/static_files && \ + mkdir -p /home/reth/eth-home/db /home/reth/eth-home/static_files && \ + cp /backup/db/mdbx.dat /home/reth/eth-home/db/ && \ + cp /backup/db/mdbx.lck /home/reth/eth-home/db/ && \ + cp -a /backup/static_files/. /home/reth/eth-home/static_files/ || true' ``` - > `docker run --volumes-from ev-reth` reuses the stopped container's volumes; adjust `alpine:3.18` if you prefer another image that provides `cp`. -5. **Restore the `ev-node` Badger datastore.** + +3. Restore ev-node volume: + ```bash - TEMP_RESTORE="$(mktemp -d backups/full-run/ev-node/restore-XXXXXX)" - badger restore --dir "${TEMP_RESTORE}" --files "${BACKUP_ROOT}/ev-node/evnode-backup-aligned.badger" + TAG= docker run --rm \ - --volumes-from ev-node-evm-single \ - -v "${TEMP_RESTORE}:/restore:ro" \ - alpine:3.18 \ - sh -ec 'rm -rf /root/.evm-single/data && mkdir -p /root/.evm-single/data/evm-single && \ - cp -a /restore/. /root/.evm-single/data/evm-single/' - rm -rf "${TEMP_RESTORE}" + --volumes-from evolveevm-ev-node-evm-single-1 \ + -v "$(pwd)/backups/full-run/ev-node:/backup:ro" \ + ghcr.io/evstack/ev-node-evm-single:main \ + restore \ + --input /backup/backup-${TAG}.badger \ + --home /root/.evm-single \ + --app-name evm-single \ + --force ``` - > Install the Badger CLI once via `go install github.com/dgraph-io/badger/v4/cmd/badger@latest` if it is not already on your `$PATH`. -6. **Start the services back up.** + +4. Align ev-node to reth height using rollback (before starting): + ```bash - (cd apps/evm/single && docker compose up -d ev-reth local-da) - (cd apps/evm/single && docker compose up -d ev-node-evm-single) + HEIGHT=$(cat backups/full-run/ev-node/target-height.txt) + + docker run --rm \ + --volumes-from evolveevm-ev-node-evm-single-1 \ + ghcr.io/evstack/ev-node-evm-single:main \ + rollback \ + --home /root/.evm-single \ + --height ${HEIGHT} \ + --sync-node ``` - If you prefer to launch the sequencer manually first, you can run: + + > **Note:** The rollback may report errors for p2p header/data stores with invalid + > ranges. This is expected and can be ignored. The main state will be correctly + > rolled back to the target height. The `--sync-node` flag is required for + > non-aggregator mode rollback. + +5. Start services with cache cleared: + ```bash - (cd apps/evm/single && docker compose run --rm \ - ev-node-evm-single start \ - --home /root/.evm-single \ - --evm.jwt-secret "$EVM_JWT_SECRET" \ - --evm.genesis-hash "$EVM_GENESIS_HASH" \ - --evm.engine-url "$EVM_ENGINE_URL" \ - --evm.eth-url "$EVM_ETH_URL" \ - --rollkit.node.block_time 1s \ - --rollkit.node.aggregator=true \ - --rollkit.signer.passphrase "$EVM_SIGNER_PASSPHRASE" \ - --rollkit.da.address "$DA_ADDRESS" \ - --evnode.clear_cache) + docker compose run --rm ev-node-evm-single start --evnode.clear_cache ``` - and once it exits cleanly, start the managed container with `docker compose up -d ev-node-evm-single`. -7. **Confirm the node resumes from the backed-up height.** Compare the logged chain height with `backups/full-run/reth/height.txt` and run: + + > **Important:** Use `--evnode.clear_cache` on first start after restore to clear + > any cached p2p data that may be inconsistent after rollback. + +6. Verify both nodes are at the same height: + ```bash - (cd apps/evm/single && docker compose exec ev-node-evm-single evm-single net-info --home /root/.evm-single) + HEIGHT=$(cat backups/full-run/ev-node/target-height.txt) + echo "Expected height: ${HEIGHT}" + + # Check ev-node logs for produced blocks + docker logs -f 2>&1 | grep "initialized state" ``` - The reported height should match the snapshot before new blocks are produced. -With this round-trip check you can be confident that the snapshot pair (MDBX + Badger) fully recreates the state captured during the backup. +## Known Limitations + +### Rollback P2P Store Errors + +When rolling back to a height significantly lower than the current state, the p2p +header and data sync stores may report "invalid range" errors. This occurs because +these stores track sync progress independently. The errors can be safely ignored as: + +1. The main blockchain state is correctly rolled back +2. Using `--evnode.clear_cache` on restart clears the inconsistent cache +3. The node will resync p2p data from the restored height + +### Timestamp Consistency + +After a restore, if significant real-world time has passed since the backup was created, +you may encounter timestamp validation errors when the node attempts to continue block +production. This occurs because: + +- Reth stores block timestamps based on when blocks were originally created +- After restore, the restored timestamps may be in the past relative to system time +- Block validators may reject new blocks with timestamps earlier than parent blocks + +**Workaround:** In production environments, coordinate restore operations to minimize +time between backup and restore, or ensure the entire network is restored simultaneously. + +## Summary + +This backup/restore workflow enables point-in-time recovery for both reth (MDBX) and +ev-node (Badger) datastores. Key points: + +- **Backup**: Hot backup while nodes are running (no downtime) +- **Restore**: Requires stopping services, restoring volumes, and aligning heights +- **Rollback**: May show p2p store errors that can be safely ignored +- **Production**: Test the full workflow in staging before deploying to production + +The process has been validated to correctly restore state and resume block production +from the backup point, with known limitations around p2p store consistency and timestamp +validation that can be mitigated with proper operational procedures. diff --git a/test/mocks/store.go b/test/mocks/store.go index 54e171062..64f69e6c5 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -40,50 +40,6 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter { return &MockStore_Expecter{mock: &_m.Mock} } -// Close provides a mock function for the type MockStore -func (_mock *MockStore) Close() error { - ret := _mock.Called() - - if len(ret) == 0 { - panic("no return value specified for Close") - } - - var r0 error - if returnFunc, ok := ret.Get(0).(func() error); ok { - r0 = returnFunc() - } else { - r0 = ret.Error(0) - } - return r0 -} - -// MockStore_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' -type MockStore_Close_Call struct { - *mock.Call -} - -// Close is a helper method to define mock.On call -func (_e *MockStore_Expecter) Close() *MockStore_Close_Call { - return &MockStore_Close_Call{Call: _e.mock.On("Close")} -} - -func (_c *MockStore_Close_Call) Run(run func()) *MockStore_Close_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockStore_Close_Call) Return(err error) *MockStore_Close_Call { - _c.Call.Return(err) - return _c -} - -func (_c *MockStore_Close_Call) RunAndReturn(run func() error) *MockStore_Close_Call { - _c.Call.Return(run) - return _c -} - // Backup provides a mock function for the type MockStore func (_mock *MockStore) Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) { ret := _mock.Called(ctx, writer, since) @@ -146,8 +102,8 @@ func (_c *MockStore_Backup_Call) Run(run func(ctx context.Context, writer io.Wri return _c } -func (_c *MockStore_Backup_Call) Return(version uint64, err error) *MockStore_Backup_Call { - _c.Call.Return(version, err) +func (_c *MockStore_Backup_Call) Return(v uint64, err error) *MockStore_Backup_Call { + _c.Call.Return(v, err) return _c } @@ -156,6 +112,50 @@ func (_c *MockStore_Backup_Call) RunAndReturn(run func(ctx context.Context, writ return _c } +// Close provides a mock function for the type MockStore +func (_mock *MockStore) Close() error { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Close") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func() error); ok { + r0 = returnFunc() + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockStore_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type MockStore_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *MockStore_Expecter) Close() *MockStore_Close_Call { + return &MockStore_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *MockStore_Close_Call) Run(run func()) *MockStore_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockStore_Close_Call) Return(err error) *MockStore_Close_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockStore_Close_Call) RunAndReturn(run func() error) *MockStore_Close_Call { + _c.Call.Return(run) + return _c +} + // GetBlockByHash provides a mock function for the type MockStore func (_mock *MockStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { ret := _mock.Called(ctx, hash) @@ -828,6 +828,63 @@ func (_c *MockStore_NewBatch_Call) RunAndReturn(run func(ctx context.Context) (s return _c } +// Restore provides a mock function for the type MockStore +func (_mock *MockStore) Restore(ctx context.Context, reader io.Reader) error { + ret := _mock.Called(ctx, reader) + + if len(ret) == 0 { + panic("no return value specified for Restore") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, io.Reader) error); ok { + r0 = returnFunc(ctx, reader) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockStore_Restore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Restore' +type MockStore_Restore_Call struct { + *mock.Call +} + +// Restore is a helper method to define mock.On call +// - ctx context.Context +// - reader io.Reader +func (_e *MockStore_Expecter) Restore(ctx interface{}, reader interface{}) *MockStore_Restore_Call { + return &MockStore_Restore_Call{Call: _e.mock.On("Restore", ctx, reader)} +} + +func (_c *MockStore_Restore_Call) Run(run func(ctx context.Context, reader io.Reader)) *MockStore_Restore_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 io.Reader + if args[1] != nil { + arg1 = args[1].(io.Reader) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockStore_Restore_Call) Return(err error) *MockStore_Restore_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockStore_Restore_Call) RunAndReturn(run func(ctx context.Context, reader io.Reader) error) *MockStore_Restore_Call { + _c.Call.Return(run) + return _c +} + // Rollback provides a mock function for the type MockStore func (_mock *MockStore) Rollback(ctx context.Context, height uint64, aggregator bool) error { ret := _mock.Called(ctx, height, aggregator) diff --git a/types/pb/evnode/v1/state_rpc.pb.go b/types/pb/evnode/v1/state_rpc.pb.go index b90cdb0c1..4e2829cb5 100644 --- a/types/pb/evnode/v1/state_rpc.pb.go +++ b/types/pb/evnode/v1/state_rpc.pb.go @@ -10,7 +10,6 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" emptypb "google.golang.org/protobuf/types/known/emptypb" - _ "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -405,10 +404,8 @@ func (x *GetGenesisDaHeightResponse) GetHeight() uint64 { // BackupRequest defines the parameters for requesting a datastore backup. type BackupRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // target_height is the Evolve height the client wants to align with. 0 skips the check. - TargetHeight uint64 `protobuf:"varint,1,opt,name=target_height,json=targetHeight,proto3" json:"target_height,omitempty"` // since_version allows incremental backups. 0 produces a full backup. - SinceVersion uint64 `protobuf:"varint,2,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` + SinceVersion uint64 `protobuf:"varint,1,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -443,13 +440,6 @@ func (*BackupRequest) Descriptor() ([]byte, []int) { return file_evnode_v1_state_rpc_proto_rawDescGZIP(), []int{7} } -func (x *BackupRequest) GetTargetHeight() uint64 { - if x != nil { - return x.TargetHeight - } - return 0 -} - func (x *BackupRequest) GetSinceVersion() uint64 { if x != nil { return x.SinceVersion @@ -461,10 +451,9 @@ func (x *BackupRequest) GetSinceVersion() uint64 { type BackupMetadata struct { state protoimpl.MessageState `protogen:"open.v1"` CurrentHeight uint64 `protobuf:"varint,1,opt,name=current_height,json=currentHeight,proto3" json:"current_height,omitempty"` - TargetHeight uint64 `protobuf:"varint,2,opt,name=target_height,json=targetHeight,proto3" json:"target_height,omitempty"` - SinceVersion uint64 `protobuf:"varint,3,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` - LastVersion uint64 `protobuf:"varint,4,opt,name=last_version,json=lastVersion,proto3" json:"last_version,omitempty"` - Completed bool `protobuf:"varint,5,opt,name=completed,proto3" json:"completed,omitempty"` + SinceVersion uint64 `protobuf:"varint,2,opt,name=since_version,json=sinceVersion,proto3" json:"since_version,omitempty"` + LastVersion uint64 `protobuf:"varint,3,opt,name=last_version,json=lastVersion,proto3" json:"last_version,omitempty"` + Completed bool `protobuf:"varint,4,opt,name=completed,proto3" json:"completed,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -506,13 +495,6 @@ func (x *BackupMetadata) GetCurrentHeight() uint64 { return 0 } -func (x *BackupMetadata) GetTargetHeight() uint64 { - if x != nil { - return x.TargetHeight - } - return 0 -} - func (x *BackupMetadata) GetSinceVersion() uint64 { if x != nil { return x.SinceVersion @@ -621,7 +603,7 @@ var File_evnode_v1_state_rpc_proto protoreflect.FileDescriptor const file_evnode_v1_state_rpc_proto_rawDesc = "" + "\n" + - "\x19evnode/v1/state_rpc.proto\x12\tevnode.v1\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x16evnode/v1/evnode.proto\x1a\x15evnode/v1/state.proto\"]\n" + + "\x19evnode/v1/state_rpc.proto\x12\tevnode.v1\x1a\x16evnode/v1/evnode.proto\x1a\x15evnode/v1/state.proto\x1a\x1bgoogle/protobuf/empty.proto\"]\n" + "\x05Block\x12/\n" + "\x06header\x18\x01 \x01(\v2\x17.evnode.v1.SignedHeaderR\x06header\x12#\n" + "\x04data\x18\x02 \x01(\v2\x0f.evnode.v1.DataR\x04data\"O\n" + @@ -641,16 +623,14 @@ const file_evnode_v1_state_rpc_proto_rawDesc = "" + "\x13GetMetadataResponse\x12\x14\n" + "\x05value\x18\x01 \x01(\fR\x05value\"4\n" + "\x1aGetGenesisDaHeightResponse\x12\x16\n" + - "\x06height\x18\x03 \x01(\x04R\x06height\"Y\n" + + "\x06height\x18\x03 \x01(\x04R\x06height\"4\n" + "\rBackupRequest\x12#\n" + - "\rtarget_height\x18\x01 \x01(\x04R\ftargetHeight\x12#\n" + - "\rsince_version\x18\x02 \x01(\x04R\fsinceVersion\"\xc2\x01\n" + + "\rsince_version\x18\x01 \x01(\x04R\fsinceVersion\"\x9d\x01\n" + "\x0eBackupMetadata\x12%\n" + "\x0ecurrent_height\x18\x01 \x01(\x04R\rcurrentHeight\x12#\n" + - "\rtarget_height\x18\x02 \x01(\x04R\ftargetHeight\x12#\n" + - "\rsince_version\x18\x03 \x01(\x04R\fsinceVersion\x12!\n" + - "\flast_version\x18\x04 \x01(\x04R\vlastVersion\x12\x1c\n" + - "\tcompleted\x18\x05 \x01(\bR\tcompleted\"m\n" + + "\rsince_version\x18\x02 \x01(\x04R\fsinceVersion\x12!\n" + + "\flast_version\x18\x03 \x01(\x04R\vlastVersion\x12\x1c\n" + + "\tcompleted\x18\x04 \x01(\bR\tcompleted\"m\n" + "\x0eBackupResponse\x127\n" + "\bmetadata\x18\x01 \x01(\v2\x19.evnode.v1.BackupMetadataH\x00R\bmetadata\x12\x16\n" + "\x05chunk\x18\x02 \x01(\fH\x00R\x05chunkB\n" + From 32578cb976abc52115f950dacbab153778039892 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Tue, 14 Oct 2025 17:20:32 +0200 Subject: [PATCH 09/14] fix(restore): simplify success message after datastore restoration --- pkg/cmd/restore.go | 10 +------ scripts/reth-backup/README.md | 50 ++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/pkg/cmd/restore.go b/pkg/cmd/restore.go index 0cd70e31c..349a5b98e 100644 --- a/pkg/cmd/restore.go +++ b/pkg/cmd/restore.go @@ -108,15 +108,7 @@ func NewRestoreCmd() *cobra.Command { return fmt.Errorf("restore failed: %w", err) } - // Get the final height - height, err := evStore.Height(ctx) - if err != nil { - cmd.Printf("Warning: could not determine restored height: %v\n", err) - } else { - cmd.Printf("Restore completed successfully\n") - cmd.Printf("Restored height: %d\n", height) - } - + cmd.Printf("Restore completed successfully\n") cmd.Printf("Datastore restored to: %s\n", dbPath) return nil diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md index f8cb363a0..6d374c5fe 100644 --- a/scripts/reth-backup/README.md +++ b/scripts/reth-backup/README.md @@ -152,9 +152,11 @@ environments (SSH, Kubernetes, etc.) without modifying the core backup logic. ```bash TAG= + + # From apps/evm/single directory, use relative path to backups docker run --rm \ --volumes-from ev-reth \ - -v "$(pwd)/backups/full-run/reth/${TAG}:/backup:ro" \ + -v "$PWD/../../backups/full-run/reth/${TAG}:/backup:ro" \ alpine:3.18 \ sh -c 'rm -rf /home/reth/eth-home/db /home/reth/eth-home/static_files && \ mkdir -p /home/reth/eth-home/db /home/reth/eth-home/static_files && \ @@ -167,9 +169,11 @@ environments (SSH, Kubernetes, etc.) without modifying the core backup logic. ```bash TAG= + + # From apps/evm/single directory, use relative path to backups docker run --rm \ --volumes-from evolveevm-ev-node-evm-single-1 \ - -v "$(pwd)/backups/full-run/ev-node:/backup:ro" \ + -v "$PWD/../../backups/full-run/ev-node:/backup:ro" \ ghcr.io/evstack/ev-node-evm-single:main \ restore \ --input /backup/backup-${TAG}.badger \ @@ -197,23 +201,51 @@ environments (SSH, Kubernetes, etc.) without modifying the core backup logic. > rolled back to the target height. The `--sync-node` flag is required for > non-aggregator mode rollback. -5. Start services with cache cleared: +5. Start reth and local-da services: ```bash - docker compose run --rm ev-node-evm-single start --evnode.clear_cache + docker compose start ev-reth local-da + ``` + +6. Start ev-node with cache cleared (first time only): + + ```bash + # Remove the stopped container and start with --evnode.clear_cache + docker rm evolveevm-ev-node-evm-single-1 + + docker run -d \ + --name evolveevm-ev-node-evm-single-1 \ + --network evolveevm_evolve-network \ + -p 7676:7676 -p 7331:7331 \ + -v evolveevm_evm-single-data:/root/.evm-single/ \ + -e EVM_ENGINE_URL=http://ev-reth:8551 \ + -e EVM_ETH_URL=http://ev-reth:8545 \ + -e EVM_JWT_SECRET=f747494bb0fb338a0d71f5f9fe5b5034c17cc988c229b59fd71e005ee692e9bf \ + -e EVM_GENESIS_HASH=0x2b8bbb1ea1e04f9c9809b4b278a8687806edc061a356c7dbc491930d8e922503 \ + -e EVM_BLOCK_TIME=1s \ + -e EVM_SIGNER_PASSPHRASE=secret \ + -e DA_ADDRESS=http://local-da:7980 \ + ghcr.io/evstack/ev-node-evm-single:main \ + start --evnode.clear_cache ``` > **Important:** Use `--evnode.clear_cache` on first start after restore to clear - > any cached p2p data that may be inconsistent after rollback. + > any cached p2p data that may be inconsistent after rollback. On subsequent restarts, + > you can use `docker compose up -d` normally. -6. Verify both nodes are at the same height: +7. Verify both nodes are at the same height: ```bash HEIGHT=$(cat backups/full-run/ev-node/target-height.txt) - echo "Expected height: ${HEIGHT}" + echo "Expected restored height: ${HEIGHT}" + + # Check ev-node is producing blocks from the restored height + docker logs evolveevm-ev-node-evm-single-1 2>&1 | grep "produced block" | head -10 - # Check ev-node logs for produced blocks - docker logs -f 2>&1 | grep "initialized state" + # Check reth current height + docker exec ev-reth curl -s -X POST -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://localhost:8545 | jq -r '.result' | xargs printf "%d\n" ``` ## Known Limitations From 097f3ce714ee19bd21e05c71f78fe76c119c2a32 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Tue, 14 Oct 2025 21:06:01 +0200 Subject: [PATCH 10/14] feat: add description for BackupResponse to clarify response types --- proto/evnode/v1/state_rpc.proto | 1 + types/pb/evnode/v1/state_rpc.pb.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/proto/evnode/v1/state_rpc.proto b/proto/evnode/v1/state_rpc.proto index 5ec2eb22d..b41b50268 100644 --- a/proto/evnode/v1/state_rpc.proto +++ b/proto/evnode/v1/state_rpc.proto @@ -83,6 +83,7 @@ message BackupMetadata { // BackupResponse multiplexes metadata and raw backup data chunks in the stream. message BackupResponse { + // response contains either metadata about the backup progress or a chunk of backup data. oneof response { BackupMetadata metadata = 1; bytes chunk = 2; diff --git a/types/pb/evnode/v1/state_rpc.pb.go b/types/pb/evnode/v1/state_rpc.pb.go index 4e2829cb5..06e1fa8d3 100644 --- a/types/pb/evnode/v1/state_rpc.pb.go +++ b/types/pb/evnode/v1/state_rpc.pb.go @@ -519,6 +519,8 @@ func (x *BackupMetadata) GetCompleted() bool { // BackupResponse multiplexes metadata and raw backup data chunks in the stream. type BackupResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + // response contains either metadata about the backup progress or a chunk of backup data. + // // Types that are valid to be assigned to Response: // // *BackupResponse_Metadata From 97d49c6982123cb46a851ed01fd4b108f12324e0 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Wed, 15 Oct 2025 11:41:37 +0200 Subject: [PATCH 11/14] feat: streamline backup process for badger4 datastore and improve error handling --- pkg/cmd/restore.go | 2 +- pkg/store/backup.go | 58 ++++++++++++++------------------------------- 2 files changed, 19 insertions(+), 41 deletions(-) diff --git a/pkg/cmd/restore.go b/pkg/cmd/restore.go index 349a5b98e..ff531833d 100644 --- a/pkg/cmd/restore.go +++ b/pkg/cmd/restore.go @@ -46,12 +46,12 @@ func NewRestoreCmd() *cobra.Command { return fmt.Errorf("failed to access backup file: %w", err) } - // Check if datastore already exists force, err := cmd.Flags().GetBool("force") if err != nil { return err } + // Check if datastore already exists dbPath := filepath.Join(nodeConfig.RootDir, nodeConfig.DBPath) if _, err := os.Stat(dbPath); err == nil && !force { return fmt.Errorf("datastore already exists at %s (use --force to overwrite)", dbPath) diff --git a/pkg/store/backup.go b/pkg/store/backup.go index 26600435e..361b90aad 100644 --- a/pkg/store/backup.go +++ b/pkg/store/backup.go @@ -17,51 +17,29 @@ func (s *DefaultStore) Backup(ctx context.Context, writer io.Writer, since uint6 return 0, err } - visited := make(map[ds.Datastore]struct{}) - current, ok := any(s.db).(ds.Datastore) - if !ok { - return 0, fmt.Errorf("backup is not supported by the configured datastore") + // Try direct badger4 cast first + if badgerDatastore, ok := s.db.(*badger4.Datastore); ok { + return backupBadger(badgerDatastore, writer, since) } - for { - // Try to leverage a native backup implementation if the underlying datastore exposes one. - type backupable interface { - Backup(io.Writer, uint64) (uint64, error) - } - if dsBackup, ok := current.(backupable); ok { - version, err := dsBackup.Backup(writer, since) - if err != nil { - return 0, fmt.Errorf("datastore backup failed: %w", err) - } - return version, nil - } - - // Default Badger datastore used across ev-node. - if badgerDatastore, ok := current.(*badger4.Datastore); ok { - // `badger.DB.Backup` internally orchestrates a consistent snapshot without pausing writes. - version, err := badgerDatastore.DB.Backup(writer, since) - if err != nil { - return 0, fmt.Errorf("badger backup failed: %w", err) + // Try to unwrap one level (e.g., PrefixTransform wrapper) + if shim, ok := s.db.(ds.Shim); ok { + children := shim.Children() + if len(children) > 0 { + if badgerDatastore, ok := children[0].(*badger4.Datastore); ok { + return backupBadger(badgerDatastore, writer, since) } - return version, nil } + } - // Attempt to unwrap shimmed datastores (e.g., prefix or mutex wrappers) to reach the backing store. - if _, seen := visited[current]; seen { - break - } - visited[current] = struct{}{} + return 0, fmt.Errorf("backup is only supported for badger4 datastore") +} - shim, ok := current.(ds.Shim) - if !ok { - break - } - children := shim.Children() - if len(children) == 0 { - break - } - current = children[0] +func backupBadger(badgerDatastore *badger4.Datastore, writer io.Writer, since uint64) (uint64, error) { + // `badger.DB.Backup` internally orchestrates a consistent snapshot without pausing writes. + version, err := badgerDatastore.DB.Backup(writer, since) + if err != nil { + return 0, fmt.Errorf("badger backup failed: %w", err) } - - return 0, fmt.Errorf("backup is not supported by the configured datastore") + return version, nil } From 0cf5f9de389489b315219651631b1b3529dcc6a1 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 16 Oct 2025 11:04:42 +0200 Subject: [PATCH 12/14] feat: add Backup interface to Store and reorder Rollback interface definition --- pkg/store/types.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/store/types.go b/pkg/store/types.go index b5f6c9df4..6e1ec0772 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -33,6 +33,7 @@ type Batch interface { // Store is minimal interface for storing and retrieving blocks, commits and state. type Store interface { Rollback + Backup Reader // SetMetadata saves arbitrary value in the store. @@ -71,11 +72,7 @@ type Reader interface { GetMetadata(ctx context.Context, key string) ([]byte, error) } -type Rollback interface { - // Rollback deletes x height from the ev-node store. - // Aggregator is used to determine if the rollback is performed on the aggregator node. - Rollback(ctx context.Context, height uint64, aggregator bool) error - +type Backup interface { // Backup writes a consistent backup stream to writer. The returned version can be used // as the starting point for incremental backups. Backup(ctx context.Context, writer io.Writer, since uint64) (uint64, error) @@ -86,3 +83,9 @@ type Rollback interface { // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error } + +type Rollback interface { + // Rollback deletes x height from the ev-node store. + // Aggregator is used to determine if the rollback is performed on the aggregator node. + Rollback(ctx context.Context, height uint64, aggregator bool) error +} From f8f5af11d8c5ed6b654486339b12de83e39ae538 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 16 Oct 2025 11:07:11 +0200 Subject: [PATCH 13/14] feat: add backup and restore commands with configuration flags --- apps/testapp/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/testapp/main.go b/apps/testapp/main.go index cd8f01970..64ab9fc42 100644 --- a/apps/testapp/main.go +++ b/apps/testapp/main.go @@ -6,6 +6,7 @@ import ( cmds "github.com/evstack/ev-node/apps/testapp/cmd" rollcmd "github.com/evstack/ev-node/pkg/cmd" + "github.com/evstack/ev-node/pkg/config" ) func main() { @@ -13,6 +14,12 @@ func main() { rootCmd := cmds.RootCmd initCmd := cmds.InitCmd() + // Add configuration flags to backup and restore commands + backupCmd := rollcmd.NewBackupCmd() + config.AddFlags(backupCmd) + restoreCmd := rollcmd.NewRestoreCmd() + config.AddFlags(restoreCmd) + // Add subcommands to the root command rootCmd.AddCommand( cmds.RunCmd, @@ -21,6 +28,8 @@ func main() { rollcmd.StoreUnsafeCleanCmd, rollcmd.KeysCmd(), cmds.NewRollbackCmd(), + backupCmd, + restoreCmd, initCmd, ) From 15401d841e5a10bde70ae6d50f4c475f5fc1f769 Mon Sep 17 00:00:00 2001 From: Randy Grok Date: Thu, 16 Oct 2025 12:36:44 +0200 Subject: [PATCH 14/14] remove scripts and move them to ev-reth --- scripts/reth-backup/Dockerfile | 24 --- scripts/reth-backup/README.md | 288 ------------------------------ scripts/reth-backup/backup-lib.sh | 165 ----------------- scripts/reth-backup/backup.sh | 220 ----------------------- 4 files changed, 697 deletions(-) delete mode 100644 scripts/reth-backup/Dockerfile delete mode 100644 scripts/reth-backup/README.md delete mode 100644 scripts/reth-backup/backup-lib.sh delete mode 100755 scripts/reth-backup/backup.sh diff --git a/scripts/reth-backup/Dockerfile b/scripts/reth-backup/Dockerfile deleted file mode 100644 index e328bae19..000000000 --- a/scripts/reth-backup/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ghcr.io/evstack/ev-reth:latest - -ARG LIBMDBX_REPO=https://github.com/erthink/libmdbx.git -ARG LIBMDBX_REF=master - -RUN set -eux; \ - apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - cmake \ - git \ - jq \ - ; \ - rm -rf /var/lib/apt/lists/* - -RUN set -eux; \ - git clone --depth 1 --branch "${LIBMDBX_REF}" "${LIBMDBX_REPO}" /tmp/libmdbx; \ - cmake -S /tmp/libmdbx -B /tmp/libmdbx/build -DCMAKE_BUILD_TYPE=Release; \ - cmake --build /tmp/libmdbx/build --target mdbx_copy mdbx_dump mdbx_chk; \ - install -m 0755 /tmp/libmdbx/build/mdbx_copy /usr/local/bin/mdbx_copy; \ - install -m 0755 /tmp/libmdbx/build/mdbx_dump /usr/local/bin/mdbx_dump; \ - install -m 0755 /tmp/libmdbx/build/mdbx_chk /usr/local/bin/mdbx_chk; \ - rm -rf /tmp/libmdbx diff --git a/scripts/reth-backup/README.md b/scripts/reth-backup/README.md deleted file mode 100644 index 6d374c5fe..000000000 --- a/scripts/reth-backup/README.md +++ /dev/null @@ -1,288 +0,0 @@ -# Reth Backup Helper - -Script to snapshot the `ev-reth` MDBX database while the node keeps running and -record the block height contained in the snapshot. - -The script supports two execution modes: - -- **local**: Backup a reth instance running directly on the host machine -- **docker**: Backup a reth instance running in a Docker container - -## Prerequisites - -### Common requirements - -- The `mdbx_copy` binary available in the target environment (see [libmdbx - documentation](https://libmdbx.dqdkfa.ru/)). -- `jq` installed on the host to parse the JSON output. - -### Docker mode - -- Docker access to the container running `ev-reth` (defaults to the service name - `ev-reth` from `docker-compose`). - -### Local mode - -- Direct filesystem access to the reth datadir. -- Sufficient permissions to read the database files. - -## Usage - -### Local mode - -When reth is running directly on your machine: - -```bash -./scripts/reth-backup/backup.sh \ - --mode local \ - --datadir /var/lib/reth \ - --mdbx-copy /usr/local/bin/mdbx_copy \ - /path/to/backups -``` - -### Docker mode - -When reth is running in a Docker container: - -```bash -./scripts/reth-backup/backup.sh \ - --mode docker \ - --container ev-reth \ - --datadir /home/reth/eth-home \ - --mdbx-copy /tmp/libmdbx/build/mdbx_copy \ - /path/to/backups -``` - -### Output structure - -Both modes create a timestamped folder under `/path/to/backups` with: - -- `db/mdbx.dat` – consistent MDBX snapshot. -- `db/mdbx.lck` – placeholder lock file (empty). -- `static_files/` – static files copied from the node. -- `stage_checkpoints.json` – raw StageCheckpoints table. -- `height.txt` – extracted block height (from the `Finish` stage). - -Additional flags: - -- `--tag LABEL` to override the timestamped folder name. -- `--keep-remote` to leave the temporary snapshot in the target environment - (useful for debugging). - -The script outputs the height at the end so you can coordinate other backups -with the same block number. - -## Architecture - -The backup script is split into two components: - -- **`backup-lib.sh`**: Abstract execution layer providing a common interface for - different execution modes (local, docker). This library defines functions like - `exec_remote`, `copy_from_remote`, `copy_to_remote`, and `cleanup_remote` - that are implemented differently for each backend. -- **`backup.sh`**: Main script that uses the library and orchestrates the backup - workflow. It's mode-agnostic and works with any backend that implements the - required interface. - -This separation allows easy extension to support additional execution -environments (SSH, Kubernetes, etc.) without modifying the core backup logic. - -## End-to-end workflow with `apps/evm/single` (Docker mode) - -### Prerequisites - -1. Build the reth image with MDBX tooling: - - ```bash - docker build -t ghcr.io/evstack/ev-reth:latest scripts/reth-backup - ``` - -2. Build the ev-node image with backup/restore commands: - - ```bash - docker build -t ghcr.io/evstack/ev-node-evm-single:main -f apps/evm/single/Dockerfile . - ``` - -3. Start the stack: - - ```bash - cd apps/evm/single && docker compose up -d - ``` - -### Backup - -1. Backup reth (captures MDBX snapshot at current height): - - ```bash - ./scripts/reth-backup/backup.sh --mode docker backups/full-run/reth - ``` - - Note the printed TAG (e.g., `20251013-104816`) and height. - -2. Backup ev-node (captures complete Badger datastore): - - ```bash - TAG= # from previous step - HEIGHT=$(cat backups/full-run/reth/${TAG}/height.txt) - - mkdir -p backups/full-run/ev-node - - docker exec evolveevm-ev-node-evm-single-1 \ - evm-single backup \ - --output /tmp/backup-${TAG}.badger \ - --force - - docker cp evolveevm-ev-node-evm-single-1:/tmp/backup-${TAG}.badger \ - backups/full-run/ev-node/ - - echo ${HEIGHT} > backups/full-run/ev-node/target-height.txt - ``` - -### Restore - -1. Stop services and recreate containers: - - ```bash - cd apps/evm/single - docker compose down - docker compose up --no-start - ``` - -2. Restore reth volume: - - ```bash - TAG= - - # From apps/evm/single directory, use relative path to backups - docker run --rm \ - --volumes-from ev-reth \ - -v "$PWD/../../backups/full-run/reth/${TAG}:/backup:ro" \ - alpine:3.18 \ - sh -c 'rm -rf /home/reth/eth-home/db /home/reth/eth-home/static_files && \ - mkdir -p /home/reth/eth-home/db /home/reth/eth-home/static_files && \ - cp /backup/db/mdbx.dat /home/reth/eth-home/db/ && \ - cp /backup/db/mdbx.lck /home/reth/eth-home/db/ && \ - cp -a /backup/static_files/. /home/reth/eth-home/static_files/ || true' - ``` - -3. Restore ev-node volume: - - ```bash - TAG= - - # From apps/evm/single directory, use relative path to backups - docker run --rm \ - --volumes-from evolveevm-ev-node-evm-single-1 \ - -v "$PWD/../../backups/full-run/ev-node:/backup:ro" \ - ghcr.io/evstack/ev-node-evm-single:main \ - restore \ - --input /backup/backup-${TAG}.badger \ - --home /root/.evm-single \ - --app-name evm-single \ - --force - ``` - -4. Align ev-node to reth height using rollback (before starting): - - ```bash - HEIGHT=$(cat backups/full-run/ev-node/target-height.txt) - - docker run --rm \ - --volumes-from evolveevm-ev-node-evm-single-1 \ - ghcr.io/evstack/ev-node-evm-single:main \ - rollback \ - --home /root/.evm-single \ - --height ${HEIGHT} \ - --sync-node - ``` - - > **Note:** The rollback may report errors for p2p header/data stores with invalid - > ranges. This is expected and can be ignored. The main state will be correctly - > rolled back to the target height. The `--sync-node` flag is required for - > non-aggregator mode rollback. - -5. Start reth and local-da services: - - ```bash - docker compose start ev-reth local-da - ``` - -6. Start ev-node with cache cleared (first time only): - - ```bash - # Remove the stopped container and start with --evnode.clear_cache - docker rm evolveevm-ev-node-evm-single-1 - - docker run -d \ - --name evolveevm-ev-node-evm-single-1 \ - --network evolveevm_evolve-network \ - -p 7676:7676 -p 7331:7331 \ - -v evolveevm_evm-single-data:/root/.evm-single/ \ - -e EVM_ENGINE_URL=http://ev-reth:8551 \ - -e EVM_ETH_URL=http://ev-reth:8545 \ - -e EVM_JWT_SECRET=f747494bb0fb338a0d71f5f9fe5b5034c17cc988c229b59fd71e005ee692e9bf \ - -e EVM_GENESIS_HASH=0x2b8bbb1ea1e04f9c9809b4b278a8687806edc061a356c7dbc491930d8e922503 \ - -e EVM_BLOCK_TIME=1s \ - -e EVM_SIGNER_PASSPHRASE=secret \ - -e DA_ADDRESS=http://local-da:7980 \ - ghcr.io/evstack/ev-node-evm-single:main \ - start --evnode.clear_cache - ``` - - > **Important:** Use `--evnode.clear_cache` on first start after restore to clear - > any cached p2p data that may be inconsistent after rollback. On subsequent restarts, - > you can use `docker compose up -d` normally. - -7. Verify both nodes are at the same height: - - ```bash - HEIGHT=$(cat backups/full-run/ev-node/target-height.txt) - echo "Expected restored height: ${HEIGHT}" - - # Check ev-node is producing blocks from the restored height - docker logs evolveevm-ev-node-evm-single-1 2>&1 | grep "produced block" | head -10 - - # Check reth current height - docker exec ev-reth curl -s -X POST -H "Content-Type: application/json" \ - --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ - http://localhost:8545 | jq -r '.result' | xargs printf "%d\n" - ``` - -## Known Limitations - -### Rollback P2P Store Errors - -When rolling back to a height significantly lower than the current state, the p2p -header and data sync stores may report "invalid range" errors. This occurs because -these stores track sync progress independently. The errors can be safely ignored as: - -1. The main blockchain state is correctly rolled back -2. Using `--evnode.clear_cache` on restart clears the inconsistent cache -3. The node will resync p2p data from the restored height - -### Timestamp Consistency - -After a restore, if significant real-world time has passed since the backup was created, -you may encounter timestamp validation errors when the node attempts to continue block -production. This occurs because: - -- Reth stores block timestamps based on when blocks were originally created -- After restore, the restored timestamps may be in the past relative to system time -- Block validators may reject new blocks with timestamps earlier than parent blocks - -**Workaround:** In production environments, coordinate restore operations to minimize -time between backup and restore, or ensure the entire network is restored simultaneously. - -## Summary - -This backup/restore workflow enables point-in-time recovery for both reth (MDBX) and -ev-node (Badger) datastores. Key points: - -- **Backup**: Hot backup while nodes are running (no downtime) -- **Restore**: Requires stopping services, restoring volumes, and aligning heights -- **Rollback**: May show p2p store errors that can be safely ignored -- **Production**: Test the full workflow in staging before deploying to production - -The process has been validated to correctly restore state and resume block production -from the backup point, with known limitations around p2p store consistency and timestamp -validation that can be mitigated with proper operational procedures. diff --git a/scripts/reth-backup/backup-lib.sh b/scripts/reth-backup/backup-lib.sh deleted file mode 100644 index c445f1459..000000000 --- a/scripts/reth-backup/backup-lib.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash - -# backup-lib.sh - Abstract execution layer for reth backup operations -# Provides a common interface for local and Docker-based executions. - -# Backend interface that must be implemented: -# - exec_remote Execute a command in the target environment -# - copy_from_remote Copy a file/directory from target to local -# - copy_to_remote Copy a file/directory from local to target -# - cleanup_remote Remove a path in the target environment - -# ============================================================================ -# LOCAL BACKEND -# ============================================================================ - -local_exec_remote() { - bash -c "$1" -} - -local_copy_from_remote() { - local src="$1" - local dst="$2" - cp -a "$src" "$dst" -} - -local_copy_to_remote() { - local src="$1" - local dst="$2" - cp -a "$src" "$dst" -} - -local_cleanup_remote() { - local path="$1" - rm -rf "$path" -} - -local_check_available() { - # Always available - return 0 -} - -# ============================================================================ -# DOCKER BACKEND -# ============================================================================ - -docker_exec_remote() { - local container="$BACKEND_CONTAINER" - docker exec "$container" bash -lc "$1" -} - -docker_copy_from_remote() { - local container="$BACKEND_CONTAINER" - local src="$1" - local dst="$2" - docker cp "${container}:${src}" "$dst" -} - -docker_copy_to_remote() { - local container="$BACKEND_CONTAINER" - local src="$1" - local dst="$2" - docker cp "$src" "${container}:${dst}" -} - -docker_cleanup_remote() { - local container="$BACKEND_CONTAINER" - local path="$1" - docker exec "$container" rm -rf "$path" -} - -docker_check_available() { - if ! command -v docker >/dev/null 2>&1; then - echo "error: docker command not found" >&2 - return 1 - fi - - local container="$BACKEND_CONTAINER" - if [[ -z "$container" ]]; then - echo "error: container name is required for docker mode" >&2 - return 1 - fi - - if ! docker ps --format '{{.Names}}' | grep -q "^${container}$"; then - echo "error: container '$container' is not running" >&2 - return 1 - fi - - return 0 -} - -# ============================================================================ -# BACKEND INITIALIZATION -# ============================================================================ - -# Set the backend mode and initialize function pointers -init_backend() { - local mode="$1" - - case "$mode" in - local) - exec_remote=local_exec_remote - copy_from_remote=local_copy_from_remote - copy_to_remote=local_copy_to_remote - cleanup_remote=local_cleanup_remote - check_backend_available=local_check_available - ;; - docker) - exec_remote=docker_exec_remote - copy_from_remote=docker_copy_from_remote - copy_to_remote=docker_copy_to_remote - cleanup_remote=docker_cleanup_remote - check_backend_available=docker_check_available - ;; - *) - echo "error: unknown backend mode '$mode'" >&2 - echo "supported modes: local, docker" >&2 - return 1 - ;; - esac - - BACKEND_MODE="$mode" - return 0 -} - -# ============================================================================ -# HIGH-LEVEL BACKUP OPERATIONS -# ============================================================================ - -# Verify that a command is available in the target environment -verify_remote_command() { - local cmd="$1" - if ! $exec_remote "command -v '$cmd' >/dev/null 2>&1 || [ -x '$cmd' ]"; then - echo "error: command '$cmd' not found in target environment" >&2 - return 1 - fi - return 0 -} - -# Create a directory in the target environment -create_remote_dir() { - local path="$1" - $exec_remote "mkdir -p '$path'" -} - -# Check if a path exists in the target environment -remote_path_exists() { - local path="$1" - $exec_remote "test -e '$path'" -} - -# Run mdbx_copy in the target environment -run_mdbx_copy() { - local mdbx_copy="$1" - local source_db="$2" - local dest_file="$3" - - echo "Running mdbx_copy..." - $exec_remote "'$mdbx_copy' -c '$source_db' '$dest_file'" -} - -# Query ev-reth for stage checkpoints -query_stage_checkpoints() { - local datadir="$1" - $exec_remote "ev-reth db --datadir '$datadir' list StageCheckpoints --len 20 --json" | sed -n '/^\[/,$p' -} diff --git a/scripts/reth-backup/backup.sh b/scripts/reth-backup/backup.sh deleted file mode 100755 index c254fecb1..000000000 --- a/scripts/reth-backup/backup.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Load the backend library -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/backup-lib.sh" - -usage() { - cat <<'EOF' -Usage: backup.sh [OPTIONS] - -Create a consistent backup of the ev-reth database using mdbx_copy and record -the block height captured in the snapshot. - -Options: - --mode MODE Execution mode: 'local' or 'docker' (default: docker) - --container NAME Docker container name running ev-reth (default: ev-reth) - Only used in docker mode. - --datadir PATH Path to the reth datadir in the target environment - (default docker: /home/reth/eth-home) - (default local: /var/lib/reth) - --mdbx-copy CMD Path to the mdbx_copy binary in the target environment - (default: mdbx_copy; override if you compiled it elsewhere) - --tag LABEL Custom label for the backup directory (default: timestamp) - --keep-remote Leave the temporary snapshot in the target environment - -h, --help Show this help message - -Modes: - local Run backup on the local machine (reth running locally) - docker Run backup on a Docker container (default) - -Requirements: - - mdbx_copy available in the target environment (compile it once if necessary). - - jq installed on the host (used to parse StageCheckpoints JSON). - - For docker mode: Docker access to the container running ev-reth. - - For local mode: Direct filesystem access to reth datadir. - -The destination directory will receive: - //db/mdbx.dat MDBX snapshot - //db/mdbx.lck Empty lock file placeholder - //static_files/... Static files copied from the node - //stage_checkpoints.json - //height.txt Height extracted from StageCheckpoints - -Examples: - # Backup from local reth instance - ./backup.sh --mode local --datadir /var/lib/reth /path/to/backups - - # Backup from Docker container - ./backup.sh --mode docker --container ev-reth /path/to/backups -EOF -} - -require_cmd() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "error: required command '$1' not found in PATH" >&2 - exit 1 - fi -} - -DEST="" -MODE="docker" -CONTAINER="ev-reth" -DATADIR="" -MDBX_COPY="mdbx_copy" -BACKUP_TAG="" -KEEP_REMOTE=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --mode) - MODE="$2" - shift 2 - ;; - --container) - CONTAINER="$2" - shift 2 - ;; - --datadir) - DATADIR="$2" - shift 2 - ;; - --mdbx-copy) - MDBX_COPY="$2" - shift 2 - ;; - --tag) - BACKUP_TAG="$2" - shift 2 - ;; - --keep-remote) - KEEP_REMOTE=1 - shift - ;; - -h|--help) - usage - exit 0 - ;; - --) - shift - break - ;; - -*) - echo "unknown option: $1" >&2 - usage >&2 - exit 1 - ;; - *) - if [[ -z "$DEST" ]]; then - DEST="$1" - shift - else - echo "unexpected argument: $1" >&2 - usage >&2 - exit 1 - fi - ;; - esac -done - -if [[ -z "$DEST" ]]; then - echo "error: destination directory is required" >&2 - usage >&2 - exit 1 -fi - -# Validate and set defaults based on mode -case "$MODE" in - local) - if [[ -z "$DATADIR" ]]; then - DATADIR="/var/lib/reth" - fi - ;; - docker) - if [[ -z "$DATADIR" ]]; then - DATADIR="/home/reth/eth-home" - fi - ;; - *) - echo "error: invalid mode '$MODE'. Use 'local' or 'docker'." >&2 - exit 1 - ;; -esac - -# Initialize the backend -if ! init_backend "$MODE"; then - exit 1 -fi - -# Set container for docker mode -if [[ "$MODE" == "docker" ]]; then - BACKEND_CONTAINER="$CONTAINER" -fi - -# Check backend availability -if ! $check_backend_available; then - exit 1 -fi - -require_cmd jq - -if [[ -z "$BACKUP_TAG" ]]; then - BACKUP_TAG="$(date +'%Y%m%d-%H%M%S')" -fi - -REMOTE_TMP="/tmp/reth-backup-${BACKUP_TAG}" -HOST_DEST="$(mkdir -p "$DEST" && cd "$DEST" && pwd)/${BACKUP_TAG}" - -echo "Mode: $MODE" -echo "Creating backup tag '$BACKUP_TAG' into ${HOST_DEST}" - -# Prepare temporary workspace in target environment -echo "Preparing temporary workspace..." -$exec_remote "rm -rf '$REMOTE_TMP' && mkdir -p '$REMOTE_TMP/db' '$REMOTE_TMP/static_files'" - -# Verify mdbx_copy availability -if ! verify_remote_command "$MDBX_COPY"; then - exit 1 -fi - -echo "Running mdbx_copy in target environment..." -run_mdbx_copy "$MDBX_COPY" "${DATADIR}/db" "$REMOTE_TMP/db/mdbx.dat" -$exec_remote "touch '$REMOTE_TMP/db/mdbx.lck'" - -echo "Copying static_files..." -$exec_remote "if [ -d '${DATADIR}/static_files' ]; then cp -a '${DATADIR}/static_files/.' '$REMOTE_TMP/static_files/' 2>/dev/null || true; fi" - -echo "Querying StageCheckpoints height..." -STAGE_JSON=$(query_stage_checkpoints "$REMOTE_TMP") -HEIGHT=$(echo "$STAGE_JSON" | jq -r '.[] | select(.[0]=="Finish") | .[1].block_number' | tr -d '\r\n') - -if [[ -z "$HEIGHT" || "$HEIGHT" == "null" ]]; then - echo "warning: could not determine height from StageCheckpoints" >&2 -fi - -echo "Copying snapshot to host..." -mkdir -p "$HOST_DEST/db" -$copy_from_remote "${REMOTE_TMP}/db/mdbx.dat" "$HOST_DEST/db/mdbx.dat" -$copy_from_remote "${REMOTE_TMP}/db/mdbx.lck" "$HOST_DEST/db/mdbx.lck" - -if remote_path_exists "${REMOTE_TMP}/static_files"; then - mkdir -p "$HOST_DEST/static_files" - $copy_from_remote "${REMOTE_TMP}/static_files/." "$HOST_DEST/static_files/" || true -fi - -echo "$STAGE_JSON" > "$HOST_DEST/stage_checkpoints.json" -if [[ -n "$HEIGHT" && "$HEIGHT" != "null" ]]; then - echo "$HEIGHT" > "$HOST_DEST/height.txt" - echo "Backup height: $HEIGHT" -else - echo "Height not captured (see stage_checkpoints.json for details)" -fi - -if [[ "$KEEP_REMOTE" -ne 1 ]]; then - echo "Cleaning up temporary files..." - $cleanup_remote "$REMOTE_TMP" -fi - -echo "Backup completed: $HOST_DEST"