From aa6ea7d154eed46f0e31b61b73a510d89bcd1914 Mon Sep 17 00:00:00 2001
From: juan518munoz <62400508+juan518munoz@users.noreply.github.com>
Date: Mon, 7 Oct 2024 14:54:52 -0300
Subject: [PATCH] feat: Da eigen implementation docs & backup scripts (#289)

* initial commit

* add more steps

* add backup and restore ecosystem scripts

* remove unnecessary step

* improve docs

* fix docs

* fix the fix docs

* add extra step

* fix restore path

* simplify restoration note

* more docs

* fix paths in backup restoration

* fix whitespace

* replacement fixes

* moved holesky rpc url to env var
---
 .gitignore             |   3 ++
 backup-ecosystem.sh    |  51 ++++++++++++++++++
 eigenda-integration.md | 118 +++++++++++++++++++++++++++++++++++++++++
 restore-ecosystem.sh   | 100 ++++++++++++++++++++++++++++++++++
 4 files changed, 272 insertions(+)
 create mode 100755 backup-ecosystem.sh
 create mode 100755 restore-ecosystem.sh

diff --git a/.gitignore b/.gitignore
index c3de7a2df84d..f95274b0e364 100644
--- a/.gitignore
+++ b/.gitignore
@@ -118,3 +118,6 @@ configs/*
 era-observability/
 core/tests/ts-integration/deployments-zk
 transactions/
+
+# Ecosystem backups
+ecosystem_backups/
diff --git a/backup-ecosystem.sh b/backup-ecosystem.sh
new file mode 100755
index 000000000000..dbdf82e3a383
--- /dev/null
+++ b/backup-ecosystem.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Check if the ecosystem name was provided as an argument
+if [ -z "$1" ]; then
+  echo "Usage: ./backup-ecosystem ECOSYSTEM_NAME"
+  exit 1
+fi
+
+# Store the first argument as ECOSYSTEM_NAME
+ECOSYSTEM_NAME=$1
+
+# Prompt for the Postgres password and store it in PGPASSWORD
+read -sp "Enter Postgres password: " PGPASSWORD
+export PGPASSWORD
+
+# Path to the secrets.yaml file
+SECRETS_FILE="./chains/${ECOSYSTEM_NAME}/configs/secrets.yaml"
+
+# Check if the secrets.yaml file exists
+if [ ! -f "$SECRETS_FILE" ]; then
+  echo "Error: $SECRETS_FILE does not exist."
+  exit 1
+fi
+
+# Extract server_url and prover_url from the secrets.yaml file
+SERVER_DB_NAME=$(grep 'server_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}')
+PROVER_DB_NAME=$(grep 'prover_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}')
+
+# Export the database names
+echo "Extracted SERVER_DB_NAME: $SERVER_DB_NAME"
+echo "Extracted PROVER_DB_NAME: $PROVER_DB_NAME"
+
+# Create backup directory
+mkdir -p "./ecosystem_backups/${ECOSYSTEM_NAME}"
+
+# Run pg_dump commands
+echo "Running pg_dump for $SERVER_DB_NAME..."
+pg_dump -U postgres -h localhost "$SERVER_DB_NAME" > "ecosystem_backups/${ECOSYSTEM_NAME}/${SERVER_DB_NAME}_backup.sql"
+echo "Running pg_dump for $PROVER_DB_NAME..."
+pg_dump -U postgres -h localhost "$PROVER_DB_NAME" > "ecosystem_backups/${ECOSYSTEM_NAME}/${PROVER_DB_NAME}_backup.sql"
+
+# Unset the PGPASSWORD variable for security
+unset PGPASSWORD
+
+# Copy the chain configuration files
+cp -r "./chains/${ECOSYSTEM_NAME}" "./ecosystem_backups/${ECOSYSTEM_NAME}/"
+
+# Copy the configs directory
+cp -r "./configs" "./ecosystem_backups/${ECOSYSTEM_NAME}/"
+
+echo "Backup completed."
diff --git a/eigenda-integration.md b/eigenda-integration.md
index 4d02fa6a824d..638719e2b4da 100644
--- a/eigenda-integration.md
+++ b/eigenda-integration.md
@@ -71,3 +71,121 @@ zk_supervisor test integration --chain eigen_da
 ### Metrics
 
 Access Grafana at [http://localhost:3000/](http://localhost:3000/), go to dashboards and select `EigenDA`.
+
+## Holesky Setup
+
+### Modify localhost chain id number
+
+Modify line 32 in `zk_toolbox/crates/types/src/l1_network.rs`:
+
+```rs
+L1Network::Localhost => 17000,
+```
+
+Then recompile the zk toolbox:
+
+```bash
+./bin/zkt
+```
+
+### Used wallets
+
+Modify `etc/env/file_based/wallets.yaml` and `configs/wallets.yaml` with the following wallets:
+
+```yaml
+# Use your own holesky wallets, be sure they have enough funds
+```
+
+> ⚠️ Some steps distribute ~5000ETH to some wallets, modify `AMOUNT_FOR_DISTRIBUTION_TO_WALLETS` to a lower value if needed.
+
+### EigenProxy RPC
+
+Get `EIGEN_SIGNER_PK` from 1password and set it as an `env` var:
+
+```bash
+export EIGEN_SIGNER_PK=<VALUE_HERE>
+export HOLESKY_RPC_URL=<VALUE_HERE>
+```
+
+Modify `docker-compose.yml` to use holesky RPCs:
+
+```rust
+  eigenda-proxy:
+    image: ghcr.io/layr-labs/eigenda-proxy
+    environment:
+      - EIGEN_SIGNER_PK=$EIGEN_SIGNER_PK
+      - HOLESKY_RPC_URL=$HOLESKY_RPC_URL
+    ports:
+      - "4242:4242"
+    command: ./eigenda-proxy --addr 0.0.0.0 --port 4242 --eigenda-disperser-rpc disperser-holesky.eigenda.xyz:443 --eigenda-signer-private-key-hex $EIGEN_SIGNER_PK --eigenda-eth-rpc $HOLESKY_RPC_URL --eigenda-svc-manager-addr 0xD4A7E1Bd8015057293f0D0A557088c286942e84b --eigenda-eth-confirmation-depth 0
+```
+
+### Create and initialize the ecosystem
+
+(be sure to have postgres container running on the background)
+
+```bash
+zk_inception chain create \
+          --chain-name holesky_eigen_da \
+          --chain-id 114411 \
+          --prover-mode no-proofs \
+          --wallet-creation localhost \
+          --l1-batch-commit-data-generator-mode validium \
+          --base-token-address 0x0000000000000000000000000000000000000001 \
+          --base-token-price-nominator 1 \
+          --base-token-price-denominator 1 \
+          --set-as-default false
+
+zk_inception ecosystem init \
+          --deploy-paymaster true \
+          --deploy-erc20 true \
+          --deploy-ecosystem true \
+          --l1-rpc-url $HOLESKY_RPC_URL \
+          --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
+          --server-db-name=zksync_server_holesky_eigen_da \
+          --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
+          --prover-db-name=zksync_prover_holesky_eigen_da \
+          --chain holesky_eigen_da \
+          --verbose
+```
+
+### Start the server
+
+```bash
+zk_inception server --chain holesky_eigen_da
+```
+
+## Backup and restoration
+
+It's possible to run the zk stack on one computer, and then migrate it to another, this is specially useful for holesky testing.
+
+### Backup
+
+Suppose that you want to make a backup of `holesky_eigen_da` ecosystem, you only need to run:
+
+```bash
+./backup-ecosystem.sh holesky_eigen_da
+```
+
+This will generate a directory inside of `ecosystem_backups` with the name `holesky_eigen_da`.
+
+### Restoration
+
+1. Move the `ecoystem_backups/holesky_eigen_da` directory to the other computer, it should be placed in the root of the project.
+
+2. Restore the ecosystem with:
+
+```bash
+./restore-ecosystem.sh holesky_eigen_da
+```
+
+Note that:
+
+- The `postgres` container has to be running.
+- The `chain_id` can't be already in use.
+- If you are restoring a local ecosystem, you have to use the same `reth` container as before.
+- If no ecosystem has been `init`ialized on this computer before, run this command:
+
+```bash
+git submodule update --init --recursive && zk_supervisor contracts
+```
diff --git a/restore-ecosystem.sh b/restore-ecosystem.sh
new file mode 100755
index 000000000000..7fbff3e9802b
--- /dev/null
+++ b/restore-ecosystem.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Check if the ecosystem name was provided as an argument
+if [ -z "$1" ]; then
+  echo "Usage: ./restore-ecosystem ECOSYSTEM_NAME"
+  exit 1
+fi
+
+# Store the first argument as ECOSYSTEM_NAME
+ECOSYSTEM_NAME=$1
+
+# Prompt for the Postgres password and store it in PGPASSWORD
+read -sp "Enter Postgres password: " PGPASSWORD
+export PGPASSWORD
+
+# Check if the chain directory exists
+CHAIN_PATH="./chains/${ECOSYSTEM_NAME}"
+BACKUP_PATH="./ecosystem_backups/${ECOSYSTEM_NAME}"
+
+# Check if the backup directory exists
+if [ ! -d "$BACKUP_PATH" ]; then
+  echo "Error: Backup not found at $BACKUP_PATH."
+  exit 1
+fi
+
+# Check if the postgres is running
+if ! docker ps --filter "name=postgres" --filter "status=running" | grep "postgres" > /dev/null; then
+  echo "Error: postgres not running, set it up first with 'zk_inception containers'."
+  exit 1
+fi
+
+# Fix backup files $ZKSYNC_HOME paths
+find_and_replace() {
+  local target_file=$1
+
+  sed -i '' -e "s|db_path:.*zksync-era/\./|db_path: $(pwd)/./|g" "$target_file"
+  sed -i '' -e "s|state_keeper_db_path:.*zksync-era/\./|state_keeper_db_path: $(pwd)/./|g" "$target_file"
+  sed -i '' -e "s|path:.*zksync-era/\./|path: $(pwd)/./|g" "$target_file"
+  sed -i '' -e "s|configs:.*zksync-era/\./|configs: $(pwd)/./|g" "$target_file"
+}
+
+# Array of specific files to modify
+files=("$BACKUP_PATH/$ECOSYSTEM_NAME/configs/general.yaml" "$BACKUP_PATH/$ECOSYSTEM_NAME/ZkStack.yaml")
+
+# Loop over the files and perform the find and replace
+for file in "${files[@]}"; do
+  if [ -f "$file" ]; then
+    find_and_replace "$file"
+  else
+    # Exit with error code
+    echo "ERROR: backup file $file does not exist."
+    exit 1
+  fi
+done
+
+# Copy the ecosystem backup folder to the chains folder, replacing any existing files
+echo "Copying backup files to $CHAIN_PATH..."
+cp -r "$BACKUP_PATH/$ECOSYSTEM_NAME" "$CHAIN_PATH"
+
+# Copy the configs folder in the backup to the configs folder in the root of the project
+# TODO: it may be suitable to warn the user about overwriting the existing configs
+# and ask for confirmation before proceeding
+echo "Copying configs folder from backup..."
+cp -r "$BACKUP_PATH/configs" "./"
+
+# Path to the secrets.yaml file
+SECRETS_FILE="$CHAIN_PATH/configs/secrets.yaml"
+
+# Check if the secrets.yaml file exists
+if [ ! -f "$SECRETS_FILE" ]; then
+  echo "Error: $SECRETS_FILE does not exist."
+  exit 1
+fi
+
+# Extract server_url and prover_url from the secrets.yaml file
+SERVER_DB_NAME=$(grep 'server_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}')
+PROVER_DB_NAME=$(grep 'prover_url' "$SECRETS_FILE" | awk -F'/' '{print $NF}')
+
+# Show the extracted database names
+echo "Extracted SERVER_DB_NAME: $SERVER_DB_NAME"
+echo "Extracted PROVER_DB_NAME: $PROVER_DB_NAME"
+
+# Create and restore the server database
+echo "Creating database $SERVER_DB_NAME..."
+createdb -U postgres -h localhost "$SERVER_DB_NAME"
+
+echo "Restoring $SERVER_DB_NAME from backup..."
+psql -U postgres -h localhost -d "$SERVER_DB_NAME" -f "$BACKUP_PATH/${SERVER_DB_NAME}_backup.sql"
+
+# Create and restore the prover database
+echo "Creating database $PROVER_DB_NAME..."
+createdb -U postgres -h localhost "$PROVER_DB_NAME"
+
+echo "Restoring $PROVER_DB_NAME from backup..."
+psql -U postgres -h localhost -d "$PROVER_DB_NAME" -f "$BACKUP_PATH/${PROVER_DB_NAME}_backup.sql"
+
+# Unset the PGPASSWORD variable for security
+unset PGPASSWORD
+
+echo "Restore completed."