Nethermind path state

eth-educators · Aug 10, 2024 · 2d8c7c8 · 2d8c7c8
1 parent 23b6f59
commit 2d8c7c8
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 21 deletions.
diff --git a/ethd b/ethd
@@ -454,15 +454,17 @@ check_disk_space() {
       echo "It should currently be auto-pruning, check logs with \"$__me logs -f --tail 500 execution | grep \
 Full\". Free space:"
     else
-      echo "If the below reads above ${__safe_prune} GiB free, prune it with \"$__me prune-nethermind\""
+      echo "You might be able to resync Nethermind for path storage if you haven't done so already, with \"$__me resync-execution\"."
+      echo "Also verify the amount of space your consensus layer client takes, and resync it as needed."
     fi
     echo
     __display_docker_dir
     __display_docker_volumes
   elif [[ "${value}" =~ "geth.yml" ]] && [[ "${__free_space}" -lt 104857600 ]]; then
     echo
     echo "You are running Geth and have less than 100 GiB of free disk space."
-    echo "You may resync from scratch to use PBSS and slow on-disk DB growth, with \"$__me resync-execution\"."
+    echo "You may resync from scratch to use PBSS and slow on-disk DB growth, if you haven't done so already, with \"$__me resync-execution\"."
+    echo "Also verify the amount of space your consensus layer client takes, and resync it as needed."
     echo
     __display_docker_dir
     __display_docker_volumes
@@ -758,13 +760,13 @@ delete_erigon() {
     return 0
   fi
 
-  if [ -z "$(dodocker volume ls -q -f "name=erigon-el-data")" ]; then # No Erigon volume
+  if [ -z "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]erigon-el-data")" ]; then # No Erigon volume
     return 0
   fi
 
 # https://discord.com/channels/687972960811745322/886550095749873666/1068994749480054866
 # Detect Erigon v3 by files in snapshots/history
-  if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=erigon-el-data")":"/var/lib/erigon" \
+  if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]name=erigon-el-data")":"/var/lib/erigon" \
       alpine:3 ls /var/lib/erigon/snapshots/history)" ]; then
     return 0 # Files in snapshots/history means it's v3
   fi
@@ -781,7 +783,7 @@ delete_erigon() {
 
   echo "Stopping Erigon container"
   docompose stop execution && docompose rm -f execution
-  dodocker volume rm "$(dodocker volume ls -q -f "name=erigon-el-data")"
+  dodocker volume rm "$(dodocker volume ls -q -f "name=name=$(basename "$(realpath .)")[_-]erigon-el-data")"
   echo
   echo "Erigon stopped and database deleted."
   echo
@@ -1706,6 +1708,12 @@ prune-nethermind() {
   var="NETWORK"
   NETWORK=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true)
 
+  if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]nm-eth1-data")":"/var/lib/nethermind" \
+      alpine:3 ls "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState")" ]; then
+    echo "This Nethermind uses path-based storage. Manual pruning is not required."
+    return 0
+  fi
+
   if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then
     __min_free=262144000
     __min_gib=250
@@ -1766,6 +1774,7 @@ prune-nethermind() {
         echo "Nethermind should auto-prune below ${threshold} GiB free. Check logs with \"$__me logs -f --tail 500 \
 execution | grep Full\" to see whether it is."
       fi
+      echo "Instead of pruning, consider resyncing Nethermind to use path-based storage, with \"$__me resync-execution\"."
       read -rp "WARNING - this will prune Nethermind's database in the background. Do you wish to continue? (No/Yes) " yn
       case $yn in
         [Yy][Ee][Ss] ) break;;

diff --git a/nethermind/docker-entrypoint.sh b/nethermind/docker-entrypoint.sh
@@ -61,25 +61,31 @@ if [ "${ARCHIVE_NODE}" = "true" ]; then
   echo "Nethermind archive node without pruning"
   __prune="--Sync.DownloadBodiesInFastSync=false --Sync.DownloadReceiptsInFastSync=false --Sync.FastSync=false --Sync.SnapSync=false --Sync.FastBlocks=false --Pruning.Mode=None --Sync.PivotNumber=0"
 else
-  __parallel=$(($(nproc)/4))
-  if [ "${__parallel}" -lt 2 ]; then
-    __parallel=2
-  fi
-  __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}"
-  if [ "${AUTOPRUNE_NM}" = true ]; then
-    __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace"
-    if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then
-      __prune="${__prune} --Pruning.FullPruningThresholdMb=375810"
-    else
-      __prune="${__prune} --Pruning.FullPruningThresholdMb=51200"
+  # Fresh DB or already path based storage
+  if [[ ! -d "/var/lib/nethermind/nethermind_db/${NETWORK}" || -d "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState" ]]; then
+    __prune=""
+  else # Hash, set pruning parameters
+    __parallel=$(($(nproc)/4))
+    if [ "${__parallel}" -lt 2 ]; then
+      __parallel=2
+    fi
+    __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}"
+    if [ "${AUTOPRUNE_NM}" = true ]; then
+      __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace"
+      if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then
+        __prune="${__prune} --Pruning.FullPruningThresholdMb=375810"
+      else
+        __prune="${__prune} --Pruning.FullPruningThresholdMb=51200"
+      fi
+    fi
+    if [ "${__memtotal}" -ge 30 ]; then
+      __prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384"
     fi
   fi
-  if [ "${__memtotal}" -ge 30 ]; then
-# Investigating instability
-    __prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath"
+  if [ -n "${__prune}" ]; then
+    echo "Using pruning parameters:"
+    echo "${__prune}"
   fi
-  echo "Using pruning parameters:"
-  echo "${__prune}"
 fi
 
 # Word splitting is desired for the command line parameters