Merge back to rampup

scripts/archive/split_precomputed_log.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 precomputed-log-file [output-folder]"
+  exit 1
+fi
+
+
+
+ARCHIVE_URI=${ARCHIVE_URI:-postgres://postgres@localhost:5432/archive}
+PRECOMPUTED_LOG_FILE=$1
+
+while IFS= read -r line; do
+	LEDGER_HASH=$(echo $line | jq -r '.protocol_state.body.blockchain_state.staged_ledger_hash.non_snark.ledger_hash')
+	FILE_NAME=$(psql $ARCHIVE_URI -t -c "SELECT 'mainnet-' || height || '-' ||state_hash || '.json' FROM blocks WHERE ledger_hash = '$LEDGER_HASH'")
+	echo  $line > $FILE_NAME 
+done < $PRECOMPUTED_LOG_FILE
+
+

scripts/mina-local-network/mina-local-network.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# set -x
+#set -x
 
 # Exit script when commands fail
 set -e
@@ -34,6 +34,7 @@ ZKAPP_TRANSACTIONS=false
 RESET=false
 UPDATE_GENESIS_TIMESTAMP=false
 PROOF_LEVEL="full"
+LOG_PRECOMPUTED_BLOCKS=false
 
 SNARK_WORKER_FEE=0.001
 TRANSACTION_FREQUENCY=10 # in seconds
@@ -116,6 +117,8 @@ help() {
   echo "                                         |   Default: ${TRANSACTION_FREQUENCY}"
   echo "-sf  |--snark-worker-fee <#>             | SNARK Worker fee"
   echo "                                         |   Default: ${SNARK_WORKER_FEE}"
+  echo "-lp  |--log-precomputed-blocks     		 | Log precomputed blocks"
+  echo "                                  		 |   Default: ${LOG_PRECOMPUTED_BLOCKS}"
   echo "-pl  |--proof-level <proof-level>        | Proof level (currently consumed by SNARK Workers only)"
   echo "                                         |   Default: ${PROOF_LEVEL}"
   echo "-r   |--reset                            | Whether to reset the Mina Local Network storage file-system (presence of argument)"
@@ -170,6 +173,8 @@ exec-daemon() {
     -log-json \
     -log-level ${LOG_LEVEL} \
     -file-log-level ${FILE_LOG_LEVEL} \
+    -precomputed-blocks-file ${FOLDER}/precomputed_blocks.log \
+    -log-precomputed-blocks ${LOG_PRECOMPUTED_BLOCKS} \
     $@
 }
 
@@ -337,6 +342,7 @@ while [[ "$#" -gt 0 ]]; do
     SNARK_WORKER_FEE="${2}"
     shift
     ;;
+  -lp | --log-precomputed-blocks) LOG_PRECOMPUTED_BLOCKS=true ;;
   -pl | --proof-level)
     PROOF_LEVEL="${2}"
     shift

src/app/ledger_export_bench/README.md

@@ -0,0 +1,23 @@
+Ledger export benckmark
+=======================
+
+This package provides a couple of simple benchmarks for the most
+resource-consuming parts of the ledger export process, namely:
+* serialising and de-serialising a large runtime config;
+* converting accounts between runtime config format and ledger format.
+
+These benchmarks depend on a sufficiently large runtime config file
+supplied from outside, because creating one within a benchmark,
+while useful, is too involved, as it basically requires connecting
+to a live network. Besides, such a benchmark would be unreliable,
+as its results wouldn't be reproducible.
+
+Sadly, a runtime config containing realistic mainnet data is too
+large to store it in git (~400MB), so users of the benchmark need to
+provide it themselves. The file's path should be given by environment
+variable `RUNTIME_CONFIG` provide to the benchmark process. If that
+variable is absent, the benchmark will not work.
+
+In order to run the benchmark, type:
+
+    $ RUNTIME_CONFIG=$HOME/mainnet.json dune exec src/app/ledger_export_bench/ledger_export_benchmark.exe

src/app/ledger_export_bench/dune

@@ -0,0 +1,15 @@
+(executable
+ (name ledger_export_benchmark)
+ (libraries
+   base
+   core
+   core_bench
+   core_kernel
+   yojson
+   ; mina libraries
+   runtime_config)
+ ; the -w list here should be the same as in src/dune
+ (flags -short-paths -g -w @a-4-29-40-41-42-44-45-48-58-59-60)
+ (instrumentation (backend bisect_ppx))
+ (preprocess (pps ppx_version ppx_jane))
+ (modes native))

src/app/ledger_export_bench/ledger_export_benchmark.ml

@@ -0,0 +1,34 @@
+open Core
+open Core_bench
+
+let load_daemon_cfg filename () =
+  let json = Yojson.Safe.from_file filename in
+  match Runtime_config.of_yojson json with
+  | Ok cfg ->
+      cfg
+  | Error err ->
+      raise (Failure err)
+
+let serialize cfg () = Runtime_config.to_yojson cfg |> Yojson.Safe.to_string
+
+let convert accounts () =
+  Runtime_config.(map_results ~f:Accounts.Single.of_account accounts)
+
+let () =
+  let runtime_config = Sys.getenv_exn "RUNTIME_CONFIG" in
+  let cfg = load_daemon_cfg runtime_config () in
+  let accounts =
+    match cfg.ledger with
+    | None | Some { base = Named _; _ } | Some { base = Hash _; _ } ->
+        []
+    | Some { base = Accounts accs; _ } ->
+        List.map ~f:Runtime_config.Accounts.Single.to_account accs
+  in
+  Command.run
+    (Bench.make_command
+       [ Bench.Test.create ~name:"parse_runtime_config"
+           (load_daemon_cfg runtime_config)
+       ; Bench.Test.create ~name:"serialize_runtime_config" (serialize cfg)
+       ; Bench.Test.create ~name:"convert_accounts_for_config"
+           (convert accounts)
+       ] )

src/lib/genesis_ledger_helper/lib/genesis_ledger_helper_lib.ml

@@ -458,13 +458,14 @@ let make_constraint_constants
       ( match config.fork with
       | None ->
           default.fork
-      | Some { previous_state_hash; previous_length; genesis_slot } ->
+      | Some { previous_state_hash; previous_length; previous_global_slot } ->
           Some
             { previous_state_hash =
                 State_hash.of_base58_check_exn previous_state_hash
             ; previous_length = Mina_numbers.Length.of_int previous_length
             ; genesis_slot =
-                Mina_numbers.Global_slot_since_genesis.of_int genesis_slot
+                Mina_numbers.Global_slot_since_genesis.of_int
+                  previous_global_slot
             } )
   }
 
@@ -497,7 +498,7 @@ let runtime_config_of_constraint_constants
           { Runtime_config.Fork_config.previous_state_hash =
               State_hash.to_base58_check previous_state_hash
           ; previous_length = Mina_numbers.Length.to_int previous_length
-          ; genesis_slot =
+          ; previous_global_slot =
               Mina_numbers.Global_slot_since_genesis.to_int genesis_slot
           } )
   }

src/lib/mina_graphql/dune

@@ -60,6 +60,7 @@
    graphql_lib
    block_time
    currency
+   merkle_ledger
    mina_lib
    mina_commands
    mina_state

src/lib/mina_graphql/mina_graphql.ml

@@ -2304,16 +2304,7 @@ module Queries = struct
         |> Runtime_config.to_yojson |> Yojson.Safe.to_basic )
 
   let fork_config =
-    let rec map_results ~f = function
-      | [] ->
-          Result.return []
-      | r :: rs ->
-          let open Result.Let_syntax in
-          let%bind r' = f r in
-          let%map rs = map_results ~f rs in
-          r' :: rs
-    in
-    field "fork_config"
+    io_field "fork_config"
       ~doc:
         "The runtime configuration for a blockchain fork intended to be a \
          continuation of the current one."
@@ -2322,63 +2313,81 @@ module Queries = struct
       ~resolve:(fun { ctx = mina; _ } () ->
         match Mina_lib.best_tip mina with
         | `Bootstrapping ->
-            `Assoc [ ("error", `String "Daemon is bootstrapping") ]
-        | `Active best_tip -> (
+            Deferred.Result.fail "Daemon is bootstrapping"
+        | `Active best_tip ->
+            let open Deferred.Result.Let_syntax in
             let block = Transition_frontier.Breadcrumb.(block best_tip) in
+            let blockchain_length = Mina_block.blockchain_length block in
             let global_slot =
-              Mina_block.blockchain_length block |> Unsigned.UInt32.to_int
+              Mina_block.consensus_state block
+              |> Consensus.Data.Consensus_state.curr_global_slot
             in
-            let accounts_or_error =
+            let staged_ledger =
               Transition_frontier.Breadcrumb.staged_ledger best_tip
               |> Staged_ledger.ledger
-              |> Ledger.foldi ~init:[] ~f:(fun _ accum act -> act :: accum)
-              |> map_results
-                   ~f:Runtime_config.Json_layout.Accounts.Single.of_account
             in
             let protocol_state =
               Transition_frontier.Breadcrumb.protocol_state best_tip
             in
-            match accounts_or_error with
-            | Error e ->
-                `Assoc [ ("error", `String e) ]
-            | Ok accounts ->
-                let runtime_config = Mina_lib.runtime_config mina in
-                let ledger = Option.value_exn runtime_config.ledger in
-                let previous_length =
-                  let open Option.Let_syntax in
-                  let%bind proof = runtime_config.proof in
-                  let%map fork = proof.fork in
-                  fork.previous_length + global_slot
-                in
-                let fork =
-                  Runtime_config.Fork_config.
-                    { previous_state_hash =
-                        State_hash.to_base58_check
-                          protocol_state.previous_state_hash
-                    ; previous_length =
-                        Option.value ~default:global_slot previous_length
-                    ; genesis_slot = global_slot
-                    }
-                in
-                let update =
-                  Runtime_config.make
-                  (* add_genesis_winner must be set to false, because this
-                     config effectively creates a continuation of the current
-                     blockchain state and therefore the genesis ledger already
-                     contains the winner of the previous block. No need to
-                     artificially add it. In fact, it wouldn't work at all,
-                     because the new node would try to create this account at
-                     startup, even though it already exists, leading to an error.*)
-                    ~ledger:
-                      { ledger with
-                        base = Accounts accounts
-                      ; add_genesis_winner = Some false
-                      }
-                    ~proof:(Runtime_config.Proof_keys.make ~fork ())
-                    ()
-                in
-                let new_config = Runtime_config.combine runtime_config update in
-                Runtime_config.to_yojson new_config |> Yojson.Safe.to_basic ) )
+            let consensus =
+              Mina_state.Protocol_state.consensus_state protocol_state
+            in
+            let staking_epoch =
+              Consensus.Proof_of_stake.Data.Consensus_state.staking_epoch_data
+                consensus
+            in
+            let next_epoch =
+              Consensus.Proof_of_stake.Data.Consensus_state.next_epoch_data
+                consensus
+            in
+            let staking_epoch_seed =
+              Mina_base.Epoch_seed.to_base58_check
+                staking_epoch.Mina_base.Epoch_data.Poly.seed
+            in
+            let next_epoch_seed =
+              Mina_base.Epoch_seed.to_base58_check
+                next_epoch.Mina_base.Epoch_data.Poly.seed
+            in
+            let runtime_config = Mina_lib.runtime_config mina in
+            let%bind staking_ledger =
+              match Mina_lib.staking_ledger mina with
+              | None ->
+                  Deferred.Result.fail "Staking ledger is not initialized."
+              | Some (Genesis_epoch_ledger l) ->
+                  return (Ledger.Any_ledger.cast (module Ledger) l)
+              | Some (Ledger_db l) ->
+                  return (Ledger.Any_ledger.cast (module Ledger.Db) l)
+            in
+            assert (
+              Mina_base.Ledger_hash.equal
+                (Ledger.Any_ledger.M.merkle_root staking_ledger)
+                staking_epoch.ledger.hash ) ;
+            let%bind next_epoch_ledger =
+              match Mina_lib.next_epoch_ledger mina with
+              | None ->
+                  Deferred.Result.fail "Next epoch ledger is not initialized."
+              | Some `Notfinalized ->
+                  return None
+              | Some (`Finalized (Genesis_epoch_ledger l)) ->
+                  return (Some (Ledger.Any_ledger.cast (module Ledger) l))
+              | Some (`Finalized (Ledger_db l)) ->
+                  return (Some (Ledger.Any_ledger.cast (module Ledger.Db) l))
+            in
+            let protocol_state_hash =
+              Transition_frontier.Breadcrumb.state_hash best_tip
+            in
+            Option.iter next_epoch_ledger ~f:(fun ledger ->
+                assert (
+                  Mina_base.Ledger_hash.equal
+                    (Ledger.Any_ledger.M.merkle_root ledger)
+                    next_epoch.ledger.hash ) ) ;
+            let%map new_config =
+              Runtime_config.make_fork_config ~staged_ledger ~global_slot
+                ~staking_ledger ~staking_epoch_seed ~next_epoch_ledger
+                ~next_epoch_seed ~blockchain_length ~protocol_state_hash
+                runtime_config
+            in
+            Runtime_config.to_yojson new_config |> Yojson.Safe.to_basic )
 
   let thread_graph =
     field "threadGraph"

src/lib/runtime_config/dune

@@ -3,6 +3,8 @@
  (public_name mina_runtime_config)
  (libraries
    ;; opam libraries
+   async
+   async_kernel
    core_kernel
    bin_prot.shape
    base.caml
@@ -16,17 +18,19 @@
    data_hash_lib
    kimchi_backend.pasta
    kimchi_backend.pasta.basic
+   merkle_ledger
    mina_base
    mina_base.import
    mina_numbers
-   signature_lib
    ppx_dhall_type
    snark_params
    unsigned_extended
    pickles
    pickles.backend
    pickles_types
    with_hash
+   signature_lib
+   staged_ledger
    )
  (instrumentation (backend bisect_ppx))
  (preprocess (pps ppx_custom_printf ppx_sexp_conv ppx_let ppx_deriving_yojson