From 661869dc740f83fdb8139f5fb4a47721397a7c8d Mon Sep 17 00:00:00 2001 From: Anthony Anderson Date: Thu, 5 May 2022 11:50:11 -0400 Subject: [PATCH 1/3] ara/deterministic-timing-rework new and improved better logs be more aggressive and check block timer on set better logs moar logs sync local to remote Pick most severe AvgBlockTime config fixes read_timer instead tweaks to logic whoops better time selection for workcation attempt at making different block history ugh --- .devcontainer/devcontainer.json | 22 ++ .devcontainer/remote.Dockerfile | 24 ++ .gitignore | 1 + .vscode/configurationCache.log | 1 + .vscode/dryrun.log | 26 ++ .vscode/settings.json | 3 + .vscode/targets.log | 448 +++++++++++++++++++++++++++ config/test.config | 9 +- config/test_short.config | 57 ++++ rebar.config | 12 +- run.sh | 4 + src/handlers/miner_hbbft_handler.erl | 6 +- src/miner.erl | 274 ++++++++++++---- 13 files changed, 813 insertions(+), 74 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/remote.Dockerfile create mode 100644 .vscode/configurationCache.log create mode 100644 .vscode/dryrun.log create mode 100644 .vscode/settings.json create mode 100644 .vscode/targets.log create mode 100644 config/test_short.config diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..be9fa531c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,22 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.159.0/containers/javascript-node +{ + "name": "Erlang", + "build": { + "dockerfile": "remote.Dockerfile", + "args": { "VARIANT": "24.3.4.0-alpine" } + }, + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": "/bin/sh" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "pgourlain.erlang" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [3000], +} diff --git a/.devcontainer/remote.Dockerfile b/.devcontainer/remote.Dockerfile new file mode 100644 index 000000000..40a32387e --- /dev/null +++ b/.devcontainer/remote.Dockerfile @@ -0,0 +1,24 @@ +# [Choice] alpine,... +ARG VARIANT="alpine" +FROM erlang:${VARIANT} as deps-compiler + +ENV DEBIAN_FRONTEND=noninteractive + +# Setup rebar diagnostics +ARG REBAR_DIAGNOSTIC=0 +ENV DIAGNOSTIC=${REBAR_DIAGNOSTIC} + +ENV CC=gcc CXX=g++ CFLAGS="-U__sun__" \ + ERLANG_ROCKSDB_OPTS="-DWITH_BUNDLE_SNAPPY=ON -DWITH_BUNDLE_LZ4=ON" \ + ERL_COMPILER_OPTIONS="[deterministic]" \ + PATH="/root/.cargo/bin:$PATH" \ + RUSTFLAGS="-C target-feature=-crt-static" + +# Install dependencies to build +RUN apk add --no-cache --update \ + autoconf automake bison build-base bzip2 cmake curl \ + dbus-dev flex git gmp-dev libsodium-dev libtool linux-headers lz4 \ + openssl-dev pkgconfig protoc sed tar wget bash parallel + +# Install Rust toolchain +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y diff --git a/.gitignore b/.gitignore index b901866a3..5b55d9f2f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ data src/pb .repos/ *.deb +*.log # make build output for external deps external/ diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log new file mode 100644 index 000000000..5232aabd5 --- /dev/null +++ b/.vscode/configurationCache.log @@ -0,0 +1 @@ +{"buildTargets":[".PHONY","all","aws","ci","clean","clean_external_svcs","clean_grpc","compile","cover","deps","devrel","devrelease","docker","external_svcs","grpc","release","seed","test","typecheck","validator"],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log new file mode 100644 index 000000000..9c3941e17 --- /dev/null +++ b/.vscode/dryrun.log @@ -0,0 +1,26 @@ +make --dry-run --always-make --keep-going --print-directory +make: Entering directory `/Users/anthonyra/Documents/GitHub/miner' +./rebar3 get-deps +REBAR_CONFIG="config/grpc_client_gen_local.config" ./rebar3 grpc gen +REBAR_CONFIG="config/grpc_client_gen.config" ./rebar3 grpc gen +/Library/Developer/CommandLineTools/usr/bin/make external_svcs + +make[1]: Entering directory `/Users/anthonyra/Documents/GitHub/miner' +echo "cloning external dependency projects" +echo "--- gateway-rs ---" +git clone --quiet https://github.com/helium/gateway-rs ./external/gateway-rs 2>/dev/null || true +(cd external/gateway-rs && git fetch && git checkout "v1.0.0-alpha.30" 2>/dev/null) +(cd ./external/gateway-rs && cargo build --release) +mkdir -p ./priv/gateway_rs +mv ./external/gateway-rs/target/release/helium_gateway ./priv/gateway_rs/ 2>/dev/null || true +cp ./external/gateway-rs/config/default.toml ./priv/gateway_rs/default.toml +echo "--- gwmp-mux ---" +git clone --quiet --depth 1 --branch "v0.9.4" https://github.com/helium/gwmp-mux ./external/gwmp-mux 2>/dev/null || true +(cd ./external/gwmp-mux && cargo build --release) +mkdir -p ./priv/gwmp_mux +mv ./external/gwmp-mux/target/release/gwmp-mux ./priv/gwmp_mux/ 2>/dev/null || true +make[1]: Leaving directory `/Users/anthonyra/Documents/GitHub/miner' + +./rebar3 compile +make: Leaving directory `/Users/anthonyra/Documents/GitHub/miner' + diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..65e1ec078 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.extensionOutputFolder": "./.vscode" +} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log new file mode 100644 index 000000000..081cf7cf3 --- /dev/null +++ b/.vscode/targets.log @@ -0,0 +1,448 @@ +make all --print-data-base --no-builtin-variables --no-builtin-rules --question +# GNU Make 3.81 +# Copyright (C) 2006 Free Software Foundation, Inc. +# This is free software; see the source for copying conditions. +# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +# This program built for i386-apple-darwin11.3.0 + + +# Make data base, printed on Fri Aug 5 15:11:09 2022 + +# Variables + +# automatic +/dev/null || true +endef +# automatic +@D = $(patsubst %/,%,$(dir $@)) +# automatic +@F = $(notdir $@) +# makefile +CURDIR := /Users/anthonyra/Documents/GitHub/miner +# makefile +SHELL = /bin/sh +# environment +VSCODE_NLS_CONFIG = {"locale":"en-us","availableLanguages":{},"_languagePackSupport":true} +# environment +_ = /usr/bin/make +# makefile (from `Makefile', line 1) +MAKEFILE_LIST := Makefile +# environment +__CFBundleIdentifier = com.microsoft.VSCode +# environment +INFOPATH = /opt/homebrew/share/info: +# environment +VSCODE_CWD = / +# environment +PATH = /Users/anthonyra/.nvm/versions/node/v16.16.0/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/anthonyra/.cargo/bin:/Users/anthonyra/go/bin:/opt/homebrew/opt/go/libexec/bin:/Users/anthonyra/Documents/GitHub/xorf-generator/target/release:/Users/anthonyra/Documents/GitHub/helium-wallet-rs/target/release +# makefile (from `Makefile', line 4) +VAL_VERSION = $(shell git describe --abbrev=0 | sed -e 's,validator,,') +# makefile (from `Makefile', line 120) +remove = @rm -rf $(1) 2>/dev/null || true +# makefile (from `Makefile', line 15) +GWMP_MUX_VSN = "v0.9.4" +# makefile (from `Makefile', line 3) +REBAR = ./rebar3 +# environment +GOPATH = /Users/anthonyra/go +# environment +NVM_BIN = /Users/anthonyra/.nvm/versions/node/v16.16.0/bin +# environment +ELECTRON_RUN_AS_NODE = 1 +# default +.FEATURES := target-specific order-only second-expansion else-if archives jobserver check-symlink +# environment +SSH_AUTH_SOCK = /private/tmp/com.apple.launchd.eCJgWhVtcR/Listeners +# automatic +%F = $(notdir $%) +# environment +PWD = /Users/anthonyra/Documents/GitHub/miner +# environment +HOMEBREW_CELLAR = /opt/homebrew/Cellar +# environment +ORIGINAL_XDG_CURRENT_DESKTOP = undefined +# environment +MANPATH = /Users/anthonyra/.nvm/versions/node/v16.16.0/share/man:/opt/homebrew/share/man:: +# environment +VSCODE_AMD_ENTRYPOINT = vs/workbench/api/node/extensionHostProcess +# environment +HOME = /Users/anthonyra +# default +MAKEFILEPATH = $(shell /usr/bin/xcode-select -print-path 2>/dev/null || echo /Developer)/Makefiles +# environment +VSCODE_CODE_CACHE_PATH = /Users/anthonyra/Library/Application Support/Code/CachedData/da76f93349a72022ca4670c1b84860304616aaa2 +# environment +LOGNAME = anthonyra +# environment +APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true +# environment +NVM_CD_FLAGS = -q +# environment +bob = cosmos174fnltx00g57dlynayzd84wzmh3mj9vkvfs4x2 +# environment +VSCODE_HANDLES_UNCAUGHT_ERRORS = true +# automatic +^D = $(patsubst %/,%,$(dir $^)) +# environment +XPC_FLAGS = 0x0 +# default +MAKE = $(MAKE_COMMAND) +# default +MAKECMDGOALS := all +# environment +SHLVL = 1 +# default +MAKE_VERSION := 3.81 +# environment +USER = anthonyra +# makefile +.DEFAULT_GOAL := all +# environment +GOROOT = /opt/homebrew/opt/go/libexec +# makefile (from `Makefile', line 9) +CIBRANCH = $(shell git rev-parse --abbrev-ref HEAD | sed 's/\//-/') +# automatic +%D = $(patsubst %/,%,$(dir $%)) +# default +MAKE_COMMAND := /Library/Developer/CommandLineTools/usr/bin/make +# default +.VARIABLES := +# environment +TMPDIR = /var/folders/sv/74lrkyx96m34spltbzxrjrdr0000gn/T/ +# automatic +*F = $(notdir $*) +# environment +VSCODE_IPC_HOOK = /Users/anthonyra/Library/Application Support/Code/1.70.0-main.sock +# environment +MallocNanoZone = 0 +# makefile +MAKEFLAGS = Rrqp +# environment +MFLAGS = -Rrqp +# automatic +*D = $(patsubst %/,%,$(dir $*)) +# environment +NVM_DIR = /Users/anthonyra/.nvm +# makefile (from `Makefile', line 111) +clone_project = @git clone --quiet --depth 1 --branch $(2) https://github.com/helium/$(1) ./external/$(1) 2>/dev/null || true +# environment +XPC_SERVICE_NAME = application.com.microsoft.VSCode.6705052.6705058 +# environment +HOMEBREW_PREFIX = /opt/homebrew +# automatic ++D = $(patsubst %/,%,$(dir $+)) +# automatic ++F = $(notdir $+) +# makefile (from `Makefile', line 12) +GRPC_SERVICES_DIR = src/grpc/autogen +# makefile (from `Makefile', line 14) +GATEWAY_RS_VSN = "v1.0.0-alpha.30" +# environment +HOMEBREW_REPOSITORY = /opt/homebrew +# environment +__CF_USER_TEXT_ENCODING = 0x1F5:0x0:0x0 +# environment +COMMAND_MODE = unix2003 +# default +MAKEFILES := +# automatic +/dev/null || true + @(cd external/gateway-rs && git fetch && git checkout $(GATEWAY_RS_VSN) 2>/dev/null) + @(cd ./external/gateway-rs && cargo build --release) + $(call install_rust_bin,gateway-rs,helium_gateway,gateway_rs) + @cp ./external/gateway-rs/config/default.toml ./priv/gateway_rs/default.toml + @echo "--- gwmp-mux ---" + $(call clone_project,gwmp-mux,$(GWMP_MUX_VSN)) + @(cd ./external/gwmp-mux && cargo build --release) + $(call install_rust_bin,gwmp-mux,gwmp-mux,gwmp_mux) + + +all: compile +# Command-line target. +# Implicit rule search has been done. +# File does not exist. +# File has been updated. +# Needs to be updated (-q is set). +# variable set hash-table stats: +# Load=0/32=0%, Rehash=0, Collisions=0/8=0% + +validator: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 47): + $(REBAR) as validator release -n miner -v $(VAL_VERSION) + + +cover: +# Phony target (prerequisite of .PHONY). +# Implicit rule search has not been done. +# File does not exist. +# File has not been updated. +# commands to execute (from `Makefile', line 50): + $(REBAR) cover + + +.PHONY: deps compile test typecheck cover +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. + +aws: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 53): + $(REBAR) as aws release + + +deps: +# Phony target (prerequisite of .PHONY). +# Implicit rule search has not been done. +# File does not exist. +# File has not been updated. +# commands to execute (from `Makefile', line 20): + $(REBAR) get-deps + + +# Not a target: +.SUFFIXES: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. + +# Not a target: +Makefile: +# Implicit rule search has been done. +# Last modified 2022-08-05 14:50:33 +# File has been updated. +# Successfully updated. +# variable set hash-table stats: +# Load=0/32=0%, Rehash=0, Collisions=0/0=0% + +typecheck: +# Phony target (prerequisite of .PHONY). +# Implicit rule search has not been done. +# File does not exist. +# File has not been updated. +# commands to execute (from `Makefile', line 38): + $(REBAR) dialyzer xref + + +test: compile +# Phony target (prerequisite of .PHONY). +# Implicit rule search has not been done. +# File does not exist. +# File has not been updated. +# commands to execute (from `Makefile', line 35): + $(REBAR) as test do eunit, ct --verbose + + +seed: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 56): + $(REBAR) as seed release + + +devrelease: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 72): + $(REBAR) as dev release + + +grpc: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 75): + + @echo "generating miner grpc services" + $(REBAR) get-deps + REBAR_CONFIG="config/grpc_client_gen_local.config" $(REBAR) grpc gen + REBAR_CONFIG="config/grpc_client_gen.config" $(REBAR) grpc gen + + +devrel: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 62): + $(REBAR) as testdev, miner1 release -n miner1 + $(REBAR) as testdev, miner2 release -n miner2 + $(REBAR) as testdev, miner3 release -n miner3 + $(REBAR) as testdev, miner4 release -n miner4 + $(REBAR) as testdev, miner5 release -n miner5 + $(REBAR) as testdev, miner6 release -n miner6 + $(REBAR) as testdev, miner7 release -n miner7 + $(REBAR) as testdev, miner8 release -n miner8 + + +compile: +# Phony target (prerequisite of .PHONY). +# Implicit rule search has not been done. +# Implicit/static pattern stem: `' +# File does not exist. +# File has been updated. +# Needs to be updated (-q is set). +# automatic +# @ := compile +# automatic +# % := +# automatic +# * := +# automatic +# + := +# automatic +# | := +# automatic +# < := +# automatic +# ^ := +# automatic +# ? := +# variable set hash-table stats: +# Load=8/32=25%, Rehash=0, Collisions=2/16=12% +# commands to execute (from `Makefile', line 23): + $(REBAR) get-deps + REBAR_CONFIG="config/grpc_client_gen_local.config" $(REBAR) grpc gen + REBAR_CONFIG="config/grpc_client_gen.config" $(REBAR) grpc gen + $(MAKE) external_svcs + $(REBAR) compile + + +release: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 44): + $(REBAR) as prod release -n miner + + +# Not a target: +.DEFAULT: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. + +ci: compile +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 41): + $(REBAR) do dialyzer,xref && ($(REBAR) do eunit,ct || (mkdir -p artifacts; tar --exclude='./_build/test/lib' --exclude='./_build/test/plugins' -czf artifacts/$(CIBRANCH).tar.gz _build/test; false)) + + +clean_external_svcs: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 104): + @echo "removing external dependency project files" + $(call remove,./external/gateway-rs) + $(call remove,./priv/gateway_rs/helium_gateway) + $(call remove,./priv/gateway_rs/default.toml) + $(call remove,./external/gwmp-mux) + $(call remove,./priv/gwmp_mux/gwmp-mux) + + +clean: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 30): + $(MAKE) clean_external_svcs + $(MAKE) clean_grpc + $(REBAR) clean + + +docker: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 59): + $(REBAR) as docker release + + +clean_grpc: +# Implicit rule search has not been done. +# Modification time never checked. +# File has not been updated. +# commands to execute (from `Makefile', line 86): + @echo "cleaning miner grpc services" + rm -rf $(GRPC_SERVICES_DIR) + + +# files hash-table stats: +# Load=23/1024=2%, Rehash=0, Collisions=0/45=0% +# VPATH Search Paths + +# No `vpath' search paths. + +# No general (`VPATH' variable) search path. + +# # of strings in strcache: 1 +# # of strcache buffers: 1 +# strcache size: total = 4096 / max = 4096 / min = 4096 / avg = 4096 +# strcache free: total = 4087 / max = 4087 / min = 4087 / avg = 4087 + +# Finished Make data base on Fri Aug 5 15:11:09 2022 + + diff --git a/config/test.config b/config/test.config index a075311fc..dfe8932ef 100644 --- a/config/test.config +++ b/config/test.config @@ -33,7 +33,8 @@ {peerbook_allow_rfc1918, true}, {peer_cache_timeout, 20000}, {sync_cooldown_time, 1}, - {skewed_sync_cooldown_time, 1} + {skewed_sync_cooldown_time, 1}, + {blocks_to_protect_from_gc, 80} ]}, {miner, [ @@ -41,12 +42,12 @@ {mode, validator}, {jsonrpc_port, 0}, {use_ebus, false}, - {block_time, 500}, - {election_interval, 10}, + {block_time, 60000}, + {election_interval, 30}, {dkg_stop_timeout, 15000}, {write_failed_txns, true}, {radio_device, undefined}, - {stabilization_period, 200}, + {stabilization_period, 50000}, %% dont perform regionalised checks in test envs %% we only really need the params below if this file is changed to specify a radio device %% as without one miner_lora is not started diff --git a/config/test_short.config b/config/test_short.config new file mode 100644 index 000000000..bd351018c --- /dev/null +++ b/config/test_short.config @@ -0,0 +1,57 @@ +%% -*- erlang -*- +[ + "config/sys.config", + {lager, + [ + {log_root, "log"} + ]}, +{libp2p, [ + {use_dns_for_seeds, false} + ]}, +{sibyl, + [ + {poc_mgr_mod, miner_poc_mgr}, + {poc_report_handler, miner_poc_report_handler} + ]}, + {blockchain, + [ + {seed_dns_cname, ""}, + {similarity_time_diff_mins, 30}, + {random_peer_pred, fun miner_util:true_predicate/1}, + {ip_confirmation_host, ""}, + {enable_nat, false}, + {gossip_version, 1}, + {testing, true}, + {honor_quick_sync, false}, + {listen_addresses, ["/ip4/0.0.0.0/tcp/0"]}, + {key, undefined}, + {num_consensus_members, 4}, + {base_dir, "data"}, + {seed_nodes, ""}, + {seed_node_dns, ""}, + {peerbook_update_interval, 60000}, + {peerbook_allow_rfc1918, true}, + {peer_cache_timeout, 20000}, + {sync_cooldown_time, 1}, + {skewed_sync_cooldown_time, 1}, + {blocks_to_protect_from_gc, 80} + ]}, + {miner, + [ + {denylist_keys, undefined}, + {mode, validator}, + {jsonrpc_port, 0}, + {use_ebus, false}, + {block_time, 60000}, + {election_interval, 30}, + {dkg_stop_timeout, 15000}, + {write_failed_txns, true}, + {radio_device, undefined}, + {stabilization_period, 2000}, + %% dont perform regionalised checks in test envs + %% we only really need the params below if this file is changed to specify a radio device + %% as without one miner_lora is not started + %% including the params anyway in case someone needs it in this env + {region_override, 'US915'} + ]} +]. diff --git a/rebar.config b/rebar.config index 2f5f970cc..07b625c79 100644 --- a/rebar.config +++ b/rebar.config @@ -181,7 +181,7 @@ [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, {miner3, [{relx, [{release, {'miner3', git}, [miner, runtime_tools, tools, recon]}, - {sys_config, "./config/test.config"}, + {sys_config, "./config/test_short.config"}, {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, @@ -191,7 +191,7 @@ [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, {miner5, [{relx, [{release, {'miner5', git}, [miner, runtime_tools, tools, recon]}, - {sys_config, "./config/test.config"}, + {sys_config, "./config/test_short.config"}, {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, @@ -210,6 +210,14 @@ {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, + {cluster, [ + {relx, [ + {release, {miner, git}, [miner, runtime_tools, tools, recon]}, + {sys_config, "./config/test.config"}, + {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]} + ] + }] + }, {prod, [ {pre_hooks, [ {compile, "make external_svcs"}, diff --git a/run.sh b/run.sh index b40c1330f..91a7091fe 100755 --- a/run.sh +++ b/run.sh @@ -102,6 +102,10 @@ exported_genesis_file="/tmp/genesis_$(date +%Y%m%d%H%M%S)" LOOP=5 while [ $LOOP -gt 0 ]; do for node in ${nodes[@]}; do + if [[ ! -e ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node ]]; then + exit 1 + fi + if [[ $(./_build/testdev\+miner$node/rel/miner$node/bin/miner$node info in_consensus) = *true* ]]; then ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node genesis export $exported_genesis_file if [ $? -eq 0 -a -f $exported_genesis_file ]; then diff --git a/src/handlers/miner_hbbft_handler.erl b/src/handlers/miner_hbbft_handler.erl index bef96fc08..afb47f0ab 100644 --- a/src/handlers/miner_hbbft_handler.erl +++ b/src/handlers/miner_hbbft_handler.erl @@ -129,7 +129,7 @@ handle_command(start_acs, State) -> {reply, ok, ignore}; {NewHBBFT, {send, Msgs}} -> ?mark(start_acs_new), - lager:notice("Started HBBFT round because of a block timeout"), + lager:notice("NextRound (~p) starting due to block timeout", [hbbft:round(NewHBBFT)]), {reply, ok, fixup_msgs(Msgs), State#state{hbbft=NewHBBFT}} end; handle_command(get_buf, State) -> @@ -192,9 +192,11 @@ handle_command( Ledger = blockchain:ledger(Chain), Version = md_version(Ledger), Seen = blockchain_utils:map_to_bitvector(S#state.seen), + TargetBlockTime = miner:calculate_next_block_time(Chain, true), HBBFT2 = hbbft:set_stamp_fun(?MODULE, metadata, [Version, #{seen => Seen, - bba_completion => S#state.bba}, + bba_completion => S#state.bba, + target_block_time => TargetBlockTime}, Chain], HBBFT1), case hbbft:next_round(HBBFT2, NextRound, []) of {HBBFT3, ok} -> diff --git a/src/miner.erl b/src/miner.erl index 460beda16..7a54892b9 100644 --- a/src/miner.erl +++ b/src/miner.erl @@ -20,9 +20,11 @@ keys/0, reset_late_block_timer/0, + calculate_next_block_time/1, calculate_next_block_time/2, start_chain/2, install_consensus/1, + group_block_time/1, remove_consensus/0, version/0 ]). @@ -55,8 +57,18 @@ snapshot_hash => binary() }. +-type metadata_v3() :: + #{ + timestamp => integer(), + seen => binary(), + bba_completion => binary(), + head_hash => blockchain_block:hash(), + snapshot_hash => binary(), + target_block_time => integer() + }. + -type metadata() :: - [{J :: pos_integer(), M :: metadata_v2() | metadata_v1()}]. + [{J :: pos_integer(), M :: metadata_v3() | metadata_v2() | metadata_v1()}]. -type swarm_keys() :: {libp2p_crypto:pubkey(), libp2p_crypto:sig_fun()}. @@ -84,6 +96,7 @@ blockchain :: undefined | blockchain:blockchain(), %% but every miner keeps a timer reference? block_timer = make_ref() :: reference(), + target_block_time :: undefined | integer(), late_block_timer = make_ref() :: reference(), current_height = -1 :: integer(), blockchain_ref = make_ref() :: reference(), @@ -315,9 +328,14 @@ start_chain(ConsensusGroup, Chain) -> install_consensus(ConsensusGroup) -> gen_server:cast(?MODULE, {install_consensus, ConsensusGroup}). +-spec group_block_time(integer()) -> ok. +group_block_time(GroupBlockTime) -> + gen_server:cast(?MODULE, {group_block_time, GroupBlockTime}). + remove_consensus() -> gen_server:cast(?MODULE, remove_consensus). + -spec version() -> integer(). version() -> %% format: @@ -375,7 +393,8 @@ handle_call(_Msg, _From, State) -> handle_cast(remove_consensus, State) -> erlang:cancel_timer(State#state.block_timer), {noreply, State#state{consensus_group = undefined, - block_timer = make_ref()}}; + block_timer = make_ref(), + target_block_time = undefined}}; handle_cast({install_consensus, NewConsensusGroup}, #state{consensus_group = Group} = State) when Group == NewConsensusGroup -> {noreply, State}; @@ -384,6 +403,20 @@ handle_cast({install_consensus, NewConsensusGroup}, lager:info("installing consensus ~p after ~p", [NewConsensusGroup, State#state.consensus_group]), {noreply, set_next_block_timer(State#state{consensus_group = NewConsensusGroup})}; +handle_cast({group_block_time, GroupBlockTime}, + #state{target_block_time=TargetBlockTime} = State) -> + Now = erlang:system_time(seconds), + %% the group block time is the next subsequent block time due to BBA + %% therefore, the GroupBlockTime should always be >= the current + %% TargetBlockTime + case Now >= GroupBlockTime orelse TargetBlockTime >= GroupBlockTime of + true -> + lager:info("Invalid target block time from group, tried ~p when already set at ~p", [GroupBlockTime, TargetBlockTime]), + {noreply, State#state{target_block_time = undefined}}; + false -> + lager:info("Setting target block timeout to ~p from group", [GroupBlockTime]), + {noreply, State#state{target_block_time = GroupBlockTime}} + end; handle_cast(_Msg, State) -> lager:warning("unhandled cast ~p", [_Msg]), {noreply, State}. @@ -392,17 +425,25 @@ handle_info({'DOWN', Ref, process, _, Reason}, State = #state{blockchain_ref=Ref lager:warning("Blockchain worker exited with reason ~p", [Reason]), {stop, Reason, State}; handle_info(block_timeout, State) when State#state.consensus_group == undefined -> - {noreply, State}; -handle_info(block_timeout, State) -> - lager:info("block timeout"), + {noreply, State#state{target_block_time=undefined}}; +handle_info(block_timeout, #state{target_block_time=TargetBlockTime} = State) -> + Now = erlang:system_time(seconds), libp2p_group_relcast:handle_input(State#state.consensus_group, start_acs), - {noreply, State}; + case Now >= TargetBlockTime of + true -> + lager:info("block timeout at ~p", [Now]), + {noreply, set_next_block_timer(State#state{target_block_time=undefined})}; + false -> + %% target block time came from group keep it to set next block timer + lager:info("block timeout at ~p, next set in ~ps", [Now, TargetBlockTime - Now]), + {noreply, set_next_block_timer(State)} + end; handle_info(late_block_timeout, State) -> LateBlockTimeout = application:get_env(miner, late_block_timeout_seconds, 120) * 1000, lager:info("late block timeout"), libp2p_group_relcast:handle_input(State#state.consensus_group, maybe_skip), LateTimer = erlang:send_after(LateBlockTimeout, self(), late_block_timeout), - {noreply, State#state{late_block_timer = LateTimer}}; + {noreply, State#state{late_block_timer=LateTimer, target_block_time=undefined}}; handle_info({blockchain_event, {add_block, Hash, Sync, Ledger}}, State=#state{consensus_group = ConsensusGroup, current_height = CurrHeight, @@ -453,7 +494,8 @@ handle_info({blockchain_event, {add_block, Hash, Sync, Ledger}}, end; {true, _, _, _} -> State#state{block_timer = make_ref(), - current_height = Height} + current_height = Height, + target_block_time = undefined} end; {error, Reason} -> lager:error("Error, Reason: ~p", [Reason]), @@ -576,7 +618,7 @@ create_block(Metadata, Txns, HBBFTRound, Chain, VotesNeeded, {MyPubKey, SignFun} Ledger = blockchain:ledger(Chain), SnapshotHash = snapshot_hash(Ledger, HeightNext, Metadata, VotesNeeded), SeenBBAs = - [{{J, S}, B} || {J, #{seen := S, bba_completion := B}} <- metadata_only_v2(Metadata)], + [{{J, S}, B} || {J, #{seen := S, bba_completion := B}} <- metadata_is_map(Metadata)], {SeenVectors, BBAs} = lists:unzip(SeenBBAs), %% if we cannot agree on the BBA results, default to flagging everyone as having completed VoteDefault = @@ -591,6 +633,10 @@ create_block(Metadata, Txns, HBBFTRound, Chain, VotesNeeded, {MyPubKey, SignFun} BBA = common_enough_or_default(VotesNeeded, BBAs, VoteDefault), {ElectionEpoch, EpochStart, TxnsToInsert, InvalidTransactions} = select_transactions(Chain, Txns, ElectionInfo, HeightCurr, HeightNext), + + GroupBlockTimeout = median_time(CurrentBlockTime, Metadata, target_block_time), + group_block_time(GroupBlockTimeout), + NewBlock = blockchain_block_v1:new(#{ prev_hash => CurrentBlockHash, @@ -598,7 +644,7 @@ create_block(Metadata, Txns, HBBFTRound, Chain, VotesNeeded, {MyPubKey, SignFun} transactions => TxnsToInsert, signatures => [], hbbft_round => HBBFTRound, - time => block_time(CurrentBlockTime, Metadata), + time => median_time(CurrentBlockTime, Metadata, timestamp), election_epoch => ElectionEpoch, epoch_start => EpochStart, seen_votes => SeenVectors, @@ -619,12 +665,13 @@ create_block(Metadata, Txns, HBBFTRound, Chain, VotesNeeded, {MyPubKey, SignFun} invalid_txns => InvalidTransactions }. --spec block_time(non_neg_integer(), metadata()) -> pos_integer(). -block_time(LastBlockTime, Metadata) -> +%% TODO - Check if works with metadata_v1 +-spec median_time(non_neg_integer(), metadata(), atom()) -> pos_integer(). +median_time(LastBlockTime, Metadata, Key) -> %% Try to rule out invalid values by not allowing timestamps to go %% backwards and take the median proposed value. - {Stamps, _} = meta_to_stamp_hashes(Metadata), - case miner_util:median([S || S <- Stamps, S >= LastBlockTime]) of + Times = meta_key_to_values(Metadata, Key), + case miner_util:median([T || T <- Times, T >= LastBlockTime]) of 0 -> LastBlockTime + 1; LastBlockTime -> LastBlockTime + 1; NewTime -> NewTime @@ -707,11 +754,15 @@ txn_is_rewards(Txn) -> Rewards = [blockchain_txn_rewards_v1, blockchain_txn_rewards_v2], lists:member(blockchain_txn:type(Txn), Rewards). --spec metadata_only_v2(metadata()) -> - [{non_neg_integer(), metadata_v2()}]. -metadata_only_v2(Metadata) -> +-spec metadata_is_map(metadata()) -> + [{non_neg_integer(), metadata_v3() | metadata_v2()}]. +metadata_is_map(Metadata) -> lists:filter(fun ({_, M}) -> is_map(M) end, Metadata). +-spec meta_key_to_values(metadata(), atom()) -> [any()]. +meta_key_to_values(Metadata, Key) -> + [Value || {_, #{Key := Value}} <- Metadata]. + -spec meta_to_stamp_hashes(metadata()) -> { Stamps :: [integer()], @@ -720,7 +771,7 @@ metadata_only_v2(Metadata) -> meta_to_stamp_hashes(Metadata) -> lists:unzip([metadata_as_v1(M) || {_, M} <- Metadata]). --spec metadata_as_v1(metadata_v1() | metadata_v2()) -> metadata_v1(). +-spec metadata_as_v1(metadata_v3() | metadata_v2() | metadata_v1()) -> metadata_v1(). metadata_as_v1(#{head_hash := H, timestamp := S}) -> {S, H}; % v2 -> v1 metadata_as_v1({S, H}) -> {S, H}. % v1 -> v1 @@ -734,7 +785,7 @@ snapshot_hash(Ledger, BlockHeightNext, Metadata, VotesNeeded) -> %% agree on one, just leave it blank, so other nodes can absorb it. case blockchain:config(?snapshot_interval, Ledger) of {ok, Interval} when (BlockHeightNext - 1) rem Interval == 0 -> - Hashes = [H || {_, #{snapshot_hash := H}} <- metadata_only_v2(Metadata)], + Hashes = [H || {_, #{snapshot_hash := H}} <- metadata_is_map(Metadata)], common_enough_or_default(VotesNeeded, Hashes, <<>>); _ -> <<>> @@ -750,11 +801,17 @@ common_enough_or_default(Threshold, Xs, Default) -> [{_, _}|_] -> Default % Not common-enough. end. -set_next_block_timer(State=#state{blockchain=Chain}) -> +-spec calculate_next_block_time(blockchain:blockchain()) -> integer(). +calculate_next_block_time(Chain) -> + calculate_next_block_time(Chain, false). + +-spec calculate_next_block_time(blockchain:blockchain(), boolean()) -> integer(). +calculate_next_block_time(Chain, Subsequent) -> Now = erlang:system_time(seconds), - {ok, BlockTime0} = blockchain:config(?block_time, blockchain:ledger(Chain)), - {ok, #block_info_v2{time=LastBlockTime, height=Height}} = blockchain:head_block_info(Chain), + Ledger = blockchain:ledger(Chain), + {ok, BlockTime0} = blockchain:config(?block_time, Ledger), BlockTime = BlockTime0 div 1000, + {ok, #block_info_v2{time=LastBlockTime, height=Height}} = blockchain:head_block_info(Chain), LastBlockTimestamp = case Height of 1 -> %% make up a plausible time for the genesis block @@ -762,63 +819,148 @@ set_next_block_timer(State=#state{blockchain=Chain}) -> _ -> LastBlockTime end, - - StartHeight0 = application:get_env(miner, stabilization_period, 0), - StartHeight = max(1, Height - StartHeight0), - {ActualStartHeight, StartBlockTime} = - case Height > StartHeight of - true -> - case blockchain:find_first_block_after(StartHeight, Chain) of - {ok, Actual, StartBlock} -> - {Actual, blockchain_block:time(StartBlock)}; - _ -> - {0, undefined} - end; - false -> - {0, undefined} + %% mimic snapshot_take functionality for block range window + %% blockchain_core:blockchain_ledger_snapshot_v1 + #{election_height := ElectionHeight} = blockchain_election:election_info(Ledger), + GraceBlocks = + case blockchain:config(?sc_grace_blocks, Ledger) of + {ok, GBs} -> + GBs; + {error, not_found} -> + 0 end, + DLedger = blockchain_ledger_v1:mode(delayed, Ledger), + {ok, DHeight0} = blockchain_ledger_v1:current_height(DLedger), + {ok, #block_info_v2{election_info={_, DHeight}}} = blockchain:get_block_info(DHeight0, Chain), + + %% mimic snapshot_take functionality for block range window + SnapshotStartHeight = max(1, min(DHeight, ElectionHeight - GraceBlocks) - 1), + SnapshotResult = get_average_block_time(Height, SnapshotStartHeight, BlockTime, LastBlockTimestamp, Chain), + %% original logic for calculationg block range window + StabilizationHeight0 = application:get_env(miner, stabilization_period, 0), + StabilizationHeight = max(1, Height - StabilizationHeight0 + 1), + StabilizationResults = get_average_block_time(Height, StabilizationHeight, BlockTime, LastBlockTimestamp, Chain), + %% grab the times with the largest positive amplitude, if amplitudes match grab the longest block history avgtime + OrderByHistory = lists:reverse(lists:keysort(2, [StabilizationResults, SnapshotResult])), + {_Amplitude, AvgBlockTime0, BlockRange0} = lists:max([{(A - BlockTime), A, R} || {A, R} <- [StabilizationResults, SnapshotResult]]), + lager:info("Selected {~p,~p} from ~p", [AvgBlockTime0, BlockRange0, OrderByHistory]), + NextBlockTime0 = get_next_block_time({AvgBlockTime0, BlockRange0}, BlockTime, LastBlockTimestamp), + NextBlockTime = case Subsequent of + true -> + BlockRange = BlockRange0 + 1, + AvgBlockTime = (AvgBlockTime0 * BlockRange0 + (NextBlockTime0 - Now)) / BlockRange, + lager:info("# blocks ~p || average block times ~p difference ~p", [BlockRange, AvgBlockTime, BlockTime - AvgBlockTime]), + get_next_block_time({AvgBlockTime, BlockRange}, BlockTime, NextBlockTime0); + false -> + lager:info("# blocks ~p || average block times ~p difference ~p", [BlockRange0, AvgBlockTime0, BlockTime - AvgBlockTime0]), + NextBlockTime0 + end, + lager:info("Next block timeout @ ~p in ~ps", [NextBlockTime, NextBlockTime - Now]), + NextBlockTime. + +%% set next block timer if not already done, used for backwards compatability assumes if block_timer is +%% set correctly then late_block_timer is also +-spec set_next_block_timer(map()) -> map() | ok. +set_next_block_timer(State=#state{blockchain=Chain, target_block_time=undefined}) -> + Now = erlang:system_time(seconds), + TargetBlockTime = calculate_next_block_time(Chain), + NextBlockTime = max(0, TargetBlockTime - Now), + lager:info("Set block timeout to ~p in ~ps", [TargetBlockTime, NextBlockTime]), + set_block_timers(State#state{target_block_time = TargetBlockTime}, NextBlockTime); +set_next_block_timer(State=#state{block_timer=BlockTimer, target_block_time=TargetBlockTime}) -> + %% not to block the critical path + erlang:read_timer(BlockTimer, [{async, true}]), + receive + {read_timer, _TimerRef, CurrentTime} -> + Now = erlang:system_time(seconds), + NextBlockTime = max(0, TargetBlockTime - Now), + case CurrentTime of + false -> + lager:info("Set block timeout to ~p in ~ps", [TargetBlockTime, NextBlockTime]), + set_block_timers(State, NextBlockTime); + _ -> + %% timer is ticking, let it be + lager:info("Current block timer has ~ps remaining", [CurrentTime div 1000]), + ok + end + end. + +%% separate to deduplicate code +-spec set_block_timers(map(), non_neg_integer()) -> map(). +set_block_timers(State, NextBlockTime) -> + lager:info("Setting next block timer to ~ps", [NextBlockTime]), + Timer = erlang:send_after(NextBlockTime * 1000, self(), block_timeout), + erlang:cancel_timer(State#state.late_block_timer), + LateBlockTimeout = application:get_env(miner, late_block_timeout_seconds, 120), + LateTimer = erlang:send_after((LateBlockTimeout + NextBlockTime) * 1000, self(), late_block_timeout), + State#state{block_timer=Timer, late_block_timer=LateTimer}. + +%% ------------------------------------------------------------------ +%% Internal Functions +%% ------------------------------------------------------------------ + +get_average_block_time(Height, StartHeight, BlockTime, LastBlockTimestamp, Chain) -> + {ActualStartHeight, StartBlockTime} = case Height > StartHeight of + true -> + case blockchain:find_first_block_after(StartHeight, Chain) of + {ok, Actual, StartBlock} -> + {Actual, blockchain_block:time(StartBlock)}; + _ -> + {0, undefined} + end; + false -> + {0, undefined} + end, + BlockRange = max(Height - ActualStartHeight, 1), AvgBlockTime = case StartBlockTime of - undefined -> - BlockTime; - _ -> - (LastBlockTimestamp - StartBlockTime) / max(Height - ActualStartHeight, 1) - end, - BlockTimeDeviation0 = BlockTime - AvgBlockTime, - lager:info("average ~p block times ~p difference ~p", [Height, AvgBlockTime, BlockTime - AvgBlockTime]), + undefined -> + BlockTime; + _ -> + (LastBlockTimestamp - StartBlockTime) / BlockRange + end, + {AvgBlockTime, BlockRange}. + +get_next_block_time({AvgBlockTime, BlockRange}, BlockTime, LastBlockTimestamp) -> + Now = erlang:system_time(seconds), + DifferenceInTime = (BlockTime - AvgBlockTime) * BlockRange, BlockTimeDeviation = - case BlockTimeDeviation0 of + case DifferenceInTime of N when N > 0 -> min(1, catchup_time(abs(N))); N -> -1 * catchup_time(abs(N)) end, - NextBlockTime = max(0, (LastBlockTimestamp + BlockTime + BlockTimeDeviation) - Now), - lager:info("Next block after ~p is in ~p seconds", [LastBlockTimestamp, NextBlockTime]), - Timer = erlang:send_after(NextBlockTime * 1000, self(), block_timeout), - - %% now figure out the late block timer - erlang:cancel_timer(State#state.late_block_timer), - LateBlockTimeout = application:get_env(miner, late_block_timeout_seconds, 120), - LateTimer = erlang:send_after((LateBlockTimeout + NextBlockTime) * 1000, self(), late_block_timeout), - - State#state{block_timer=Timer, late_block_timer = LateTimer}. + %% if chain has been stopped longer then LastBlockTimeout prevent 0 NextBlockTime + max(Now, LastBlockTimestamp) + BlockTime + BlockTimeDeviation. %% input in fractional seconds, the number of seconds between the %% target block time and the average total time over the target period %% output in seconds of adjustment to apply to the block time target -%% the constants here are by feel: currently at 100k target and 5sec -%% adjustment it takes roughly a day of blocks to make up a tenth of a -%% second. with the new 50k target we can expect double the adjustment -%% leverage, so 1s adjustments will take roughly a day to make up the -%% final 0.04 (twice as much leverage, but 20% of the rate). - %% when drift is small or 0, let it accumulate for a bit catchup_time(N) when N < 0.001 -> 0; -%% when it's still relatively small, apply gentle adjustments -catchup_time(N) when N < 0.01 -> - 1; -%% if it's large, jam on the gas -catchup_time(_) -> - 10. +%% try and catch up within 10 blocks, max 10 seconds +catchup_time(N) -> + min(10, ceil(N / 10)). + + +%% ------------------------------------------------------------------ +%% EUNIT Tests +%% ------------------------------------------------------------------ +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +%% Confirm changes to make catchup_time proportional +catchup_time_test() -> + ?assertEqual(catchup_time(0.0005), 0), + ?assertEqual(catchup_time(0.01), 1), + ?assertEqual(catchup_time(0.015), 2), + ?assertEqual(catchup_time(0.02), 2), + ?assertEqual(catchup_time(0.05), 5), + ?assertEqual(catchup_time(0.09), 9), + ?assertEqual(catchup_time(0.090001), 10), + ?assertEqual(catchup_time(0.1), 10), + ?assertEqual(catchup_time(1), 10). + +-endif. From d4ab1233d9153ed456bede5844afbc3a018a201f Mon Sep 17 00:00:00 2001 From: Anthony Anderson Date: Fri, 5 Aug 2022 15:31:41 -0400 Subject: [PATCH 2/3] Remove ignored files --- .gitignore | 3 +- .vscode/configurationCache.log | 1 - .vscode/dryrun.log | 26 -- .vscode/settings.json | 3 - .vscode/targets.log | 448 --------------------------------- external/.gitkeep | 0 6 files changed, 2 insertions(+), 479 deletions(-) delete mode 100644 .vscode/configurationCache.log delete mode 100644 .vscode/dryrun.log delete mode 100644 .vscode/settings.json delete mode 100644 .vscode/targets.log delete mode 100644 external/.gitkeep diff --git a/.gitignore b/.gitignore index 5b55d9f2f..95970175a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,6 @@ data src/pb .repos/ *.deb -*.log # make build output for external deps external/ @@ -36,3 +35,5 @@ helium_gateway # gwmp-mux generated files gwmp-mux + +.vscode/ \ No newline at end of file diff --git a/.vscode/configurationCache.log b/.vscode/configurationCache.log deleted file mode 100644 index 5232aabd5..000000000 --- a/.vscode/configurationCache.log +++ /dev/null @@ -1 +0,0 @@ -{"buildTargets":[".PHONY","all","aws","ci","clean","clean_external_svcs","clean_grpc","compile","cover","deps","devrel","devrelease","docker","external_svcs","grpc","release","seed","test","typecheck","validator"],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}} \ No newline at end of file diff --git a/.vscode/dryrun.log b/.vscode/dryrun.log deleted file mode 100644 index 9c3941e17..000000000 --- a/.vscode/dryrun.log +++ /dev/null @@ -1,26 +0,0 @@ -make --dry-run --always-make --keep-going --print-directory -make: Entering directory `/Users/anthonyra/Documents/GitHub/miner' -./rebar3 get-deps -REBAR_CONFIG="config/grpc_client_gen_local.config" ./rebar3 grpc gen -REBAR_CONFIG="config/grpc_client_gen.config" ./rebar3 grpc gen -/Library/Developer/CommandLineTools/usr/bin/make external_svcs - -make[1]: Entering directory `/Users/anthonyra/Documents/GitHub/miner' -echo "cloning external dependency projects" -echo "--- gateway-rs ---" -git clone --quiet https://github.com/helium/gateway-rs ./external/gateway-rs 2>/dev/null || true -(cd external/gateway-rs && git fetch && git checkout "v1.0.0-alpha.30" 2>/dev/null) -(cd ./external/gateway-rs && cargo build --release) -mkdir -p ./priv/gateway_rs -mv ./external/gateway-rs/target/release/helium_gateway ./priv/gateway_rs/ 2>/dev/null || true -cp ./external/gateway-rs/config/default.toml ./priv/gateway_rs/default.toml -echo "--- gwmp-mux ---" -git clone --quiet --depth 1 --branch "v0.9.4" https://github.com/helium/gwmp-mux ./external/gwmp-mux 2>/dev/null || true -(cd ./external/gwmp-mux && cargo build --release) -mkdir -p ./priv/gwmp_mux -mv ./external/gwmp-mux/target/release/gwmp-mux ./priv/gwmp_mux/ 2>/dev/null || true -make[1]: Leaving directory `/Users/anthonyra/Documents/GitHub/miner' - -./rebar3 compile -make: Leaving directory `/Users/anthonyra/Documents/GitHub/miner' - diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 65e1ec078..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "makefile.extensionOutputFolder": "./.vscode" -} \ No newline at end of file diff --git a/.vscode/targets.log b/.vscode/targets.log deleted file mode 100644 index 081cf7cf3..000000000 --- a/.vscode/targets.log +++ /dev/null @@ -1,448 +0,0 @@ -make all --print-data-base --no-builtin-variables --no-builtin-rules --question -# GNU Make 3.81 -# Copyright (C) 2006 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. -# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -# This program built for i386-apple-darwin11.3.0 - - -# Make data base, printed on Fri Aug 5 15:11:09 2022 - -# Variables - -# automatic -/dev/null || true -endef -# automatic -@D = $(patsubst %/,%,$(dir $@)) -# automatic -@F = $(notdir $@) -# makefile -CURDIR := /Users/anthonyra/Documents/GitHub/miner -# makefile -SHELL = /bin/sh -# environment -VSCODE_NLS_CONFIG = {"locale":"en-us","availableLanguages":{},"_languagePackSupport":true} -# environment -_ = /usr/bin/make -# makefile (from `Makefile', line 1) -MAKEFILE_LIST := Makefile -# environment -__CFBundleIdentifier = com.microsoft.VSCode -# environment -INFOPATH = /opt/homebrew/share/info: -# environment -VSCODE_CWD = / -# environment -PATH = /Users/anthonyra/.nvm/versions/node/v16.16.0/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/anthonyra/.cargo/bin:/Users/anthonyra/go/bin:/opt/homebrew/opt/go/libexec/bin:/Users/anthonyra/Documents/GitHub/xorf-generator/target/release:/Users/anthonyra/Documents/GitHub/helium-wallet-rs/target/release -# makefile (from `Makefile', line 4) -VAL_VERSION = $(shell git describe --abbrev=0 | sed -e 's,validator,,') -# makefile (from `Makefile', line 120) -remove = @rm -rf $(1) 2>/dev/null || true -# makefile (from `Makefile', line 15) -GWMP_MUX_VSN = "v0.9.4" -# makefile (from `Makefile', line 3) -REBAR = ./rebar3 -# environment -GOPATH = /Users/anthonyra/go -# environment -NVM_BIN = /Users/anthonyra/.nvm/versions/node/v16.16.0/bin -# environment -ELECTRON_RUN_AS_NODE = 1 -# default -.FEATURES := target-specific order-only second-expansion else-if archives jobserver check-symlink -# environment -SSH_AUTH_SOCK = /private/tmp/com.apple.launchd.eCJgWhVtcR/Listeners -# automatic -%F = $(notdir $%) -# environment -PWD = /Users/anthonyra/Documents/GitHub/miner -# environment -HOMEBREW_CELLAR = /opt/homebrew/Cellar -# environment -ORIGINAL_XDG_CURRENT_DESKTOP = undefined -# environment -MANPATH = /Users/anthonyra/.nvm/versions/node/v16.16.0/share/man:/opt/homebrew/share/man:: -# environment -VSCODE_AMD_ENTRYPOINT = vs/workbench/api/node/extensionHostProcess -# environment -HOME = /Users/anthonyra -# default -MAKEFILEPATH = $(shell /usr/bin/xcode-select -print-path 2>/dev/null || echo /Developer)/Makefiles -# environment -VSCODE_CODE_CACHE_PATH = /Users/anthonyra/Library/Application Support/Code/CachedData/da76f93349a72022ca4670c1b84860304616aaa2 -# environment -LOGNAME = anthonyra -# environment -APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = true -# environment -NVM_CD_FLAGS = -q -# environment -bob = cosmos174fnltx00g57dlynayzd84wzmh3mj9vkvfs4x2 -# environment -VSCODE_HANDLES_UNCAUGHT_ERRORS = true -# automatic -^D = $(patsubst %/,%,$(dir $^)) -# environment -XPC_FLAGS = 0x0 -# default -MAKE = $(MAKE_COMMAND) -# default -MAKECMDGOALS := all -# environment -SHLVL = 1 -# default -MAKE_VERSION := 3.81 -# environment -USER = anthonyra -# makefile -.DEFAULT_GOAL := all -# environment -GOROOT = /opt/homebrew/opt/go/libexec -# makefile (from `Makefile', line 9) -CIBRANCH = $(shell git rev-parse --abbrev-ref HEAD | sed 's/\//-/') -# automatic -%D = $(patsubst %/,%,$(dir $%)) -# default -MAKE_COMMAND := /Library/Developer/CommandLineTools/usr/bin/make -# default -.VARIABLES := -# environment -TMPDIR = /var/folders/sv/74lrkyx96m34spltbzxrjrdr0000gn/T/ -# automatic -*F = $(notdir $*) -# environment -VSCODE_IPC_HOOK = /Users/anthonyra/Library/Application Support/Code/1.70.0-main.sock -# environment -MallocNanoZone = 0 -# makefile -MAKEFLAGS = Rrqp -# environment -MFLAGS = -Rrqp -# automatic -*D = $(patsubst %/,%,$(dir $*)) -# environment -NVM_DIR = /Users/anthonyra/.nvm -# makefile (from `Makefile', line 111) -clone_project = @git clone --quiet --depth 1 --branch $(2) https://github.com/helium/$(1) ./external/$(1) 2>/dev/null || true -# environment -XPC_SERVICE_NAME = application.com.microsoft.VSCode.6705052.6705058 -# environment -HOMEBREW_PREFIX = /opt/homebrew -# automatic -+D = $(patsubst %/,%,$(dir $+)) -# automatic -+F = $(notdir $+) -# makefile (from `Makefile', line 12) -GRPC_SERVICES_DIR = src/grpc/autogen -# makefile (from `Makefile', line 14) -GATEWAY_RS_VSN = "v1.0.0-alpha.30" -# environment -HOMEBREW_REPOSITORY = /opt/homebrew -# environment -__CF_USER_TEXT_ENCODING = 0x1F5:0x0:0x0 -# environment -COMMAND_MODE = unix2003 -# default -MAKEFILES := -# automatic -/dev/null || true - @(cd external/gateway-rs && git fetch && git checkout $(GATEWAY_RS_VSN) 2>/dev/null) - @(cd ./external/gateway-rs && cargo build --release) - $(call install_rust_bin,gateway-rs,helium_gateway,gateway_rs) - @cp ./external/gateway-rs/config/default.toml ./priv/gateway_rs/default.toml - @echo "--- gwmp-mux ---" - $(call clone_project,gwmp-mux,$(GWMP_MUX_VSN)) - @(cd ./external/gwmp-mux && cargo build --release) - $(call install_rust_bin,gwmp-mux,gwmp-mux,gwmp_mux) - - -all: compile -# Command-line target. -# Implicit rule search has been done. -# File does not exist. -# File has been updated. -# Needs to be updated (-q is set). -# variable set hash-table stats: -# Load=0/32=0%, Rehash=0, Collisions=0/8=0% - -validator: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 47): - $(REBAR) as validator release -n miner -v $(VAL_VERSION) - - -cover: -# Phony target (prerequisite of .PHONY). -# Implicit rule search has not been done. -# File does not exist. -# File has not been updated. -# commands to execute (from `Makefile', line 50): - $(REBAR) cover - - -.PHONY: deps compile test typecheck cover -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. - -aws: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 53): - $(REBAR) as aws release - - -deps: -# Phony target (prerequisite of .PHONY). -# Implicit rule search has not been done. -# File does not exist. -# File has not been updated. -# commands to execute (from `Makefile', line 20): - $(REBAR) get-deps - - -# Not a target: -.SUFFIXES: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. - -# Not a target: -Makefile: -# Implicit rule search has been done. -# Last modified 2022-08-05 14:50:33 -# File has been updated. -# Successfully updated. -# variable set hash-table stats: -# Load=0/32=0%, Rehash=0, Collisions=0/0=0% - -typecheck: -# Phony target (prerequisite of .PHONY). -# Implicit rule search has not been done. -# File does not exist. -# File has not been updated. -# commands to execute (from `Makefile', line 38): - $(REBAR) dialyzer xref - - -test: compile -# Phony target (prerequisite of .PHONY). -# Implicit rule search has not been done. -# File does not exist. -# File has not been updated. -# commands to execute (from `Makefile', line 35): - $(REBAR) as test do eunit, ct --verbose - - -seed: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 56): - $(REBAR) as seed release - - -devrelease: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 72): - $(REBAR) as dev release - - -grpc: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 75): - - @echo "generating miner grpc services" - $(REBAR) get-deps - REBAR_CONFIG="config/grpc_client_gen_local.config" $(REBAR) grpc gen - REBAR_CONFIG="config/grpc_client_gen.config" $(REBAR) grpc gen - - -devrel: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 62): - $(REBAR) as testdev, miner1 release -n miner1 - $(REBAR) as testdev, miner2 release -n miner2 - $(REBAR) as testdev, miner3 release -n miner3 - $(REBAR) as testdev, miner4 release -n miner4 - $(REBAR) as testdev, miner5 release -n miner5 - $(REBAR) as testdev, miner6 release -n miner6 - $(REBAR) as testdev, miner7 release -n miner7 - $(REBAR) as testdev, miner8 release -n miner8 - - -compile: -# Phony target (prerequisite of .PHONY). -# Implicit rule search has not been done. -# Implicit/static pattern stem: `' -# File does not exist. -# File has been updated. -# Needs to be updated (-q is set). -# automatic -# @ := compile -# automatic -# % := -# automatic -# * := -# automatic -# + := -# automatic -# | := -# automatic -# < := -# automatic -# ^ := -# automatic -# ? := -# variable set hash-table stats: -# Load=8/32=25%, Rehash=0, Collisions=2/16=12% -# commands to execute (from `Makefile', line 23): - $(REBAR) get-deps - REBAR_CONFIG="config/grpc_client_gen_local.config" $(REBAR) grpc gen - REBAR_CONFIG="config/grpc_client_gen.config" $(REBAR) grpc gen - $(MAKE) external_svcs - $(REBAR) compile - - -release: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 44): - $(REBAR) as prod release -n miner - - -# Not a target: -.DEFAULT: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. - -ci: compile -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 41): - $(REBAR) do dialyzer,xref && ($(REBAR) do eunit,ct || (mkdir -p artifacts; tar --exclude='./_build/test/lib' --exclude='./_build/test/plugins' -czf artifacts/$(CIBRANCH).tar.gz _build/test; false)) - - -clean_external_svcs: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 104): - @echo "removing external dependency project files" - $(call remove,./external/gateway-rs) - $(call remove,./priv/gateway_rs/helium_gateway) - $(call remove,./priv/gateway_rs/default.toml) - $(call remove,./external/gwmp-mux) - $(call remove,./priv/gwmp_mux/gwmp-mux) - - -clean: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 30): - $(MAKE) clean_external_svcs - $(MAKE) clean_grpc - $(REBAR) clean - - -docker: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 59): - $(REBAR) as docker release - - -clean_grpc: -# Implicit rule search has not been done. -# Modification time never checked. -# File has not been updated. -# commands to execute (from `Makefile', line 86): - @echo "cleaning miner grpc services" - rm -rf $(GRPC_SERVICES_DIR) - - -# files hash-table stats: -# Load=23/1024=2%, Rehash=0, Collisions=0/45=0% -# VPATH Search Paths - -# No `vpath' search paths. - -# No general (`VPATH' variable) search path. - -# # of strings in strcache: 1 -# # of strcache buffers: 1 -# strcache size: total = 4096 / max = 4096 / min = 4096 / avg = 4096 -# strcache free: total = 4087 / max = 4087 / min = 4087 / avg = 4087 - -# Finished Make data base on Fri Aug 5 15:11:09 2022 - - diff --git a/external/.gitkeep b/external/.gitkeep deleted file mode 100644 index e69de29bb..000000000 From 25d6327b8bf3a57bcccb58b704fd81ed2da2a002 Mon Sep 17 00:00:00 2001 From: Anthony Anderson Date: Fri, 19 Aug 2022 09:04:01 -0400 Subject: [PATCH 3/3] testing --- rebar.config | 4 +-- start.sh | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 start.sh diff --git a/rebar.config b/rebar.config index 07b625c79..7b673fb4c 100644 --- a/rebar.config +++ b/rebar.config @@ -181,7 +181,7 @@ [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, {miner3, [{relx, [{release, {'miner3', git}, [miner, runtime_tools, tools, recon]}, - {sys_config, "./config/test_short.config"}, + {sys_config, "./config/test.config"}, {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, @@ -191,7 +191,7 @@ [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, {miner5, [{relx, [{release, {'miner5', git}, [miner, runtime_tools, tools, recon]}, - {sys_config, "./config/test_short.config"}, + {sys_config, "./config/test.config"}, {overlay, [{template, "config/vm_dev.args", "{{output_dir}}/releases/{{release_version}}/vm.args"}]}]}]}, diff --git a/start.sh b/start.sh new file mode 100644 index 000000000..0c4f35050 --- /dev/null +++ b/start.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +nodes=$(seq 8) + +start_dev_release() { + ./_build/testdev\+miner$1/rel/miner$1/bin/miner$1 daemon +} + +export -f start_dev_release +parallel -k --tagstring miner{} start_dev_release ::: $nodes + +sleep 5s +# peer addresses for node 1 +addresses=() +# get the peer addrs for each node +peer_addrs=() + +# function to join array values +function join_by { local IFS="$1"; shift; echo "$*"; } + +# connect node1 to every _other_ node +for node in ${nodes[@]}; do + peer_addrs+=($(./_build/testdev\+miner$node/rel/miner$node/bin/miner$node peer addr)) + + if (( $node != 1 )); then + for address in ${addresses[@]}; do + echo "## Node $node trying to connect to seed node 1 which has listen address: $address" + ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node peer connect $address + done + else + addresses+=($(./_build/testdev\+miner1/rel/miner1/bin/miner1 peer listen --format=csv | tail -n +2)) + fi + + sleep 1s +done + +priv_key=$(./_build/testdev\+miner1/rel/miner1/bin/miner1 genesis key) +echo $priv_key +proof=($(./_build/testdev\+miner1/rel/miner1/bin/miner1 genesis proof $priv_key | grep -v ":")) +echo ${proof[@]} + +forge_genesis_block() { + ./_build/testdev\+miner$1/rel/miner$1/bin/miner$1 genesis forge $2 $3 $4 +} +export -f forge_genesis_block + +parallel -k --tagstring miner{} forge_genesis_block ::: $nodes ::: ${proof[1]} ::: ${proof[0]} ::: $(join_by , ${peer_addrs[@]}) + +# show which node is in the consensus group +exported_genesis_file="/tmp/genesis_$(date +%Y%m%d%H%M%S)" +non_consensus_node="" +for node in ${nodes[@]}; do + if [[ $(./_build/testdev\+miner$node/rel/miner$node/bin/miner$node info in_consensus) = *true* ]]; then + echo "miner$node, in_consensus: true" + if [ ! -f $exported_genesis_file ]; then + ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node genesis export $exported_genesis_file + fi + else + echo "miner$node, in_consensus: false" + non_consensus_node+=" $node" + fi +done +echo "Node not in consensus: $non_consensus_node" + +if [ -f $exported_genesis_file ]; then + echo "Exported Genesis file: $exported_genesis_file" + + echo "Loading Genesis block on $non_consensus_node" + for node in $non_consensus_node; do + ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node genesis load $exported_genesis_file + done + + # check everyone has the chain now + for node in ${nodes[@]}; do + ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node info height > /dev/null + if [[ $? -ne 0 ]]; then + ./_build/testdev\+miner$node/rel/miner$node/bin/miner$node genesis load $exported_genesis_file + fi + done +else + echo "couldn't export genesis file" + exit 1 +fi