diff --git a/.github/actions/setup-ubuntu/action.yaml b/.github/actions/setup-ubuntu/action.yaml new file mode 100644 index 000000000..298ce15f9 --- /dev/null +++ b/.github/actions/setup-ubuntu/action.yaml @@ -0,0 +1,29 @@ +--- +name: setup-ubuntu +runs: + using: 'composite' + steps: + - name: Install dependencies + # Shell must explicitly specify the shell for each step. https://github.com/orgs/community/discussions/18597 + shell: bash + run: sudo apt install -y make direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + shell: bash + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff + - name: Detect required Go version + shell: bash + run: | + toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') + echo "GOVERSION=${toolchain}" >> $GITHUB_ENV + - uses: actions/setup-go@v5 + shell: bash + with: + go-version: "${{ env.GOVERSION }}" + check-latest: true + - name: set environment + shell: bash + uses: HatsuneMiku3939/direnv-action@v1 + with: + masks: '' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dce7dc468..6987724b1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -16,6 +16,10 @@ jobs: - run: git fetch --prune --unshallow - name: Install dependencies run: sudo apt install -y make direnv unzip lz4 wget curl npm jq pv coreutils + - name: Install git-cliff + uses: baptiste0928/cargo-install@v3 + with: + crate: git-cliff - name: Detect required Go version run: | toolchain=$(./script/tools.sh gotoolchain | sed 's/go*//') diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b01518a27..028f87e87 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -22,10 +22,6 @@ jobs: run: | brew install bash direnv pv lz4 git-cliff sudo chsh -s /usr/local/bin/bash - - name: Install git-cliff - uses: baptiste0928/cargo-install@v3 - with: - crate: git-cliff - name: Hook direnv to bash run: echo 'eval "$(direnv hook bash)"' >> $HOME/.bashrc - uses: actions/checkout@v4 @@ -43,6 +39,7 @@ jobs: direnv allow direnv export gha >> "$GITHUB_ENV" - run: make bins + build-bins: runs-on: ubuntu-latest steps: @@ -64,8 +61,6 @@ jobs: check-latest: true - name: set environment uses: HatsuneMiku3939/direnv-action@v1 - with: - masks: '' - run: make bins - run: make docker-image diff --git a/script/tools.sh b/script/tools.sh index 6a3071902..478aaba9c 100755 --- a/script/tools.sh +++ b/script/tools.sh @@ -5,6 +5,30 @@ SEMVER=$SCRIPT_DIR/semver.sh gomod="$SCRIPT_DIR/../go.mod" +macos_deps=( + "bash" + "direnv" + "pv" + "lz4" + "git-cliff" +) + +debian_deps=( + "make" + "build-essentials" + "direnv" + "unzip" + "wget" + "curl" + "npm" + "jq" + "coreutils" +) + +is_command() { + command -v "$1" >/dev/null +} + function get_gotoolchain() { local gotoolchain local goversion @@ -117,6 +141,80 @@ function replace_import_path() { replace_paths "./go/go.mod" "$curr_module_name" "$new_module_name" } +function install_gha_deps() { + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Detected Darwin based system" + + if ! is_command brew; then + echo "homebrew is not installed. visit https://brew.sh" + exit 1 + fi + + local tools + + if ! is_command make || [[ $(make --version | head -1 | cut -d" " -f3 | cut -d"." -f1) -lt 4 ]]; then + tools="$tools make" + fi + + # shellcheck disable=SC2068 + for dep in ${macos_deps[@]}; do + echo -n "detecting $dep ..." + status="(installed)" + if ! brew list "$dep" >/dev/null 2>&1; then + tools="$tools $dep" + status="(not installed)" + fi + + echo " $status" + done + + if [[ "$tools" != "" ]]; then + # don't put quotes around $tools! + # shellcheck disable=SC2086 + brew install $tools + else + echo "All requirements already met. Nothing to install" + fi + elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + if is_command dpkg; then + echo "Detected Debian based system" + local tools + + # shellcheck disable=SC2068 + for dep in ${debian_deps[@]}; do + echo -n "detecting $dep ..." + status="(installed)" + if ! dpkg -l "$dep"; then + tools="$tools $dep" + status="(not installed)" + fi + echo " $status" + done + + cmd="apt-get" + + if is_command sudo; then + cmd="sudo $cmd" + fi + + if [[ "$tools" != "" ]]; then + $cmd update + # don't put quotes around $tools! + # shellcheck disable=SC2086 + ( + set -x + $cmd install -y $tools + ) + else + echo "All requirements already met. Nothing to install" + fi + fi + else + echo "Unsupported OS $OSTYPE" + exit 1 + fi +} + case "$1" in gotoolchain) get_gotoolchain diff --git a/tests/upgrade/upgrade_test.go b/tests/upgrade/upgrade_test.go index 8c26a3dc8..8debe54d5 100644 --- a/tests/upgrade/upgrade_test.go +++ b/tests/upgrade/upgrade_test.go @@ -35,7 +35,7 @@ import ( ) const ( - blockTimeWindow = 7 * time.Second + blockTimeWindow = 180 * time.Second ) type nodeEvent int @@ -65,7 +65,8 @@ const ( const ( nodeTestStagePreUpgrade nodeTestStage = iota nodeTestStageUpgrade - nodeTestStagePostUpgrade + nodeTestStagePostUpgrade1 + nodeTestStagePostUpgrade2 ) const ( @@ -86,9 +87,10 @@ type publisher interface { var ( nodeTestStageMapStr = map[nodeTestStage]string{ - nodeTestStagePreUpgrade: "preupgrade", - nodeTestStageUpgrade: "upgrade", - nodeTestStagePostUpgrade: "postupgrade", + nodeTestStagePreUpgrade: "preupgrade", + nodeTestStageUpgrade: "upgrade", + nodeTestStagePostUpgrade1: "postupgrade1", + nodeTestStagePostUpgrade2: "postupgrade2", } testModuleStatusMapStr = map[testModuleStatus]string{ @@ -193,7 +195,7 @@ type validatorParams struct { rpcPort uint16 upgradeName string env []string - pub pubsub.Publisher + bus pubsub.Publisher } type validator struct { @@ -439,6 +441,8 @@ func TestUpgrade(t *testing.T) { } } + listenAddr := "127.0.0.1" + for name, params := range initParams { var unconditionalPeerIDs string var persistentPeers string @@ -449,7 +453,7 @@ func TestUpgrade(t *testing.T) { } unconditionalPeerIDs += params1.nodeID + "," - persistentPeers += fmt.Sprintf("%s@127.0.0.1:%d,", params1.nodeID, params1.p2pPort) + persistentPeers += fmt.Sprintf("%s@%s:%d,", params1.nodeID, listenAddr, params1.p2pPort) } validatorsParams[name] = validatorParams{ @@ -462,7 +466,7 @@ func TestUpgrade(t *testing.T) { p2pPort: params.p2pPort, rpcPort: params.rpc.port, upgradeName: *upgradeName, - pub: bus, + bus: bus, env: []string{ fmt.Sprintf("DAEMON_NAME=akash"), fmt.Sprintf("DAEMON_HOME=%s", params.homedir), @@ -478,12 +482,12 @@ func TestUpgrade(t *testing.T) { fmt.Sprintf("AKASH_P2P_SEEDS=%s", strings.TrimSuffix(persistentPeers, ",")), fmt.Sprintf("AKASH_P2P_PERSISTENT_PEERS=%s", strings.TrimSuffix(persistentPeers, ",")), fmt.Sprintf("AKASH_P2P_UNCONDITIONAL_PEER_IDS=%s", strings.TrimSuffix(unconditionalPeerIDs, ",")), - fmt.Sprintf("AKASH_P2P_LADDR=tcp://127.0.0.1:%d", params.p2pPort), - fmt.Sprintf("AKASH_RPC_LADDR=tcp://127.0.0.1:%d", params.rpc.port), - fmt.Sprintf("AKASH_RPC_GRPC_LADDR=tcp://127.0.0.1:%d", params.rpc.grpc), - fmt.Sprintf("AKASH_RPC_PPROF_LADDR=localhost:%d", params.pprofPort), - fmt.Sprintf("AKASH_GRPC_ADDRESS=tcp://127.0.0.1:%d", params.grpcPort), - fmt.Sprintf("AKASH_GRPC_WEB_ADDRESS=tcp://127.0.0.1:%d", params.grpcWebPort), + fmt.Sprintf("AKASH_P2P_LADDR=tcp://%s:%d", listenAddr, params.p2pPort), + fmt.Sprintf("AKASH_RPC_LADDR=tcp://%s:%d", listenAddr, params.rpc.port), + fmt.Sprintf("AKASH_RPC_GRPC_LADDR=tcp://%s:%d", listenAddr, params.rpc.grpc), + fmt.Sprintf("AKASH_RPC_PPROF_LADDR=%s:%d", listenAddr, params.pprofPort), + fmt.Sprintf("AKASH_GRPC_ADDRESS=tcp://%s:%d", listenAddr, params.grpcPort), + fmt.Sprintf("AKASH_GRPC_WEB_ADDRESS=tcp://%s:%d", listenAddr, params.grpcWebPort), "AKASH_P2P_PEX=true", "AKASH_P2P_ADDR_BOOK_STRICT=false", "AKASH_P2P_ALLOW_DUPLICATE_IP=true", @@ -531,7 +535,9 @@ func TestUpgrade(t *testing.T) { }(name) } + t.Logf("waiting for validator(s) to complete tasks") err = group.Wait() + t.Logf("all validators finished") assert.NoError(t, err) fail := false @@ -548,6 +554,8 @@ func TestUpgrade(t *testing.T) { } } + bus.Close() + if fail { t.Fail() } @@ -611,8 +619,10 @@ loop: }) if !result { - l.t.Error("post upgrade test handler failed") + l.t.Error("post upgrade test FAIL") return fmt.Errorf("post-upgrade check failed") + } else { + l.t.Log("post upgrade test PASS") } return nil @@ -832,6 +842,9 @@ func (l *validator) run() error { } l.group.Go(func() error { + defer l.t.Logf("[%s] log scanner finished", l.params.name) + l.t.Logf("[%s] log scanner started", l.params.name) + defer func() { if r := recover(); r != nil { l.t.Fatal(r) @@ -842,6 +855,9 @@ func (l *validator) run() error { }) l.group.Go(func() error { + defer l.t.Logf("[%s] test case watcher finished", l.params.name) + l.t.Logf("[%s] test case watcher started", l.params.name) + defer func() { if r := recover(); r != nil { l.t.Fatal(r) @@ -857,6 +873,9 @@ func (l *validator) run() error { }) l.group.Go(func() error { + defer l.t.Logf("[%s] stdout reader finished", l.params.name) + l.t.Logf("[%s] stdout reader started", l.params.name) + defer func() { if r := recover(); r != nil { l.t.Fatal(r) @@ -871,6 +890,9 @@ func (l *validator) run() error { }) l.group.Go(func() error { + defer l.t.Logf("[%s] blocks watchdog finished", l.params.name) + l.t.Logf("[%s] blocks watchdog started", l.params.name) + defer func() { if r := recover(); r != nil { l.t.Fatal(r) @@ -882,18 +904,23 @@ func (l *validator) run() error { return err } + defer sub.Close() + return l.blocksWatchdog(l.ctx, sub) }) // state machine l.group.Go(func() error { + defer l.t.Logf("[%s] state machine finished", l.params.name) + l.t.Logf("[%s] state machine started", l.params.name) + defer func() { if r := recover(); r != nil { l.t.Fatal(r) } }() - return l.stateMachine(l.pubsub) + return l.stateMachine() }) err = cmd.Wait() @@ -906,6 +933,7 @@ func (l *validator) run() error { select { case <-l.upgradeSuccessful: err = nil + l.t.Logf("[%s] all workers finished", l.params.name) default: l.t.Logf("[%s] cosmovisor finished with error. check %s", l.params.name, lStderr.Name()) } @@ -913,14 +941,14 @@ func (l *validator) run() error { return err } -func (l *validator) stateMachine(bus pubsub.Bus) error { +func (l *validator) stateMachine() error { defer l.cancel() var err error var sub pubsub.Subscriber - sub, err = bus.Subscribe() + sub, err = l.pubsub.Subscribe() if err != nil { return err } @@ -931,7 +959,7 @@ func (l *validator) stateMachine(bus pubsub.Bus) error { wdCtrl := func(ctx context.Context, ctrl watchdogCtrl) { resp := make(chan struct{}, 1) - _ = bus.Publish(wdReq{ + _ = l.pubsub.Publish(wdReq{ event: ctrl, resp: resp, }) @@ -955,7 +983,7 @@ loop: case nodeEventStart: l.t.Logf("[%s][%s]: node started", l.params.name, nodeTestStageMapStr[stage]) if stage == nodeTestStageUpgrade { - stage = nodeTestStagePostUpgrade + stage = nodeTestStagePostUpgrade1 blocksCount = 0 replayDone = false } @@ -979,14 +1007,15 @@ loop: } if stage == nodeTestStagePreUpgrade && blocksCount == 1 { - _ = l.params.pub.Publish(nodePreUpgradeReady{ + _ = l.params.bus.Publish(nodePreUpgradeReady{ name: l.params.name, }) - } else if stage == nodeTestStagePostUpgrade && blocksCount == 10 { + } else if stage == nodeTestStagePostUpgrade1 && blocksCount >= 10 { + stage = nodeTestStagePostUpgrade2 l.t.Logf("[%s][%s]: counted 10 blocks. signaling has performed upgrade", l.params.name, nodeTestStageMapStr[stage]) l.upgradeSuccessful <- struct{}{} - _ = l.params.pub.Publish(nodePostUpgradeReady{ + _ = l.params.bus.Publish(nodePostUpgradeReady{ name: l.params.name, }) } @@ -1136,14 +1165,23 @@ func (l *validator) blocksWatchdog(ctx context.Context, sub pubsub.Subscriber) e defer func() { if err != nil { l.t.Logf("blocksWatchdog finished with error: %s", err.Error()) + } else { + l.t.Logf("blocksWatchdog finished") } }() + // first few blocks may take a while to produce. + // give a dog generous timeout on them + + blockWindow := 20 * time.Minute + + blocksTm := time.NewTicker(blockWindow) + blocksTm.Stop() + + blocks := 0 + loop: for { - blocksTm := time.NewTicker(blockTimeWindow) - blocksTm.Stop() - select { case <-ctx.Done(): break loop @@ -1159,11 +1197,21 @@ loop: case watchdogCtrlStart: fallthrough case watchdogCtrlBlock: - blocksTm.Reset(blockTimeWindow) + blocks++ + + if blocks > 3 { + blockWindow = blockTimeWindow + } + + blocksTm.Reset(blockWindow) case watchdogCtrlPause: blocksTm.Stop() + blocks = 0 + blockWindow = 20 * time.Minute case watchdogCtrlStop: + blocks = 0 blocksTm.Stop() + blockWindow = 20 * time.Minute break loop } }