diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 3d7050197a6..3a32d7e12c7 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -166,7 +166,7 @@ jobs: uses: ./.github/actions/setup-go with: # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache - restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_race_tests' || matrix.type.cmd == 'go_core_fuzz' }} + restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }} build-cache-version: ${{ matrix.type.cmd }} - name: Replace chainlink-evm deps @@ -220,12 +220,13 @@ jobs: go install ./pkg/chainlink/cmd/chainlink-starknet popd - - name: Increase Race Timeout - # Increase race timeout for scheduled runs only + - name: Increase Timeouts for Fuzz/Race + # Increase timeouts for scheduled runs only if: ${{ github.event.schedule != '' && needs.filter.outputs.should-run-ci-core == 'true' }} run: | echo "TIMEOUT=10m" >> $GITHUB_ENV echo "COUNT=50" >> $GITHUB_ENV + echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV - name: Install gotestloghelper if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} diff --git a/fuzz/fuzz_all_native.py b/fuzz/fuzz_all_native.py index aa191fc5e8d..2d1cc4ccb29 100755 --- a/fuzz/fuzz_all_native.py +++ b/fuzz/fuzz_all_native.py @@ -6,6 +6,7 @@ import re import subprocess import sys +import time def main(): parser = argparse.ArgumentParser( @@ -22,35 +23,51 @@ def main(): # use float for remaining_seconds so we can represent infinity if args.seconds: - remaining_seconds = float(args.seconds) + total_time = float(args.seconds) else: - remaining_seconds = float("inf") + total_time = float("inf") + + start_time = time.time() + remaining_seconds = total_time fuzzers = discover_fuzzers(args.go_module_root) - print(f"🐝 Discovered fuzzers:", file=sys.stderr) + num_fuzzers = len(fuzzers) + print(f"🐝 Discovered {num_fuzzers} fuzzers:", file=sys.stderr) for fuzzfn, path in fuzzers.items(): print(f"{fuzzfn} in {path}", file=sys.stderr) + if num_fuzzers == 0: + print(f"No fuzzers found, this is likely an error. Exiting.") + exit(1) + + # run forever or until --seconds, with increasingly longer durations per fuzz run + durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) if args.ci: - # only run each fuzzer once for 60 seconds in CI - durations_seconds = [60] - else: - # run forever or until --seconds, with increasingly longer durations per fuzz run - durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) + # In CI - default to 60s fuzzes for scheduled runs, and 45 seconds for everything else + durations_seconds = [60] if os.getenv('GITHUB_EVENT_NAME') == 'scheduled' else [45] + if args.seconds: + # However, if seconds was specified, evenly divide total time among all fuzzers + # leaving a 10 second buffer for processing/building time between fuzz runs + actual_fuzz_time = total_time - (num_fuzzers * 10) + if actual_fuzz_time <= 5 * num_fuzzers: + print(f"Seconds (--seconds {arg.seconds}) is too low to properly run fuzzers for 5sec each. Exiting.") + exit(1) + durations_seconds = [ actual_fuzz_time / num_fuzzers ] for duration_seconds in durations_seconds: print(f"🐝 Running each fuzzer for {duration_seconds}s before switching to next fuzzer", file=sys.stderr) for fuzzfn, path in fuzzers.items(): + elapsed_time = time.time() - start_time + remaining_seconds = total_time - elapsed_time + if remaining_seconds <= 0: print(f"🐝 Time budget of {args.seconds}s is exhausted. Exiting.", file=sys.stderr) return next_duration_seconds = min(remaining_seconds, duration_seconds) - remaining_seconds -= next_duration_seconds - - print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr) + print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s (Elapsed: {elapsed_time:.2f}s, Remaining: {remaining_seconds:.2f}s)", file=sys.stderr) run_fuzzer(fuzzfn, path, next_duration_seconds, args.go_module_root) - print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr) + print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s.", file=sys.stderr) def discover_fuzzers(go_module_root): fuzzers = {} diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index eb0334fe7ca..49aaf33b65e 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,14 +4,22 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m} +FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"3"} +TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) +if (( TOTAL_SECONDS >= 120 )); then + # Allow for a 30 second buffer between the timeout, and fuzz test runtime + FUZZ_SECONDS=$((TOTAL_SECONDS - 30)) +else + echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES" + exit 1 +fi + +echo "timeout minutes: $FUZZ_TIMEOUT_MINUTES" +echo "fuzz seconds: $FUZZ_SECONDS" echo "Failed fuzz tests and panics: ---------------------" echo "" -# the amount of --seconds here is subject to change based on how long the CI job takes in the future -# as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. -# We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed) -timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds 420 --go_module_root ./ | tee $OUTPUT_FILE +timeout "${FUZZ_TIMEOUT_MINUTES}"m ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index d09a8d903e4..2c4071bc20f 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -TIMEOUT="${TIMEOUT:-30s}" -COUNT="${COUNT:-10}" +TIMEOUT="${TIMEOUT:-10s}" +COUNT="${COUNT:-5}" echo "Failed tests and panics: ---------------------" echo "" @@ -10,13 +10,13 @@ if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then if [[ $DEBUG == "true" ]]; then GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE else - GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE + GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE fi else if [[ $DEBUG == "true" ]]; then GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE else - GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE + GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE fi fi EXITCODE=${PIPESTATUS[0]}