From 8a0ae4b92c54514b453dddbdee65ed9c8bfccefd Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 13:03:28 -0800 Subject: [PATCH 01/13] chore: lower time for fuzz/race tests --- .github/workflows/ci-core.yml | 5 +++-- tools/bin/go_core_fuzz | 3 ++- tools/bin/go_core_race_tests | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 3d7050197a6..2fdb2483890 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -220,12 +220,13 @@ jobs: go install ./pkg/chainlink/cmd/chainlink-starknet popd - - name: Increase Race Timeout - # Increase race timeout for scheduled runs only + - name: Increase Timeouts for Fuzz/Race + # Increase timeouts for scheduled runs only if: ${{ github.event.schedule != '' && needs.filter.outputs.should-run-ci-core == 'true' }} run: | echo "TIMEOUT=10m" >> $GITHUB_ENV echo "COUNT=50" >> $GITHUB_ENV + echo "FUZZ_SECONDS=500">> $GITHUB_ENV - name: Install gotestloghelper if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index eb0334fe7ca..7cf7a4ddbc0 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -5,13 +5,14 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m} +FUZZ_SECONDS=${FUZZ_SECONDS:-"300"} echo "Failed fuzz tests and panics: ---------------------" echo "" # the amount of --seconds here is subject to change based on how long the CI job takes in the future # as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. # We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed) -timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds 420 --go_module_root ./ | tee $OUTPUT_FILE +timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index d09a8d903e4..678837ae5e8 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -TIMEOUT="${TIMEOUT:-30s}" +TIMEOUT="${TIMEOUT:-20s}" COUNT="${COUNT:-10}" echo "Failed tests and panics: ---------------------" From c30badb99587080b2dfb442a1298679d38bc4709 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 13:24:30 -0800 Subject: [PATCH 02/13] fix: use fuzz timeout directly --- .github/workflows/ci-core.yml | 2 +- tools/bin/go_core_fuzz | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 2fdb2483890..88dc1693d65 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -226,7 +226,7 @@ jobs: run: | echo "TIMEOUT=10m" >> $GITHUB_ENV echo "COUNT=50" >> $GITHUB_ENV - echo "FUZZ_SECONDS=500">> $GITHUB_ENV + echo "FUZZ_TIMEOUT_MINUTES=10">> $GITHUB_ENV - name: Install gotestloghelper if: ${{ needs.filter.outputs.should-run-ci-core == 'true' }} diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 7cf7a4ddbc0..ee48b05f93e 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,15 +4,21 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -FUZZ_TIMEOUT=${FUZZ_TIMEOUT:-10m} -FUZZ_SECONDS=${FUZZ_SECONDS:-"300"} +FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"6"} + +TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) +if (( TOTAL_SECONDS > 60 )); then + FUZZ_SECONDS=$((TOTAL_SECONDS - 60)) +else + FUZZ_SECONDS=$TOTAL_SECONDS +fi echo "Failed fuzz tests and panics: ---------------------" echo "" # the amount of --seconds here is subject to change based on how long the CI job takes in the future # as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. # We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed) -timeout "${FUZZ_TIMEOUT}" ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE +timeout "${FUZZ_TIMEOUT_MINUTES}"m ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} # Assert no known sensitive strings present in test logger output From c0402741c963c6f4ffd4911d09a78d7497ade070 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 15:40:35 -0800 Subject: [PATCH 03/13] fix: greater gap for fuzz test seconds vs timeout --- tools/bin/go_core_fuzz | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index ee48b05f93e..f5ae76c1ca7 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -7,17 +7,16 @@ OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"6"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) -if (( TOTAL_SECONDS > 60 )); then - FUZZ_SECONDS=$((TOTAL_SECONDS - 60)) +if (( TOTAL_SECONDS > 120 )); then + # Allow for a 90 second buffer between the timeout, and fuzz test runtime + FUZZ_SECONDS=$((TOTAL_SECONDS - 90)) else - FUZZ_SECONDS=$TOTAL_SECONDS + echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES" + exit 1 fi echo "Failed fuzz tests and panics: ---------------------" echo "" -# the amount of --seconds here is subject to change based on how long the CI job takes in the future -# as we add more fuzz tests, we should take into consideration increasing this timelapse, so we can have enough coverage. -# We are timing out after ~10mins in case the tests hang. (Current CI duration is ~8m, modify if needed) timeout "${FUZZ_TIMEOUT_MINUTES}"m ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE EXITCODE=${PIPESTATUS[0]} From 0fbdfe7eeb45a6c5104f01036eb71cb4028cc38a Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 15:52:38 -0800 Subject: [PATCH 04/13] fix: greater gap for fuzz test seconds vs timeout --- tools/bin/go_core_fuzz | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index f5ae76c1ca7..2c50a612f90 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -7,14 +7,16 @@ OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"6"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) -if (( TOTAL_SECONDS > 120 )); then - # Allow for a 90 second buffer between the timeout, and fuzz test runtime - FUZZ_SECONDS=$((TOTAL_SECONDS - 90)) +if (( TOTAL_SECONDS >= 180 )); then + # Allow for a 120 second buffer between the timeout, and fuzz test runtime + FUZZ_SECONDS=$((TOTAL_SECONDS - 120)) else - echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES" + echo "Increase FUZZ_TIMEOUT_MINUTES to >=3, received $FUZZ_TIMEOUT_MINUTES" exit 1 fi +echo "timeout minutes: $FUZZ_TIMEOUT_MINUTES" +echo "fuzz seconds: $FUZZ_SECONDS" echo "Failed fuzz tests and panics: ---------------------" echo "" timeout "${FUZZ_TIMEOUT_MINUTES}"m ./fuzz/fuzz_all_native.py --ci --seconds "$FUZZ_SECONDS" --go_module_root ./ | tee $OUTPUT_FILE From 09497d3f1c932a0e2ecd99a5337636c699a9e66e Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 16:17:03 -0800 Subject: [PATCH 05/13] fix: lower fuzz test duration --- fuzz/fuzz_all_native.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fuzz/fuzz_all_native.py b/fuzz/fuzz_all_native.py index aa191fc5e8d..27ba4d419bd 100755 --- a/fuzz/fuzz_all_native.py +++ b/fuzz/fuzz_all_native.py @@ -32,8 +32,10 @@ def main(): print(f"{fuzzfn} in {path}", file=sys.stderr) if args.ci: - # only run each fuzzer once for 60 seconds in CI - durations_seconds = [60] + if env.GITHUB_EVENT_NAME == 'scheduled': + durations_seconds = [60] + else: + duration_seconds = [45] else: # run forever or until --seconds, with increasingly longer durations per fuzz run durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) From aea4fa7154970a2ec70dfac2cedce7e63fdfc8ee Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Wed, 6 Nov 2024 16:25:42 -0800 Subject: [PATCH 06/13] fix: fuzzing timeout calculation --- fuzz/fuzz_all_native.py | 40 ++++++++++++++++++++++++++-------------- tools/bin/go_core_fuzz | 8 ++++---- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/fuzz/fuzz_all_native.py b/fuzz/fuzz_all_native.py index 27ba4d419bd..4e7358c42a7 100755 --- a/fuzz/fuzz_all_native.py +++ b/fuzz/fuzz_all_native.py @@ -6,6 +6,7 @@ import re import subprocess import sys +import time def main(): parser = argparse.ArgumentParser( @@ -22,37 +23,48 @@ def main(): # use float for remaining_seconds so we can represent infinity if args.seconds: - remaining_seconds = float(args.seconds) + total_time = float(args.seconds) else: - remaining_seconds = float("inf") + total_time = float("inf") + + start_time = time.time() + remaining_seconds = total_time fuzzers = discover_fuzzers(args.go_module_root) - print(f"🐝 Discovered fuzzers:", file=sys.stderr) + num_fuzzers = len(fuzzers) + print(f"🐝 Discovered {num_fuzzers} fuzzers:", file=sys.stderr) for fuzzfn, path in fuzzers.items(): print(f"{fuzzfn} in {path}", file=sys.stderr) + if num_fuzzers == 0: + print(f"No fuzzers found, this is likely an error. Exiting.") + exit(1) + + # run forever or until --seconds, with increasingly longer durations per fuzz run + durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) if args.ci: - if env.GITHUB_EVENT_NAME == 'scheduled': - durations_seconds = [60] - else: - duration_seconds = [45] - else: - # run forever or until --seconds, with increasingly longer durations per fuzz run - durations_seconds = itertools.chain([5, 10, 30, 90, 270], itertools.repeat(600)) + # In CI - default to 60s fuzzes for scheduled runs, and 45 seconds for everything else + durations_seconds = [60] if os.getenv('GITHUB_EVENT_NAME') == 'scheduled' else [45] + if args.seconds: + # However, if seconds was specified, evenly divide total time among all fuzzers + # leaving a 5 second buffer for processing/building time between fuzz runs + actual_fuzz_time = total_time - (num_fuzzers * 5) + durations_seconds = [ actual_fuzz_time / num_fuzzers ] for duration_seconds in durations_seconds: print(f"🐝 Running each fuzzer for {duration_seconds}s before switching to next fuzzer", file=sys.stderr) for fuzzfn, path in fuzzers.items(): + elapsed_time = time.time() - start_time + remaining_seconds = total_time - elapsed_time + if remaining_seconds <= 0: print(f"🐝 Time budget of {args.seconds}s is exhausted. Exiting.", file=sys.stderr) return next_duration_seconds = min(remaining_seconds, duration_seconds) - remaining_seconds -= next_duration_seconds - - print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s before switching to next fuzzer", file=sys.stderr) + print(f"🐝 Running {fuzzfn} in {path} for {next_duration_seconds}s (Elapsed: {elapsed_time:.2f}s, Remaining: {remaining_seconds:.2f}s)", file=sys.stderr) run_fuzzer(fuzzfn, path, next_duration_seconds, args.go_module_root) - print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s. Total remaining time is {remaining_seconds}s", file=sys.stderr) + print(f"🐝 Completed running {fuzzfn} in {path} for {next_duration_seconds}s.", file=sys.stderr) def discover_fuzzers(go_module_root): fuzzers = {} diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 2c50a612f90..8ea3b8eaa54 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -7,11 +7,11 @@ OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"6"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) -if (( TOTAL_SECONDS >= 180 )); then - # Allow for a 120 second buffer between the timeout, and fuzz test runtime - FUZZ_SECONDS=$((TOTAL_SECONDS - 120)) +if (( TOTAL_SECONDS >= 120 )); then + # Allow for a 60 second buffer between the timeout, and fuzz test runtime + FUZZ_SECONDS=$((TOTAL_SECONDS - 60)) else - echo "Increase FUZZ_TIMEOUT_MINUTES to >=3, received $FUZZ_TIMEOUT_MINUTES" + echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES" exit 1 fi From 2763f0238db2a6f4613bcdba45a9a425c813b64e Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 12:02:25 -0800 Subject: [PATCH 07/13] adjust runtimes --- tools/bin/go_core_fuzz | 2 +- tools/bin/go_core_race_tests | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 8ea3b8eaa54..2aecf6711db 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,7 +4,7 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"6"} +FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"5"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) if (( TOTAL_SECONDS >= 120 )); then diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index 678837ae5e8..93ed8448aa3 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -2,7 +2,7 @@ set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} TIMEOUT="${TIMEOUT:-20s}" -COUNT="${COUNT:-10}" +COUNT="${COUNT:-8}" echo "Failed tests and panics: ---------------------" echo "" From ab1e45271e8825367f45dca2a3c569cf7678846d Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 12:14:35 -0800 Subject: [PATCH 08/13] Adjust fuzzing runtime --- fuzz/fuzz_all_native.py | 7 +++++-- tools/bin/go_core_fuzz | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fuzz/fuzz_all_native.py b/fuzz/fuzz_all_native.py index 4e7358c42a7..2d1cc4ccb29 100755 --- a/fuzz/fuzz_all_native.py +++ b/fuzz/fuzz_all_native.py @@ -47,8 +47,11 @@ def main(): durations_seconds = [60] if os.getenv('GITHUB_EVENT_NAME') == 'scheduled' else [45] if args.seconds: # However, if seconds was specified, evenly divide total time among all fuzzers - # leaving a 5 second buffer for processing/building time between fuzz runs - actual_fuzz_time = total_time - (num_fuzzers * 5) + # leaving a 10 second buffer for processing/building time between fuzz runs + actual_fuzz_time = total_time - (num_fuzzers * 10) + if actual_fuzz_time <= 5 * num_fuzzers: + print(f"Seconds (--seconds {arg.seconds}) is too low to properly run fuzzers for 5sec each. Exiting.") + exit(1) durations_seconds = [ actual_fuzz_time / num_fuzzers ] for duration_seconds in durations_seconds: diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 2aecf6711db..9b7d5cfea7a 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -8,8 +8,8 @@ FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"5"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) if (( TOTAL_SECONDS >= 120 )); then - # Allow for a 60 second buffer between the timeout, and fuzz test runtime - FUZZ_SECONDS=$((TOTAL_SECONDS - 60)) + # Allow for a 30 second buffer between the timeout, and fuzz test runtime + FUZZ_SECONDS=$((TOTAL_SECONDS - 30)) else echo "Increase FUZZ_TIMEOUT_MINUTES to >=2, received $FUZZ_TIMEOUT_MINUTES" exit 1 From e4c63f67378868fc5747b00413074a6b70934872 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 12:41:24 -0800 Subject: [PATCH 09/13] adjust race time --- tools/bin/go_core_race_tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index 93ed8448aa3..65e7c961b1d 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -TIMEOUT="${TIMEOUT:-20s}" -COUNT="${COUNT:-8}" +TIMEOUT="${TIMEOUT:-15s}" +COUNT="${COUNT:-5}" echo "Failed tests and panics: ---------------------" echo "" From fd83712be827a05014e6dee1b6da624740f54362 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 13:55:36 -0800 Subject: [PATCH 10/13] chore: further lower race/fuzz times --- tools/bin/go_core_fuzz | 2 +- tools/bin/go_core_race_tests | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index 9b7d5cfea7a..d5bc7268106 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,7 +4,7 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"5"} +FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"4"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) if (( TOTAL_SECONDS >= 120 )); then diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index 65e7c961b1d..a50144688fa 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -ex OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -TIMEOUT="${TIMEOUT:-15s}" +TIMEOUT="${TIMEOUT:-10s}" COUNT="${COUNT:-5}" echo "Failed tests and panics: ---------------------" From 6d375691630341d1be216566841fd9ae521ae3a6 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 14:05:32 -0800 Subject: [PATCH 11/13] remove -json for non-debug runs --- tools/bin/go_core_race_tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/bin/go_core_race_tests b/tools/bin/go_core_race_tests index a50144688fa..2c4071bc20f 100755 --- a/tools/bin/go_core_race_tests +++ b/tools/bin/go_core_race_tests @@ -10,13 +10,13 @@ if [[ $GITHUB_EVENT_NAME == "schedule" ]]; then if [[ $DEBUG == "true" ]]; then GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE else - GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE + GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE fi else if [[ $DEBUG == "true" ]]; then GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | tee $OUTPUT_FILE else - GORACE="log_path=$PWD/race" go test -json -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE + GORACE="log_path=$PWD/race" go test -vet=off -race -shuffle on -timeout "$TIMEOUT" -count "$COUNT" $1 | cat > $OUTPUT_FILE fi fi EXITCODE=${PIPESTATUS[0]} From 34922bebd622b6d1f82dca44c3253eba412af95a Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 14:42:11 -0800 Subject: [PATCH 12/13] test: enable build cache for race test --- .github/workflows/ci-core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 88dc1693d65..3a32d7e12c7 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -166,7 +166,7 @@ jobs: uses: ./.github/actions/setup-go with: # race/fuzz tests don't benefit repeated caching, so restore from develop's build cache - restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_race_tests' || matrix.type.cmd == 'go_core_fuzz' }} + restore-build-cache-only: ${{ matrix.type.cmd == 'go_core_fuzz' }} build-cache-version: ${{ matrix.type.cmd }} - name: Replace chainlink-evm deps From cd071fb2b803631c69259ef429aad1f8435e1df8 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 7 Nov 2024 17:01:01 -0800 Subject: [PATCH 13/13] lower fuzz test timeout further --- tools/bin/go_core_fuzz | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/go_core_fuzz b/tools/bin/go_core_fuzz index d5bc7268106..49aaf33b65e 100755 --- a/tools/bin/go_core_fuzz +++ b/tools/bin/go_core_fuzz @@ -4,7 +4,7 @@ set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} -FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"4"} +FUZZ_TIMEOUT_MINUTES=${FUZZ_TIMEOUT_MINUTES:-"3"} TOTAL_SECONDS=$((FUZZ_TIMEOUT_MINUTES * 60)) if (( TOTAL_SECONDS >= 120 )); then