-
Notifications
You must be signed in to change notification settings - Fork 63
235 lines (200 loc) · 10.5 KB
/
rpc-performance-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
name: QA - RPC Performance Tests
on:
workflow_dispatch:
schedule:
- cron: '0 0 * * 1-6' # Runs every day from Monday to Saturday at 00:00 AM UTC
jobs:
performance-test-suite:
runs-on: self-hosted
env:
ERIGON_DIR: /opt/erigon-versions/reference-version
ERIGON_DATA_DIR: /opt/erigon-versions/reference-version/datadir
RPC_PAST_TEST_DIR: /opt/rpc-past-tests
ERIGON_QA_PATH: /opt/erigon-qa
steps:
- name: Checkout Silkworm repository
uses: actions/checkout@v3
with:
submodules: recursive
fetch-depth: "0"
- name: Checkout RPC Tests Repository & Install Requirements
run: |
rm -rf ${{runner.workspace}}/rpc-tests
git -c advice.detachedHead=false clone --depth 1 --branch v0.26.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
cd ${{runner.workspace}}/rpc-tests
pip3 install -r requirements.txt
- name: Clean Build Directory
run: rm -rf ${{runner.workspace}}/silkworm/build
- name: Create Build Environment
run: cmake -E make_directory ${{runner.workspace}}/silkworm/build
- name: Configure CMake
working-directory: ${{runner.workspace}}/silkworm/build
run: |
cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Release
- name: Build Silkworm RpcDaemon
working-directory: ${{runner.workspace}}/silkworm/build
run: cmake --build . --config Release --target rpcdaemon -j 8
- name: Pause the Erigon instance dedicated to db maintenance
run: |
python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
- name: Run Silkworm RpcDaemon
working-directory: ${{runner.workspace}}/silkworm/build/cmd
run: |
./rpcdaemon --datadir $ERIGON_DATA_DIR --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt ./jwt.hex --skip_protocol_check --http-compression --eth.addr 127.0.0.1:51515 &
SILKWORM_RPC_DAEMON_PID=$!
echo "SILKWORM_RPC_DAEMON_PID=$SILKWORM_RPC_DAEMON_PID" >> $GITHUB_ENV
- name: Run Erigon RpcDaemon
run: |
$ERIGON_DIR/rpcdaemon --datadir $ERIGON_DATA_DIR --http.api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --verbosity 1 &
ERIGON_RPC_DAEMON_PID=$!
echo "ERIGON_RPC_DAEMON_PID=$ERIGON_RPC_DAEMON_PID" >> $GITHUB_ENV
- name: Run RPC Performance Tests
id: test_step
run: |
set +e # Disable exit on error
failed_test=0
commit=$(git -C ${{runner.workspace}}/silkworm rev-parse --short HEAD) # use ${{ github.sha }} or GITHUB_SHA
past_test_dir=$RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_perf_$commit
echo "past_test_dir=$past_test_dir" >> $GITHUB_ENV
# Prepare historical test results directory
mkdir -p $past_test_dir
rm -rf $RPC_PAST_TEST_DIR/mainnet_bin # we want only the latest binary files
mkdir -p $RPC_PAST_TEST_DIR/mainnet_bin
run_perf () {
servers=("silkworm" "erigon")
for i in 1 2
do
network=$1
method=$2
pattern=$3
sequence=$4
# clean temporary area
cd ${{runner.workspace}}/rpc-tests/perf
rm -rf ./reports/
python3 ./run_perf_tests.py --blockchain "$network" \
--test-type "$method" \
--pattern-file pattern/"$network"/"$pattern".tar \
--test-sequence "$sequence" \
--repetitions 5 \
--silk-dir ${{runner.workspace}}/silkworm \
--erigon-dir $ERIGON_DATA_DIR \
--test-mode $i \
--test-report \
--json-report ./reports/mainnet/result.json \
--testing-daemon ${servers[i-1]}
# Capture test runner script exit status
perf_exit_status=$?
# Preserve test results
mv ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/result.json ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json
# Detect the pre-built db version
db_version=$(python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/prod_info.py $ERIGON_DIR/production.ini production erigon_repo_commit)
# Check test runner script exit status
if [ $perf_exit_status -eq 0 ]; then
# save all vegeta binary report
echo "Save current vegeta binary files"
cp -r ${{runner.workspace}}/rpc-tests/perf/reports/bin $RPC_PAST_TEST_DIR/mainnet_bin
echo "Save test result on DB"
cd ${{runner.workspace}}/silkworm
python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
--repo silkworm \
--branch ${{ github.ref_name }} \
--commit $(git rev-parse HEAD) \
--test_name rpc-performance-test-${servers[i-1]}-$method \
--chain mainnet \
--runner ${{ runner.name }} \
--db_version $db_version \
--outcome success \
--result_file ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/${servers[i-1]}-$method-result.json
if [ $? -ne 0 ]; then
failed_test=1
echo "Failure saving test results on DB"
fi
echo "Execute Latency Percentile HDR Analysis"
cd ${{runner.workspace}}/rpc-tests/perf/reports/mainnet/
python3 $ERIGON_QA_PATH/test_system/qa-tests/rpc-tests/perf_hdr_analysis.py \
--test_name ${servers[i-1]}-$method \
--input_file ./result.json \
--output_file ./${servers[i-1]}-${method}-latency_hdr_analysis.pdf
else
failed_test=1
cd ${{runner.workspace}}/silkworm
python3 $ERIGON_QA_PATH/test_system/qa-tests/uploads/upload_test_results.py \
--repo silkworm \
--branch ${{ github.ref_name }} \
--commit $(git rev-parse HEAD) \
--test_name rpc-performance-test-${servers[i-1]}-$method \
--chain mainnet \
--runner ${{ runner.name }} \
--db_version $db_version \
--outcome failure
fi
# Save test results to a directory with timestamp and commit hash
cp -r ${{runner.workspace}}/rpc-tests/perf/reports/mainnet $past_test_dir
done
}
# Launch the RPC performance test runner
failed_test= 0
run_perf mainnet eth_call stress_test_eth_call_001_14M 1:1,100:30,1000:20,10000:20,20000:20
run_perf mainnet eth_getLogs stress_test_eth_getLogs_15M 1:1,100:30,1000:20,10000:20,20000:20
run_perf mainnet eth_getBalance stress_test_eth_getBalance_15M 1:1,100:30,1000:20,10000:20,20000:20
run_perf mainnet eth_getBlockByHash stress_test_eth_getBlockByHash_14M 1:1,100:30,1000:20,10000:20
run_perf mainnet eth_getBlockByNumber stress_test_eth_getBlockByNumber_13M 1:1,100:30,1000:20,5000:20
run_perf mainnet eth_getTransactionByHash stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20
run_perf mainnet eth_getTransactionReceipt stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20
run_perf mainnet eth_createAccessList stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20
if [ $failed_test -eq 0 ]; then
echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
echo "Tests completed successfully"
else
echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
echo "Error detected during tests"
fi
- name: Stop Silkworm RpcDaemon
working-directory: ${{runner.workspace}}/silkworm/build/cmd
run: |
# Clean up process if it's still running
if kill -0 $SILKWORM_RPC_DAEMON_PID 2> /dev/null; then
echo "Terminating Silkworm RpcDaemon"
kill $SILKWORM_RPC_DAEMON_PID
else
echo "Silkworm RpcDaemon has already terminated"
fi
- name: Stop Erigon RpcDaemon
run: |
# Clean up process if it's still running
if kill -0 $ERIGON_RPC_DAEMON_PID 2> /dev/null; then
echo "Terminating Erigon RpcDaemon"
kill $ERIGON_RPC_DAEMON_PID
else
echo "Erigon RpcDaemon has already terminated"
fi
- name: Resume the Erigon instance dedicated to db maintenance
run: |
python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
- name: Run change point analysis
if: steps.test_step.outputs.TEST_RESULT == 'success'
working-directory: ${{runner.workspace}}/rpc-tests/perf/reports/mainnet
run: |
set +e # Disable exit on error
open_change_points=0
python3 $ERIGON_QA_PATH/test_system/qa-tests/change-points/change_point_analysis.py
open_change_points=$?
cp change_point_analysis.pdf $past_test_dir
if [ $open_change_points -ne 0 ]; then
echo "Change point analysis found points that need to be investigated"
#echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT" -- enable in the future
fi
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results
path: ${{ env.past_test_dir }}
- name: Action for Success
if: steps.test_step.outputs.TEST_RESULT == 'success'
run: echo "::notice::Tests completed successfully"
- name: Action for Not Success
if: steps.test_step.outputs.TEST_RESULT != 'success'
run: |
echo "::error::Error detected during tests"
exit 1