-
Notifications
You must be signed in to change notification settings - Fork 5
251 lines (251 loc) · 8.56 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
name: CI
on: [push, pull_request]
jobs:
dependencies:
name: Dependencies
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache dependencies
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
target
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
- name: Build dependencies
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
# We need to manually clean Haura until there is a way to build only dependencies.
# https://github.com/rust-lang/cargo/issues/2644
run: |
cd betree/tests
cargo build --tests
cargo clean --package betree-tests
cd ..
cargo build --tests
cargo clean --package betree_storage_stack
betree-integration:
name: Integration Tests
needs: dependencies
runs-on: ubuntu-22.04
timeout-minutes: 60
env:
RUST_BACKTRACE: 1
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache dependencies
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
target
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
- name: Check dependencies
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: |
exit 1
- name: Run integration tests
env:
HAURA_NUM_THREAD: 1
QUICKCHECK_TESTS: 20
run: |
cd betree/tests
./scripts/ci-test.sh
betree-unit:
name: Unit Tests
needs: dependencies
runs-on: ubuntu-22.04
timeout-minutes: 60
env:
RUST_BACKTRACE: 1
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache dependencies
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
target
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
- name: Check dependencies
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
run: |
exit 1
- name: Run unit tests
env:
HAURA_NUM_THREAD: 8
QUICKCHECK_TESTS: 20
run: |
cd betree
./tests/scripts/ci-test.sh
betree-msrv:
name: MSRV Check
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache MSRV
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-msrv
- name: Prepare JULEA
run: |
sudo apt update || true
sudo apt --yes --no-install-recommends install pkgconf libglib2.0-dev libbson-dev
# FIXME use forked version
git clone --depth 1 --branch modules-conditional-unload https://github.com/tilpner/julea.git "${HOME}/julea"
- name: Install MSRV
run: |
cargo install cargo-msrv
- name: Verify betree
run: |
cd betree
cargo msrv verify
- name: Verify bectl
run: |
cd bectl
cargo msrv verify
- name: Verify julea-betree
run: |
cd julea-betree
export JULEA_INCLUDE="${HOME}/julea/include"
export BINDGEN_EXTRA_CLANG_ARGS="$(pkg-config --cflags glib-2.0) $(pkg-config --cflags libbson-1.0)"
cargo msrv verify
fio-haura:
name: fio ioengine for Haura
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: dependencies
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache fio
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-fio-haura
- name: Build betree
run: |
cd betree
cargo build
- name: Build fio-engine
run: |
cd fio-haura
source ./env-dev.sh
make
- name: Test run
run: |
test -e ./target/debug/deps/betree_storage_stack.d
cd fio-haura
source ./env-dev.sh
# The fio version provided by ubuntu is too outdated, so let's just
# build it from the copy we already have.
make fio
export BETREE_CONFIG=$(realpath ./.ci/haura.json)
./fio --direct=1 --rw=write --bs=4M --ioengine=external:src/fio-engine-haura.o --numjobs=1 --name=iops-test-job --size=128M
haura-benchmarks:
name: Benchmark Compatibility Tests
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: dependencies
steps:
- name: Checkout
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install Rust
run: |
rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
. "${HOME}/.cargo/env"
echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
- name: Cache
id: cache
uses: actions/cache@v3
# Environment variables do not seem to work, use ~ instead.
with:
path: |
~/.cargo
~/.cache/pip
~/.cache/poetry
key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-haura-benchmarks
- name: Build haura-benchmarks
run: |
cd betree/haura-benchmarks
cargo build --release
- name: Prepare scripts and configuration
run: |
cd betree/haura-benchmarks
echo "ci" >> run.sh
- name: Run benchmark smoke test
run: |
cd betree/haura-benchmarks
cp example_config/.ci-config.json perf-config.json
./run.sh smoke-test
rm perf-config.json
- name: Run plots
run: |
sudo apt update || true
sudo apt --yes --no-install-recommends install python3-poetry
cd betree/haura-benchmarks/haura-plots
poetry install
poetry run plots ../results/*/*
rm -rf results