-
Notifications
You must be signed in to change notification settings - Fork 16
204 lines (172 loc) · 6.21 KB
/
LocalTesting.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
name: Local functional tests
on: [push, pull_request,repository_dispatch]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
azurite-tests-linux:
name: Azurite (local azure test server) tests (Linux)
runs-on: ubuntu-latest
container: 'quay.io/pypa/manylinux2014_x86_64'
env:
VCPKG_TARGET_TRIPLET: 'x64-linux'
GEN: Ninja
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
AZURE_STORAGE_ACCOUNT: devstoreaccount1
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: 'true'
- name: install Azure test service
run: |
yum install -y nodejs npm
npm install -g azurite
echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo
yum install -y azure-cli
- name: Setup ManyLinux2014
run: |
./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
- name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Setup vcpkg
uses: lukka/[email protected]
with:
vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
- name: Handle OpenSSL dependency for rust build
run: |
echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
# Build extension
- name: Build extension
env:
GEN: ninja
run: |
make release
- name: Launch & populate Azure test service
run: |
azurite > azurite_log.txt 2>&1 &
sleep 10
./scripts/upload_test_files_to_azurite.sh
- name: Test extension
run: |
make test
- name: Azure test server log
if: always()
shell: bash
run: |
echo "## azurite"
cat azurite_log.txt
minio-tests-linux:
name: Minio (local S3 test server) tests (Linux)
runs-on: ubuntu-latest
env:
S3_TEST_SERVER_AVAILABLE: 1
GEN: ninja
VCPKG_TARGET_TRIPLET: x64-linux
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: 'true'
- name: Checkout DuckDB to version
if: ${{ matrix.duckdb_version != '<submodule_version>'}}
run: |
cd duckdb
git checkout ${{ matrix.duckdb_version }}
- uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install Ninja
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
- name: Setup vcpkg
uses: lukka/[email protected]
with:
vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
- name: Build
shell: bash
run: make
- name: Start S3/HTTP test server
shell: bash
run: |
cd duckdb
mkdir data/attach_test
touch data/attach_test/attach.db
sudo ./scripts/install_s3_test_server.sh
source ./scripts/run_s3_test_server.sh
sleep 30
- name: Write AWS credentials file
shell: bash
run: |
./scripts/create_minio_credential_file.sh
- name: Copy files to minio
shell: bash
env:
DUCKDB_MINIO_TEST_SERVER_AVAILABLE: 1
AWS_ACCESS_KEY_ID: minio_duckdb_user
AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
AWS_DEFAULT_REGION: eu-west-1
AWS_ENDPOINT: duckdb-minio.com:9000
run: |
./scripts/upload_test_files_to_minio.sh
- name: Test
shell: bash
run: |
make test
- name: Run Env tests
shell: bash
env:
DUCKDB_MINIO_TEST_SERVER_AVAILABLE: 1
AWS_ACCESS_KEY_ID: minio_duckdb_user
AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
AWS_DEFAULT_REGION: eu-west-1
AWS_ENDPOINT: duckdb-minio.com:9000
run: |
./build/release/test/unittest "*/test/sql/cloud/minio_local/*"
generated-tests-linux:
name: Generated Tests (Linux)
runs-on: ubuntu-latest
env:
GEN: ninja
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: 'true'
- name: Install
shell: bash
run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
- name: Setup Ccache
uses: hendrikmuhs/ccache-action@main
with:
key: ${{ github.job }}
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Setup vcpkg
uses: lukka/[email protected]
with:
vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
- name: Build
shell: bash
run: make generate-data
- name: Test
shell: bash
run: |
GENERATED_DATA_AVAILABLE=1 make test