diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml new file mode 100644 index 0000000..5a3c88d --- /dev/null +++ b/.github/workflows/TestCITools.yml @@ -0,0 +1,37 @@ +# +# This tests the extension-ci-tools works as expected +# +name: Test CI Tools +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + extension-template-main: + name: Extension template + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: quack + override_repository: duckdb/extension-template + override_ref: main + duckdb_version: v1.0.0 + override_ci_tools_repository: ${{ github.repository }} + override_ci_tools_ref: ${{ github.sha }} + + delta-extension-main: + name: Rust builds (using Delta extension) + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: delta + enable_rust: true + override_repository: duckdb/duckdb_delta + override_ref: main + override_ci_tools_repository: ${{ github.repository }} + override_ci_tools_ref: ${{ github.sha }} + duckdb_version: v1.0.0 + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 647f9cf..30b508e 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -59,6 +59,17 @@ on: required: false type: string default: "" + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "" + # Override the ref for the CI tools (for testing CI tools itself) + override_ci_tools_ref: + required: false + type: string + default: "" + jobs: generate_matrix: name: Generate matrix @@ -124,6 +135,7 @@ jobs: linux: name: Linux runs-on: ubuntu-latest + container: ${{ matrix.container }} needs: generate_matrix if: ${{ needs.generate_matrix.outputs.linux_matrix != '{}' && needs.generate_matrix.outputs.linux_matrix != '' }} strategy: @@ -131,19 +143,66 @@ jobs: env: VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + GEN: ninja BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} - REPO_REF: ${{ inputs.override_ref }} - REPO_NAME: ${{ inputs.override_repository != '' && inputs.override_repository || github.repository }} - GITHUB_REPOSITORY: ${{ github.repository }} - ENABLE_RUST: '0' - CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || 'gcc' }} - CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || 'g++' }} - RUN_TESTS: ${{ matrix.duckdb_arch != 'linux_arm64' && '1' }} - AARCH64_CROSS_COMPILE: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }} - CONTAINER: ${{ matrix.container }} + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: + ### + # Setup environment (TODO: this should ideally be part of a container image) + ### + - name: Install required ubuntu packages + if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} + run: | + apt-get update -y -qq + apt-get install -y -qq software-properties-common + apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config + + - name: Install cross compiler for linux_arm64 + shell: bash + if: ${{ matrix.duckdb_arch == 'linux_arm64' }} + run: | + apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install Git 2.18.5 + if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} + run: | + wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz + tar xvf v2.18.5.tar.gz + cd git-2.18.5 + make + make prefix=/usr install + git --version + + - name: Setup Rust + if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64'}} + uses: dtolnay/rust-toolchain@stable + + - name: Setup Rust for cross compilation + if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_arm64'}} + uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-unknown-linux-gnu + + - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here) + if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64_gcc4' }} + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Install CMake 3.21 + if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} + shell: bash + run: | + wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh + chmod +x cmake-3.21.3-linux-x86_64.sh + ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local + cmake --version + + ### + # Checkin out repositories + ### - uses: actions/checkout@v3 name: Checkout override repository if: ${{inputs.override_repository != ''}} @@ -160,156 +219,64 @@ jobs: fetch-depth: 0 submodules: 'true' - - name: Setup and build extension - shell: bash + - name: Checkout DuckDB to version run: | - export CURR_DIR=/home/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/} - cat > script.sh << ENDOFLINE - VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} - export VCPKG_BUILD=1 - export DUCKDB_PLATFORM="${DUCKDB_PLATFORM}" - export BUILD_SHELL="${BUILD_SHELL}" - export VCPKG_TOOLCHAIN_PATH=$CURR_DIR/vcpkg/scripts/buildsystems/vcpkg.cmake - set -eo pipefail - ### - echo "Install required ubuntu packages" - cd $CURR_DIR - if [ "${DUCKDB_PLATFORM}" != "linux_amd64_gcc4" ]; then - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - fi - ### - echo "Install Git 2.18.5" - cd $CURR_DIR - if [ "${DUCKDB_PLATFORM}" != "linux_amd64_gcc4" ]; then - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - fi - ### - echo "Checkout DuckDB to version" - cd $CURR_DIR - git config --global --add safe.directory $CURR_DIR - git config --global --add safe.directory $CURR_DIR/duckdb - cd duckdb - git checkout ${{ inputs.duckdb_version }} - ### - echo "Fix CentOS 7 EOL" - cd $CURR_DIR - if [[ "${DUCKDB_PLATFORM}" = "linux_amd64_gcc4" ]]; then - # Workaround the fact that CentOS 7 is EOL - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - yum-config-manager --disable centos-sclo-rh - yum install epel-release -y curl - fi - ### - echo "Setup ManyLinux2014" - cd $CURR_DIR - if [ "${DUCKDB_PLATFORM}" = "linux_amd64_gcc4" ]; then - ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl vcpkg - else - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config git - wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh - chmod +x cmake-3.21.3-linux-x86_64.sh - ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local - cmake --version - # echo "Install Python 3.8" - # wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz - # tar xvf Python-3.8.17.tgz - # cd Python-3.8.17 - # mkdir -p pythonbin - # ./configure --with-ensurepip=install - # make -j - # make install - # python3.8 --version - # python3.8 -m pip install pip - # python3.8 -m pip install requests awscli - echo "Install" - if [ -n "${AARCH64_CROSS_COMPILE}" ]; then - apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - fi - git checkout ${{ inputs.git_ref }} - #python3 --version - #python3 -m pip install pip - #python3 -m pip install requests awscli - echo "Version Check" - ldd --version ldd - #python3 --version - git --version - git log -1 --format=%h - fi - ### - echo "Checkout, again" - cd $CURR_DIR - if [ -n "${REPO_REF}" ]; then - git checkout $REPO_REF - fi - ### - echo "Checkout DuckDB to version" - cd $CURR_DIR cd duckdb git checkout ${{ inputs.duckdb_version }} - ### - echo "Setup vcpkg" - cd $CURR_DIR - git clone https://github.com/microsoft/vcpkg.git - cd vcpkg - git fetch - git checkout a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 - #export CC="${CC}" - #export CXX="${CXX}" - ./bootstrap-vcpkg.sh - ### - echo "Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)" - cd $CURR_DIR - if [ -n "${ENABLE_RUST}" ]; then - curl --proto "https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - #PATH="$HOME/.cargo/bin:$PATH" - #. "$HOME/.cargo/env" - fi - # - name: Setup Ccache - # uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - # continue-on-error: true - # with: - # key: ${{ github.job }}-${{ matrix.duckdb_arch }} - ### - #echo "Configure OpenSSL for Rust" - #cd $CURR_DIR - #if [ -n "${ENABLE_RUST}" ]; then - OPENSSL_ROOT_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET - OPENSSL_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET - OPENSSL_INCLUDE_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET/include - OPENSSL_USE_STATIC_LIBS=true - #fi - ### - echo "Build extension" - cd $CURR_DIR - # export VCPKG_TOOLCHAIN_PATH=/home/runner/work/quack/quack/vcpkgs/script/buildsystems/vcpkg.cmake - #export CC="${CC}" - #export CXX="${CXX}" - GEN=ninja make release - ### - echo "Test extension" - cd $CURR_DIR - if [ -n "${RUN_TESTS}" ]; then - make test - fi - ENDOFLINE - chmod +x script.sh - cat script.sh | docker run -i --rm -v /home/runner/work:/home/runner/work --workdir $CURR_DIR $CONTAINER - - uses: actions/upload-artifact@v3 + - name: Setup ManyLinux2014 + if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }} + run: | + ./duckdb/scripts/setup_manylinux2014.sh general ccache ssh python_alias openssl + + - uses: actions/checkout@v3 + name: Checkout Extension CI tools + if: ${{inputs.override_ci_tools_ref != ''}} + with: + path: 'extension-ci-tools' + ref: ${{ inputs.override_ci_tools_ref }} + repository: ${{ inputs.override_ci_tools_repository }} + fetch-depth: 0 + + ### + # Runtime configuration before build + ### + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + continue-on-error: true + with: + key: ${{ github.job }}-${{ matrix.duckdb_arch }} + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} + + - name: Configure OpenSSL for Rust + if: ${{ inputs.enable_rust }} + run: | + echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV + echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV + echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV + + ### + # Building & testing + ### + - name: Build extension + env: + GEN: ninja + CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} + CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + run: | + make release + + - name: Test extension + if: ${{ matrix.duckdb_arch != 'linux_arm64'}} + run: | + make test + + - uses: actions/upload-artifact@v2 with: name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} path: | @@ -361,6 +328,15 @@ jobs: with: python-version: '3.11' + - uses: actions/checkout@v3 + name: Checkout Extension CI tools + if: ${{inputs.override_ci_tools_ref != ''}} + with: + path: 'extension-ci-tools' + ref: ${{ inputs.override_ci_tools_ref }} + repository: ${{ inputs.override_ci_tools_repository }} + fetch-depth: 0 + - name: Checkout DuckDB to version run: | cd duckdb @@ -449,6 +425,15 @@ jobs: update-rtools: true rtools-version: '42' # linker bug in 43 + - uses: actions/checkout@v3 + name: Checkout Extension CI tools + if: ${{inputs.override_ci_tools_ref != ''}} + with: + path: 'extension-ci-tools' + ref: ${{ inputs.override_ci_tools_ref }} + repository: ${{ inputs.override_ci_tools_repository }} + fetch-depth: 0 + - name: Checkout DuckDB to version run: | cd duckdb @@ -508,6 +493,15 @@ jobs: fetch-depth: 0 submodules: 'true' + - uses: actions/checkout@v3 + name: Checkout Extension CI tools + if: ${{inputs.override_ci_tools_ref != ''}} + with: + path: 'extension-ci-tools' + ref: ${{ inputs.override_ci_tools_ref }} + repository: ${{ inputs.override_ci_tools_repository }} + fetch-depth: 0 + - name: Checkout DuckDB to version run: | cd duckdb @@ -543,4 +537,4 @@ jobs: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} path: | - build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm + build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm \ No newline at end of file diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 27b6808..7adc9e0 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -65,6 +65,11 @@ release: cmake $(GENERATOR) $(BUILD_FLAGS) $(EXT_RELEASE_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(DUCKDB_SRCDIR) -B build/release cmake --build build/release --config Release +reldebug: + mkdir -p build/reldebug + cmake $(GENERATOR) $(BUILD_FLAGS) $(EXT_RELEASE_FLAGS) -DCMAKE_BUILD_TYPE=RelWithDebInfo -S $(DUCKDB_SRCDIR) -B build/reldebug + cmake --build build/reldebug + # Main tests test: test_release @@ -74,6 +79,9 @@ test_release: release test_debug: debug ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_reldebug: reldebug + ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" + # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -DSKIP_EXTENSIONS="parquet;json" $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS)