From 8d4a9e2d53b62d72dc98b49c46105c182c1f4af0 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 11:18:34 +0200 Subject: [PATCH 01/41] small ci tweaks --- .github/workflows/_extension_distribution.yml | 13 +++++++++++-- makefiles/duckdb_extension.Makefile | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 3bd4cb0..b268c11 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -308,10 +308,19 @@ jobs: run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release - - name: Test extension + - name: Test extension (inside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + env: + LINUX_TESTS_IN_DOCKER=1 + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release + + - name: Test extension (outside docker) + if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + env: + LINUX_TESTS_OUTSIDE_DOCKER=1 run: | - docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test + make test_release - uses: actions/upload-artifact@v3 with: diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 7e43936..dba37f8 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -78,13 +78,13 @@ reldebug: # Main tests test: test_release -test_release: release +test_release: ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" -test_debug: debug +test_debug: ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" -test_reldebug: reldebug +test_reldebug: ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" # WASM config From 5f7049596eeb7b2c480c1209dc6680a452cdd507 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 11:47:28 +0200 Subject: [PATCH 02/41] add skipping option to makefile --- makefiles/duckdb_extension.Makefile | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index dba37f8..39d405b 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -6,8 +6,9 @@ # EXT_FLAGS : Extra CMake flags to pass to the build # EXT_RELEASE_FLAGS : Extra CMake flags to pass to the release build # EXT_DEBUG_FLAGS : Extra CMake flags to pass to the debug build +# SKIP_TESTS : Replaces all test targets with a NOP step -.PHONY: all clean format debug release pull update wasm_mvp wasm_eh wasm_threads +.PHONY: all clean clean-python format debug release pull update wasm_mvp wasm_eh wasm_threads test test_release test_debug test_reldebug test_release_internal test_debug_internal test_reldebug_internal set_duckdb_version set_duckdb_tag output_distribution_matrix all: release @@ -78,15 +79,30 @@ reldebug: # Main tests test: test_release -test_release: - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +TEST_RELEASE_TARGET=test_release_internal +TEST_DEBUG_TARGET=test_debug_internal +TEST_RELDEBUG_TARGET=test_reldebug_internal -test_debug: - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped + TEST_RELDEBUG_TARGET=tests_skipped +endif + +test_release: $(TEST_RELEASE_TARGET) +test_debug: $(TEST_DEBUG_TARGET) +test_reldebug: $(TEST_RELDEBUG_TARGET) -test_reldebug: +test_release_internal: + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_debug_internal: + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_reldebug_internal: ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" +tests_skipped: + @echo "Skipping tests..." + # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) From 5f13e930044febb0acffe35337714b3a1cd11ba9 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 11:51:24 +0200 Subject: [PATCH 03/41] call test_release explicitly instead of test --- .github/workflows/_extension_distribution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index b268c11..2039149 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -465,7 +465,7 @@ jobs: if: ${{ matrix.osx_build_arch == 'arm64' && inputs.skip_tests == false }} shell: bash run: | - make test + make test_release - uses: actions/upload-artifact@v3 with: From f5646332ea620bfba20cabfda19704f55baa7b4c Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 12:04:21 +0200 Subject: [PATCH 04/41] use : not = --- .github/workflows/_extension_distribution.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 2039149..b86fa5e 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -311,14 +311,14 @@ jobs: - name: Test extension (inside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} env: - LINUX_TESTS_IN_DOCKER=1 + LINUX_TESTS_IN_DOCKER: 1 run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release - name: Test extension (outside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} env: - LINUX_TESTS_OUTSIDE_DOCKER=1 + LINUX_TESTS_OUTSIDE_DOCKER: 1 run: | make test_release From df506fdde3527e0431bda8122fcc8cac42ee462b Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 13:03:06 +0200 Subject: [PATCH 05/41] skip tests outside docker for now --- makefiles/duckdb_extension.Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 39d405b..1927666 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -83,6 +83,11 @@ TEST_RELEASE_TARGET=test_release_internal TEST_DEBUG_TARGET=test_debug_internal TEST_RELDEBUG_TARGET=test_reldebug_internal +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq($(LINUX_TESTS_OUTSIDE_DOCKER),1) + SKIP_TESTS=1 +endif + ifeq ($(SKIP_TESTS),1) TEST_RELEASE_TARGET=tests_skipped TEST_DEBUG_TARGET=tests_skipped From 12c1fc226fc5db613efe60b608bfda76e93c69fe Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 14:08:13 +0200 Subject: [PATCH 06/41] tabs not whitespaces --- makefiles/duckdb_extension.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 1927666..7e99745 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -84,8 +84,8 @@ TEST_DEBUG_TARGET=test_debug_internal TEST_RELDEBUG_TARGET=test_reldebug_internal # Disable testing outside docker: the unittester is currently dynamically linked by default -ifeq($(LINUX_TESTS_OUTSIDE_DOCKER),1) - SKIP_TESTS=1 +ifeq ($(LINUX_TESTS_OUTSIDE_DOCKER),1) + SKIP_TESTS=1 endif ifeq ($(SKIP_TESTS),1) From e4955f0498b7e84f4a9f01c8feeaf0be0c0373e0 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 15 Oct 2024 15:00:56 +0200 Subject: [PATCH 07/41] fix windows; switch to make configure_ci step instead of custom_toolchain_script --- .github/workflows/TestCITools.yml | 1 - .github/workflows/_extension_distribution.yml | 39 ++++++++++--------- makefiles/duckdb_extension.Makefile | 3 ++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 0f86112..c9e525a 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -23,7 +23,6 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;python3' - custom_toolchain_script: true delta-extension-main: name: Rust builds (using Delta extension) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index b86fa5e..f957fc4 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -76,11 +76,6 @@ on: required: false type: string default: "" - # If set, the setup will look for a script in `./scripts/custom-toolchain-script.sh` to run - custom_toolchain_script: - required: false - type: boolean - default: false rust_logs: required: false type: boolean @@ -283,6 +278,7 @@ jobs: echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt + echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt - name: Generate timestamp for Ccache entry @@ -304,6 +300,11 @@ jobs: restore-keys: | ccache-extension-distribution-${{ matrix.duckdb_arch }}- + - name: Run configure CI + shell: bash + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci + - name: Build extension run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release @@ -448,16 +449,15 @@ jobs: echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure CI shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension shell: bash - env: - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} run: | make release @@ -592,21 +592,23 @@ jobs: vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} vcpkgGitURL: ${{ inputs.vcpkg_url }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure CI shell: bash + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension - if: ${{ inputs.skip_tests == true }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} run: | make release - - name: Build & test extension + - name: Test extension if: ${{ inputs.skip_tests == false }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} @@ -710,11 +712,12 @@ jobs: with: key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure CI shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build Wasm module run: | diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 7e99745..f2865d5 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -159,3 +159,6 @@ set_duckdb_tag: output_distribution_matrix: cat duckdb/.github/config/distribution_matrix.json + +configure_ci: + @echo "configure_ci is a NOP" From 3d1c62e9d2259babb9d46fe858587ee0746cab6a Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 13:28:24 +0100 Subject: [PATCH 08/41] allow passing platform as file --- scripts/append_extension_metadata.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 7ea59a0..1fdd77a 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -33,7 +33,10 @@ def main(): arg_parser.add_argument('-o', '--out-file', type=str, help='Explicit path for the output file', default='') - arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode', required=True) + # The platform + arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode') + arg_parser.add_argument('-pf', '--duckdb-platform-file', type=str, help='The file containing the DuckDB platform to encode') + arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type ' 'this encodes the duckdb version or the C API version', required=True) arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True) @@ -51,6 +54,21 @@ def main(): print(f" - Output file: {OUTPUT_FILE}") shutil.copyfile(args.library_file, OUTPUT_FILE_TMP) + # Handle the platform + PLATFORM = "" + if args.duckdb_platform: + PLATFORM = args.duckdb_platform + elif args.duckdb_platform_file: + try: + with open(args.duckdb_platform_file, 'r') as file: + PLATFORM = file.read().strip() + if not PLATFORM: + raise Exception("File is empty!") + except Exception as e: + raise Exception(f"Failed to read platform from file: {args.duckdb_platform_file}: {e}") + else: + raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") + # Then append the metadata to the tmp file print(f" - Metadata:") with open(OUTPUT_FILE_TMP, 'ab') as file: @@ -67,8 +85,8 @@ def main(): file.write(padded_byte_string(args.extension_version)) print(f" - FIELD3 (duckdb_version) = {args.duckdb_version}") file.write(padded_byte_string(args.duckdb_version)) - print(f" - FIELD2 (duckdb_platform) = {args.duckdb_platform}") - file.write(padded_byte_string(args.duckdb_platform)) + print(f" - FIELD2 (duckdb_platform) = {PLATFORM}") + file.write(padded_byte_string(PLATFORM)) print(f" - FIELD1 (header signature) = 4 (special value to identify a duckdb extension)") file.write(padded_byte_string("4")) From 0d7e751a73841e0df3b165d66ffd82c052507bb1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 15:21:00 +0100 Subject: [PATCH 09/41] update workflow for upcoming rust template --- .github/workflows/_extension_distribution.yml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index f957fc4..33c9ca9 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -279,6 +279,7 @@ jobs: echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt + echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt - name: Generate timestamp for Ccache entry @@ -300,26 +301,31 @@ jobs: restore-keys: | ccache-extension-distribution-${{ matrix.duckdb_arch }}- - - name: Run configure CI + - name: Run configure (outside Docker) + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + run: | + make configure_ci + + - name: Run configure (inside Docker) shell: bash run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci - - name: Build extension + - name: Build extension (inside Docker) run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release - name: Test extension (inside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} - env: - LINUX_TESTS_IN_DOCKER: 1 run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release - name: Test extension (outside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} env: - LINUX_TESTS_OUTSIDE_DOCKER: 1 + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | make test_release @@ -449,7 +455,7 @@ jobs: echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV - - name: Run configure CI + - name: Run configure shell: bash env: DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} @@ -592,7 +598,7 @@ jobs: vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} vcpkgGitURL: ${{ inputs.vcpkg_url }} - - name: Run configure CI + - name: Run configure shell: bash env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} @@ -712,7 +718,7 @@ jobs: with: key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - name: Run configure CI + - name: Run configure shell: bash env: DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} From 3930e25d4bcd90e6ec8fc95234442f8847fccaa1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 15:46:43 +0100 Subject: [PATCH 10/41] cleaner error handling in metadata script --- scripts/append_extension_metadata.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 1fdd77a..277d5d3 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -62,10 +62,11 @@ def main(): try: with open(args.duckdb_platform_file, 'r') as file: PLATFORM = file.read().strip() - if not PLATFORM: - raise Exception("File is empty!") except Exception as e: - raise Exception(f"Failed to read platform from file: {args.duckdb_platform_file}: {e}") + print(f"Failed to read platform from file {args.duckdb_platform_file}") + raise + if not PLATFORM: + raise Exception(f"Platform file passed to script is empty: {args.duckdb_platform_file}") else: raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") From fbcf3da68f858796e4f6ac089c5fa135e685d493 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 15:57:17 +0100 Subject: [PATCH 11/41] fix disabling test outside docker --- .github/workflows/_extension_distribution.yml | 2 ++ makefiles/duckdb_extension.Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 33c9ca9..9c984ae 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -305,6 +305,7 @@ jobs: shell: bash env: DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 run: | make configure_ci @@ -326,6 +327,7 @@ jobs: if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} env: DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 run: | make test_release diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index f2865d5..8caff9a 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -84,7 +84,7 @@ TEST_DEBUG_TARGET=test_debug_internal TEST_RELDEBUG_TARGET=test_reldebug_internal # Disable testing outside docker: the unittester is currently dynamically linked by default -ifeq ($(LINUX_TESTS_OUTSIDE_DOCKER),1) +ifeq ($(LINUX_CI_IN_DOCKER),0) SKIP_TESTS=1 endif From 74f0e97314aa0bd7bc473e26eedf423998cbb937 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 16:22:48 +0100 Subject: [PATCH 12/41] cleaning comments for skipping --- makefiles/duckdb_extension.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 8caff9a..c248f7c 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -106,7 +106,7 @@ test_reldebug_internal: ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" tests_skipped: - @echo "Skipping tests..." + @echo "Tests are skipped in this run..." # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake @@ -161,4 +161,4 @@ output_distribution_matrix: cat duckdb/.github/config/distribution_matrix.json configure_ci: - @echo "configure_ci is a NOP" + @echo "configure_ci step is skipped for this extension build..." From 6ec869bceaab2572fd3c613bab2bf7374f1a7a8c Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 6 Nov 2024 16:28:50 +0100 Subject: [PATCH 13/41] switch to matrix distro script in ci --- .github/workflows/_extension_distribution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 9c984ae..81a8d03 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -49,7 +49,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" # Enable building the DuckDB Shell build_duckdb_shell: required: false From 795955cdaef5b5b1dffd375604ecb8800ddc5e85 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Thu, 7 Nov 2024 14:06:39 +0100 Subject: [PATCH 14/41] add platform/version detection script --- scripts/append_extension_metadata.py | 25 ++++++++++++++-- scripts/configure_helper.py | 45 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 scripts/configure_helper.py diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 277d5d3..d15262d 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -39,7 +39,9 @@ def main(): arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type ' 'this encodes the duckdb version or the C API version', required=True) - arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True) + arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode') + arg_parser.add_argument('-evf', '--extension-version-file', type=str, help='The file containing the Extension version to encode') + arg_parser.add_argument('--abi-type', type=str, help='The ABI type to encode, set to C_STRUCT by default', default='C_STRUCT') args = arg_parser.parse_args() @@ -70,6 +72,23 @@ def main(): else: raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") + EXTENSION_VERSION = "" + if args.extension_version: + EXTENSION_VERSION = args.extension_version + elif args.extension_version_file: + try: + with open(args.extension_version_file, 'r') as file: + EXTENSION_VERSION = file.read().strip() + except Exception as e: + print(f"Failed to read extension version from file {args.extension_version_file}") + raise + if not EXTENSION_VERSION: + raise Exception(f"Extension version file passed to script is empty: {args.extension_version_file}") + else: + raise Exception(f"Neither --extension-version nor --extension-version-file found, please specify the extension version using either") + + + # Then append the metadata to the tmp file print(f" - Metadata:") with open(OUTPUT_FILE_TMP, 'ab') as file: @@ -82,8 +101,8 @@ def main(): file.write(padded_byte_string("")) print(f" - FIELD5 (abi_type) = {args.abi_type}") file.write(padded_byte_string(args.abi_type)) - print(f" - FIELD4 (extension_version) = {args.extension_version}") - file.write(padded_byte_string(args.extension_version)) + print(f" - FIELD4 (extension_version) = {EXTENSION_VERSION}") + file.write(padded_byte_string(EXTENSION_VERSION)) print(f" - FIELD3 (duckdb_version) = {args.duckdb_version}") file.write(padded_byte_string(args.duckdb_version)) print(f" - FIELD2 (duckdb_platform) = {PLATFORM}") diff --git a/scripts/configure_helper.py b/scripts/configure_helper.py new file mode 100644 index 0000000..9685410 --- /dev/null +++ b/scripts/configure_helper.py @@ -0,0 +1,45 @@ +import subprocess +import argparse +from pathlib import Path +import os + +def main(): + arg_parser = argparse.ArgumentParser(description='Script to aid in running the configure step of the extension build process') + + + arg_parser.add_argument('-o', '--output-directory', type=str, help='Specify the output directory', default='configure') + + arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true') + arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true') + + args = arg_parser.parse_args() + + OUTPUT_DIR = args.output_directory + + Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True) + + # Write version + if args.extension_version: + git_tag = subprocess.getoutput("git tag --points-at HEAD") + if git_tag: + EXTENSION_VERSION = git_tag + else: + EXTENSION_VERSION = subprocess.getoutput("git --no-pager log -1 --format=%h") + + version_file = Path(os.path.join(OUTPUT_DIR, "extension_version.txt")) + with open(version_file, 'w') as f: + print(f"Writing version {EXTENSION_VERSION} to {version_file}") + f.write(EXTENSION_VERSION) + + # Write duck + if args.duckdb_platform: + import duckdb + platform_file = Path(os.path.join(OUTPUT_DIR, "platform.txt")) + duckdb_platform = duckdb.execute('pragma platform').fetchone()[0] + with open(platform_file, 'w') as f: + print(f"Writing platform {duckdb_platform} to {platform_file}") + f.write(duckdb_platform) + + +if __name__ == '__main__': + main() \ No newline at end of file From 27facd3b63aa4c7c23e80e726851db132f1db264 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Thu, 7 Nov 2024 14:53:09 +0100 Subject: [PATCH 15/41] add makefiles for C extensions --- makefiles/duckdb_extension.Makefile | 2 +- makefiles/duckdb_extension_c_api.Makefile | 211 ++++++++++++++++++++++ makefiles/duckdb_extension_rs.Makefile | 39 ++++ 3 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 makefiles/duckdb_extension_c_api.Makefile create mode 100644 makefiles/duckdb_extension_rs.Makefile diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index c248f7c..47f32b4 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -1,4 +1,4 @@ -# Reusable makefile for building out-of-tree extension with the DuckDB extension template +# Reusable makefile for building out-of-tree extension with the DuckDB C++ based extension template # # Inputs # EXT_NAME : Upper case string describing the name of the out-of-tree extension diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile new file mode 100644 index 0000000..6ae8635 --- /dev/null +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -0,0 +1,211 @@ +# Reusable makefile for C API based extensions +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports +# EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected +# DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected +# DUCKDB_TEST_VERSION : the version of DuckDB to test with, if left blank will default to latest stable on PyPi +# DUCKDB_GIT_VERSION : set by CI currently, should probably be removed at some point +# LINUX_CI_IN_DOCKER : indicates that the build is being run in/out of Docker in the linux CI +# SKIP_TESTS : makes the test targets turn into NOPs + +.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure + +############################################# +### Platform dependent config +############################################# +PYTHON_BIN=python3 + +ifeq ($(OS),Windows_NT) + EXTENSION_LIB_FILENAME=$(EXTENSION_NAME).dll + PYTHON_VENV_BIN=./configure/venv/Scripts/python.exe +else + PYTHON_VENV_BIN=./configure/venv/bin/python3 + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).so + endif + ifeq ($(UNAME_S),Darwin) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dylib + endif +endif + +############################################# +### Main extension parameters +############################################# + +# The minimum DuckDB version that this extension supports +ifeq ($(MINIMUM_DUCKDB_VERSION),) + MINIMUM_DUCKDB_VERSION = v0.0.1 +endif + +EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension + +############################################# +### Platform Detection +############################################# + +# Write the platform we are building for +platform: configure/platform.txt + +# Either autodetect or use the provided value +PLATFORM_COMMAND?= +ifeq ($(DUCKDB_PLATFORM),) + PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform +else + # Sets the platform using DUCKDB_PLATFORM variable + PLATFORM_COMMAND=echo $(DUCKDB_PLATFORM) > configure/platform.txt +endif + +configure/platform.txt: + @ $(PLATFORM_COMMAND) + +############################################# +### Extension Version Detection +############################################# + +# Either autodetect or use the provided value +VERSION_COMMAND?= +ifeq ($(EXTENSION_VERSION),) + VERSION_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --extension-version +else + # Sets the platform using DUCKDB_PLATFORM variable + VERSION_COMMAND=echo "$(EXTENSION_VERSION)" > configure/extension_version.txt +endif + +extension_version: configure/extension_version.txt + +configure/extension_version.txt: + @ $(VERSION_COMMAND) + +############################################# +### Testing +############################################# + +# Note: to override the default test runner, create a symlink to a different venv +TEST_RUNNER=$(PYTHON_VENV_BIN) -m duckdb_sqllogictest + +TEST_RUNNER_BASE=$(TEST_RUNNER) --test-dir test/sql $(EXTRA_EXTENSIONS_PARAM) +TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/rusty_quack.duckdb_extension +TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/rusty_quack.duckdb_extension + +# By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version +DUCKDB_INSTALL_VERSION?= +ifneq ($(DUCKDB_TEST_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION) +endif + +ifneq ($(DUCKDB_GIT_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION) +endif + +TEST_RELEASE_TARGET=test_extension_release_internal +TEST_DEBUG_TARGET=test_extension_debug_internal + +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq ($(LINUX_CI_IN_DOCKER),1) + SKIP_TESTS=1 +endif + +# TODO: for some weird reason the Ubuntu 22.04 Runners on Github Actions don't actually grab the glibc 2.24 wheels but the +# gilbc 2.17 ones. What this means is that we can't run the tests on linux_amd64 because we are installing the duckdb +# linux_amd64_gcc4 test runner +ifeq ($(DUCKDB_PLATFORM),linux_amd64) + SKIP_TESTS=1 +endif + +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped +endif + +test_extension_release: $(TEST_RELEASE_TARGET) +test_extension_debug: $(TEST_DEBUG_TARGET) + +test_extension_release_internal: check_configure + @echo "Running RELEASE tests.." + @$(TEST_RUNNER_RELEASE) + +test_extension_debug_internal: check_configure + @echo "Running DEBUG tests.." + @$(TEST_RUNNER_DEBUG) + +tests_skipped: + @echo "Skipping tests.." + + +############################################# +### Misc +############################################# + +clean_build: + rm -rf build + rm -rf duckdb_unittest_tempdir + +clean_configure: + rm -rf configure + +nop: + @echo "NOP" + +set_duckdb_tag: nop + +set_duckdb_version: nop + +output_distribution_matrix: + cat extension-ci-tools/config/distribution_matrix.json + +############################################# +### Building +############################################# +build_extension_with_metadata_debug: check_configure + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/debug/$(EXTENSION_LIB_FILENAME) \ + -o build/debug/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/debug/$(EXTENSION_FILENAME)', 'build/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +build_extension_with_metadata_release: check_configure + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/release/$(EXTENSION_LIB_FILENAME) \ + -o build/release/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/release/$(EXTENSION_FILENAME)', 'build/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +############################################# +### Python +############################################# + +# Installs the test runner using the selected DuckDB version (latest stable by default) +# TODO: switch to PyPI distribution +venv: configure/venv + +configure/venv: + $(PYTHON_BIN) -m venv configure/venv + $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' + $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python + +############################################# +### Configure +############################################# + +CONFIGURE_CI_STEP?= +ifeq ($(LINUX_CI_IN_DOCKER),1) + CONFIGURE_CI_STEP=nop +else + CONFIGURE_CI_STEP=configure +endif + +configure_ci: $(CONFIGURE_CI_STEP) + +# Because the configure_ci may differ from configure, we don't automatically run configure on make build, this makes the error a bit nicer +check_configure: + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/platform.txt'), 'The configure step appears to not be run. Please try running make configure'" + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'" \ No newline at end of file diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/duckdb_extension_rs.Makefile new file mode 100644 index 0000000..027fea5 --- /dev/null +++ b/makefiles/duckdb_extension_rs.Makefile @@ -0,0 +1,39 @@ +# Reusable makefile for the Rust extensions targeting the C extension API +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# EXTENSION_LIB_FILENAME : the library name that is produced by the build +# LOCAL_DUCKDB_RS_PATH : overrides the duckdb-rs path + +.PHONY: build_extension_library_debug build_extension_library_release + +############################################# +### Development config +############################################# + +# Allows overriding the duckdb-rs crates with a local version +CARGO_OVERRIDE_DUCKDB_RS_FLAG?= +ifneq ($(LOCAL_DUCKDB_RS_PATH),) + CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"' +endif + +############################################# +### Rust Build targets +############################################# + +build_extension_library_debug: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + +build_extension_library_release: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + +############################################# +### Misc +############################################# + +clean_rust: + cargo clean \ No newline at end of file From df8d46f746538a255ba477802a3cb8d3e9fa7ffd Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 12:41:56 +0100 Subject: [PATCH 16/41] bump download artifact --- .github/workflows/_extension_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index b9912f2..59be0ef 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -92,7 +92,7 @@ jobs: cd duckdb git checkout ${{ inputs.duckdb_version }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} path: | From 21d3a39629ed9237c9a8e89d514b2fa289518de2 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 12:45:43 +0100 Subject: [PATCH 17/41] also bump upload --- .github/workflows/_extension_distribution.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 3bd4cb0..021f744 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -313,7 +313,7 @@ jobs: run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} path: | @@ -458,7 +458,7 @@ jobs: run: | make test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} @@ -605,7 +605,7 @@ jobs: run: | make test_release - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} @@ -711,7 +711,7 @@ jobs: run: | make ${{ matrix.duckdb_arch }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} From ab001a8357bd17965c5f7ca483c1eddd7687420e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 14:15:24 +0100 Subject: [PATCH 18/41] simplify matrix generation step of build --- .github/workflows/_extension_distribution.yml | 47 ++----------------- 1 file changed, 5 insertions(+), 42 deletions(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 021f744..9d5001b 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -49,7 +49,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" # Enable building the DuckDB Shell build_duckdb_shell: required: false @@ -115,22 +115,6 @@ jobs: osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }} wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }} steps: - - uses: actions/checkout@v4 - name: Checkout override repository - if: ${{inputs.override_repository != ''}} - with: - repository: ${{ inputs.override_repository }} - ref: ${{ inputs.override_ref }} - fetch-depth: 0 - submodules: 'true' - - - uses: actions/checkout@v4 - name: Checkout current repository - if: ${{inputs.override_repository == ''}} - with: - fetch-depth: 0 - submodules: 'true' - - uses: actions/checkout@v4 name: Checkout Extension CI tools with: @@ -138,34 +122,13 @@ jobs: ref: ${{ inputs.ci_tools_version }} repository: ${{ inputs.override_ci_tools_repository }} - - name: Checkout DuckDB to version - if: ${{inputs.duckdb_version != ''}} - run: | - DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version - - - name: Tag extension - if: ${{inputs.extension_tag != ''}} - run: | - git tag ${{ inputs.extension_tag }} - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != '' && (inputs.duckdb_version == 'main' || inputs.duckdb_version[0] == 'v') }} - run: | - echo "When duckdb_tag is provied an explcit git ref is expected" && exit 1 - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != ''}} - run: | - DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag - - id: parse-matrices run: | mkdir build - make output_distribution_matrix | tail -n +2 > build/distribution_matrix.json - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - id: set-matrix-linux run: | From 74cca858bd3a654e4ae2595e6b42cd14d65b9787 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 8 Nov 2024 14:30:33 +0100 Subject: [PATCH 19/41] simplify matrix creation step for deploy --- .github/workflows/_extension_deploy.yml | 25 ++++++++++++------- .github/workflows/_extension_distribution.yml | 1 + 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index 59be0ef..d9d68bc 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -18,6 +18,16 @@ on: duckdb_version: required: true type: string + # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`. + # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264 + ci_tools_version: + required: true + type: string + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "duckdb/extension-ci-tools" # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' exclude_archs: required: false @@ -47,7 +57,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" jobs: generate_matrix: @@ -57,18 +67,15 @@ jobs: deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} steps: - uses: actions/checkout@v4 + name: Checkout Extension CI tools with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} - id: parse-matrices run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty deploy_matrix="`cat deploy_matrix.json`" echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 9d5001b..bcc4c6d 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -105,6 +105,7 @@ on: required: false type: boolean default: false + jobs: generate_matrix: name: Generate matrix From f164d865e4d754f89125cf428f34070970376894 Mon Sep 17 00:00:00 2001 From: Joe Walnes Date: Sat, 9 Nov 2024 17:41:10 -0800 Subject: [PATCH 20/41] Fix hardcoded extension_name --- makefiles/duckdb_extension_c_api.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 6ae8635..65df080 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -87,8 +87,8 @@ configure/extension_version.txt: TEST_RUNNER=$(PYTHON_VENV_BIN) -m duckdb_sqllogictest TEST_RUNNER_BASE=$(TEST_RUNNER) --test-dir test/sql $(EXTRA_EXTENSIONS_PARAM) -TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/rusty_quack.duckdb_extension -TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/rusty_quack.duckdb_extension +TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSION_NAME).duckdb_extension +TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension # By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version DUCKDB_INSTALL_VERSION?= From cd15603463d74bcd8a66528ddc54913e26f460cf Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 12 Nov 2024 13:27:22 +0100 Subject: [PATCH 21/41] formatting --- makefiles/duckdb_extension_c_api.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 65df080..5dcffe4 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -52,7 +52,7 @@ platform: configure/platform.txt # Either autodetect or use the provided value PLATFORM_COMMAND?= ifeq ($(DUCKDB_PLATFORM),) - PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform + PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform else # Sets the platform using DUCKDB_PLATFORM variable PLATFORM_COMMAND=echo $(DUCKDB_PLATFORM) > configure/platform.txt From a1a430d00bcb7b30dade1b0a55153c23a3fa93ff Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 12 Nov 2024 13:39:04 +0100 Subject: [PATCH 22/41] Add EXTENSION_FILENAME_NO_METADATA concept --- makefiles/duckdb_extension_c_api.Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 5dcffe4..c05a5b0 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -41,6 +41,7 @@ ifeq ($(MINIMUM_DUCKDB_VERSION),) endif EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension +EXTENSION_FILENAME_NO_METADATA=$(EXTENSION_LIB_FILENAME) ############################################# ### Platform Detection @@ -161,7 +162,7 @@ output_distribution_matrix: ############################################# build_extension_with_metadata_debug: check_configure $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/debug/$(EXTENSION_LIB_FILENAME) \ + -l build/debug/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(MINIMUM_DUCKDB_VERSION) \ @@ -171,7 +172,7 @@ build_extension_with_metadata_debug: check_configure build_extension_with_metadata_release: check_configure $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/release/$(EXTENSION_LIB_FILENAME) \ + -l build/release/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(MINIMUM_DUCKDB_VERSION) \ From 9b647256e87da4e77aeaeb0681dca188ec7afd1d Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 12 Nov 2024 13:40:07 +0100 Subject: [PATCH 23/41] Add wasm detection, and handling both in Capi and rust --- makefiles/duckdb_extension_c_api.Makefile | 31 ++++++++++++++++++++--- makefiles/duckdb_extension_rs.Makefile | 13 +++++++--- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index c05a5b0..3329db6 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -43,6 +43,14 @@ endif EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension EXTENSION_FILENAME_NO_METADATA=$(EXTENSION_LIB_FILENAME) +DUCKDB_WASM_PLATFORM=$(filter wasm_mvp wasm_eh wasm_threads,$(DUCKDB_PLATFORM)) + +ifneq ($(DUCKDB_WASM_PLATFORM),) + EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension.wasm + EXTENSION_FILENAME_NO_METADATA=$(EXTENSION_NAME).no_metadata.wasm + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).a +endif + ############################################# ### Platform Detection ############################################# @@ -158,9 +166,26 @@ output_distribution_matrix: cat extension-ci-tools/config/distribution_matrix.json ############################################# -### Building +### Linking +############################################# +ifneq ($(DUCKDB_WASM_PLATFORM),) + +link_wasm_debug: + emcc build/debug/$(EXTENSION_LIB_FILENAME) -o build/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" -v + +link_wasm_release: + emcc build/release/$(EXTENSION_LIB_FILENAME) -o build/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" -v + +else +link_wasm_debug: +link_wasm_release: + +endif + +############################################# +### Adding metadata ############################################# -build_extension_with_metadata_debug: check_configure +build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ -l build/debug/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/debug/$(EXTENSION_FILENAME) \ @@ -170,7 +195,7 @@ build_extension_with_metadata_debug: check_configure -pf configure/platform.txt $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/debug/$(EXTENSION_FILENAME)', 'build/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" -build_extension_with_metadata_release: check_configure +build_extension_with_metadata_release: check_configure link_wasm_release $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ -l build/release/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/release/$(EXTENSION_FILENAME) \ diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/duckdb_extension_rs.Makefile index 027fea5..e07168b 100644 --- a/makefiles/duckdb_extension_rs.Makefile +++ b/makefiles/duckdb_extension_rs.Makefile @@ -17,19 +17,24 @@ ifneq ($(LOCAL_DUCKDB_RS_PATH),) CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"' endif +ifneq ($(DUCKDB_WASM_PLATFORM),) + TARGET=wasm32-unknown-emscripten + TARGET_INFO=--target $(TARGET) +endif + ############################################# ### Rust Build targets ############################################# build_extension_library_debug: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" ############################################# ### Misc From 1f7a0feb84e80577f0a14559c9800a6bd9ca87ad Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 13 Nov 2024 00:52:39 +0100 Subject: [PATCH 24/41] Workaround for crate-type --- makefiles/duckdb_extension_rs.Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/duckdb_extension_rs.Makefile index e07168b..ffa817d 100644 --- a/makefiles/duckdb_extension_rs.Makefile +++ b/makefiles/duckdb_extension_rs.Makefile @@ -17,9 +17,12 @@ ifneq ($(LOCAL_DUCKDB_RS_PATH),) CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"' endif +IS_EXAMPLE= + ifneq ($(DUCKDB_WASM_PLATFORM),) TARGET=wasm32-unknown-emscripten - TARGET_INFO=--target $(TARGET) + TARGET_INFO=--target $(TARGET) --example $(EXTENSION_NAME) + IS_EXAMPLE=examples endif ############################################# @@ -29,16 +32,16 @@ endif build_extension_library_debug: check_configure DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" ############################################# ### Misc ############################################# clean_rust: - cargo clean \ No newline at end of file + cargo clean From 78eb99832898e6603001df1336ac877f5a7bf2e0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 13 Nov 2024 00:54:07 +0100 Subject: [PATCH 25/41] emscripten invokation: skip verbose --- makefiles/duckdb_extension_c_api.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 3329db6..0e757f4 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -171,10 +171,10 @@ output_distribution_matrix: ifneq ($(DUCKDB_WASM_PLATFORM),) link_wasm_debug: - emcc build/debug/$(EXTENSION_LIB_FILENAME) -o build/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" -v + emcc build/debug/$(EXTENSION_LIB_FILENAME) -o build/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" link_wasm_release: - emcc build/release/$(EXTENSION_LIB_FILENAME) -o build/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" -v + emcc build/release/$(EXTENSION_LIB_FILENAME) -o build/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" else link_wasm_debug: From ce31455b6707055e6da21a9ce6d950bb2925992b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 13 Nov 2024 14:01:31 +0100 Subject: [PATCH 26/41] Move Wasm targets instructions into extension-ci-tools --- makefiles/duckdb_extension_c_api.Makefile | 33 ++++++++++++++++------- makefiles/duckdb_extension_rs.Makefile | 8 +++--- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 0e757f4..8f4c652 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -10,7 +10,7 @@ # LINUX_CI_IN_DOCKER : indicates that the build is being run in/out of Docker in the linux CI # SKIP_TESTS : makes the test targets turn into NOPs -.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure +.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure move_wasm_extension ############################################# ### Platform dependent config @@ -171,10 +171,10 @@ output_distribution_matrix: ifneq ($(DUCKDB_WASM_PLATFORM),) link_wasm_debug: - emcc build/debug/$(EXTENSION_LIB_FILENAME) -o build/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" + emcc build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME) -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" link_wasm_release: - emcc build/release/$(EXTENSION_LIB_FILENAME) -o build/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" + emcc build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME) -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" else link_wasm_debug: @@ -187,23 +187,23 @@ endif ############################################# build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/debug/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build/debug/$(EXTENSION_FILENAME) \ + -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(MINIMUM_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/debug/$(EXTENSION_FILENAME)', 'build/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" build_extension_with_metadata_release: check_configure link_wasm_release $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/release/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build/release/$(EXTENSION_FILENAME) \ + -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(MINIMUM_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/release/$(EXTENSION_FILENAME)', 'build/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" ############################################# ### Python @@ -234,4 +234,17 @@ configure_ci: $(CONFIGURE_CI_STEP) # Because the configure_ci may differ from configure, we don't automatically run configure on make build, this makes the error a bit nicer check_configure: @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/platform.txt'), 'The configure step appears to not be run. Please try running make configure'" - @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'" \ No newline at end of file + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'" + +move_wasm_extension: + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +wasm_mvp: + DUCKDB_PLATFORM=wasm_mvp make configure release move_wasm_extension + +wasm_eh: + DUCKDB_PLATFORM=wasm_eh make configure release move_wasm_extension + +wasm_threads: + DUCKDB_PLATFORM=wasm_coi make configure release move_wasm_extension diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/duckdb_extension_rs.Makefile index ffa817d..7e01edb 100644 --- a/makefiles/duckdb_extension_rs.Makefile +++ b/makefiles/duckdb_extension_rs.Makefile @@ -31,13 +31,13 @@ endif build_extension_library_debug: check_configure DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) - $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) - $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" ############################################# ### Misc From ca1ea1202815ace28787d64a88397d127b19ac02 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 13 Nov 2024 21:45:56 +0100 Subject: [PATCH 27/41] add c makefile --- makefiles/duckdb_extension_c.Makefile | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 makefiles/duckdb_extension_c.Makefile diff --git a/makefiles/duckdb_extension_c.Makefile b/makefiles/duckdb_extension_c.Makefile new file mode 100644 index 0000000..9edc51b --- /dev/null +++ b/makefiles/duckdb_extension_c.Makefile @@ -0,0 +1,102 @@ +# Reusable makefile for the C/C++ extensions targeting the C extension API +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# EXTENSION_LIB_FILENAME : the library name that is produced by the build +# MINIMUM_DUCKDB_VERSION : full version tag (including v) +# MINIMUM_DUCKDB_VERSION_MAJOR : major version +# MINIMUM_DUCKDB_VERSION_MINOR : minor version +# MINIMUM_DUCKDB_VERSION_PATCH : patch version +# CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass +# VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain +# VCPKG_TARGET_TRIPLET : vcpkg triplet to override +# GEN : allow specifying ninja as generator + +.PHONY: build_extension_library_debug build_extension_library_release update_duckdb_headers + +############################################# +### Base config +############################################# + +# Create build params to pass name and version +CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MAJOR=$(MINIMUM_DUCKDB_VERSION_MAJOR) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MINOR=$(MINIMUM_DUCKDB_VERSION_MINOR) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_PATCH=$(MINIMUM_DUCKDB_VERSION_PATCH) + +CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS) + +############################################# +### Vcpkg +############################################# + +ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") + CMAKE_BUILD_FLAGS += -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' +endif +ifneq ("${VCPKG_TARGET_TRIPLET}", "") + CMAKE_BUILD_FLAGS += -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' +endif + +############################################# +### Ninja +############################################# + +ifeq ($(GEN),ninja) + CMAKE_BUILD_FLAGS += -G "Ninja" +endif + +############################################# +### Windows weirdness +############################################# + +ifeq ($(OS),Windows_NT) + OUTPUT_LIB_PATH_DEBUG=cmake_build/debug/Debug/$(EXTENSION_LIB_FILENAME) + OUTPUT_LIB_PATH_RELEASE=cmake_build/release/Release/$(EXTENSION_LIB_FILENAME) +else + OUTPUT_LIB_PATH_DEBUG=cmake_build/debug/$(EXTENSION_LIB_FILENAME) + OUTPUT_LIB_PATH_RELEASE=cmake_build/release/$(EXTENSION_LIB_FILENAME) +endif + +ifeq ($(DUCKDB_PLATFORM),windows_amd64_rtools) + MINGW=1 +endif +ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) + MINGW=1 +endif +ifeq ($(MINGW),1) + EXTRA_COPY_STEP_DEBUG=$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('cmake_build/debug/Debug/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_DEBUG)')" + EXTRA_COPY_STEP_RELEASE=$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('cmake_build/release/Release/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_RELEASE)')" +endif + +############################################# +### Build targets +############################################# + +build_extension_library_debug: check_configure + cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug + cmake --build cmake_build/debug --config Debug + $(EXTRA_COPY_STEP_DEBUG) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + +build_extension_library_release: check_configure + cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release + cmake --build cmake_build/release --config Release + $(EXTRA_COPY_STEP_RELEASE) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + +############################################# +### Misc +############################################# +# TODO: switch this to use the $(MINIMUM_DUCKDB_VERSION) after v1.2.0 is released +BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include +DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h +DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h + +update_duckdb_headers: + $(PYTHON_VENV_BIN) -c "import urllib.request;urllib.request.urlretrieve('$(DUCKDB_C_HEADER_URL)', 'duckdb_capi/duckdb.h')" + $(PYTHON_VENV_BIN) -c "import urllib.request;urllib.request.urlretrieve('$(DUCKDB_C_EXTENSION_HEADER_URL)', 'duckdb_capi/duckdb_extension.h')" + +clean_cmake: + rm -rf cmake_build \ No newline at end of file From f801a7a404c14b77ad7b51a446f09903030dcb3a Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Thu, 14 Nov 2024 16:51:52 +0100 Subject: [PATCH 28/41] two small tweaks --- makefiles/duckdb_extension_c.Makefile | 4 ++-- makefiles/duckdb_extension_c_api.Makefile | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension_c.Makefile b/makefiles/duckdb_extension_c.Makefile index 9edc51b..6a5934f 100644 --- a/makefiles/duckdb_extension_c.Makefile +++ b/makefiles/duckdb_extension_c.Makefile @@ -64,8 +64,8 @@ ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) MINGW=1 endif ifeq ($(MINGW),1) - EXTRA_COPY_STEP_DEBUG=$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('cmake_build/debug/Debug/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_DEBUG)')" - EXTRA_COPY_STEP_RELEASE=$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('cmake_build/release/Release/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_RELEASE)')" + EXTRA_COPY_STEP_DEBUG=$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./cmake_build/debug/Debug').mkdir(parents=True, exist_ok=True);import shutil;shutil.copyfile('cmake_build/debug/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_DEBUG)')" + EXTRA_COPY_STEP_RELEASE=$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./cmake_build/release/Release').mkdir(parents=True, exist_ok=True);import shutil;shutil.copyfile('cmake_build/release/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_RELEASE)')" endif ############################################# diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile index 8f4c652..e8e5a4b 100644 --- a/makefiles/duckdb_extension_c_api.Makefile +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -124,6 +124,14 @@ ifeq ($(DUCKDB_PLATFORM),linux_amd64) SKIP_TESTS=1 endif +# The mingw/rtools can not be tested using the python test runner unfortunately +ifeq ($(DUCKDB_PLATFORM),windows_amd64_rtools) + SKIP_TESTS=1 +endif +ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) + SKIP_TESTS=1 +endif + ifeq ($(SKIP_TESTS),1) TEST_RELEASE_TARGET=tests_skipped TEST_DEBUG_TARGET=tests_skipped From 43646773afe48c99ce18cc43105df70523400a49 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 15 Nov 2024 10:14:55 +0100 Subject: [PATCH 29/41] rename --- .../base.Makefile} | 0 .../c_cpp.Makefile} | 0 .../rust.Makefile} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename makefiles/{duckdb_extension_c_api.Makefile => c_api_extensions/base.Makefile} (100%) rename makefiles/{duckdb_extension_c.Makefile => c_api_extensions/c_cpp.Makefile} (100%) rename makefiles/{duckdb_extension_rs.Makefile => c_api_extensions/rust.Makefile} (100%) diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/c_api_extensions/base.Makefile similarity index 100% rename from makefiles/duckdb_extension_c_api.Makefile rename to makefiles/c_api_extensions/base.Makefile diff --git a/makefiles/duckdb_extension_c.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile similarity index 100% rename from makefiles/duckdb_extension_c.Makefile rename to makefiles/c_api_extensions/c_cpp.Makefile diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/c_api_extensions/rust.Makefile similarity index 100% rename from makefiles/duckdb_extension_rs.Makefile rename to makefiles/c_api_extensions/rust.Makefile From 37750a4015a9f47ac030069b51bd12348bb21475 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 15 Nov 2024 10:46:16 +0100 Subject: [PATCH 30/41] add new templates to CI --- .github/workflows/TestCITools.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 3492ae9..443429d 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -24,6 +24,30 @@ jobs: ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;python3' + extension-template-capi: + name: Extension template (C API) + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: capi_quack + override_repository: duckdb/extension-template-c + override_ref: main + duckdb_version: v1.1.3 + override_ci_tools_repository: ${{ github.repository }} + ci_tools_version: ${{ github.sha }} + extra_toolchains: 'parser_tools;fortran;omp;go;python3' + + extension-template-rust: + name: Extension template (Rust) + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: rusty_quack + override_repository: duckdb/extension-template-rs + override_ref: main + duckdb_version: v1.1.3 + override_ci_tools_repository: ${{ github.repository }} + ci_tools_version: ${{ github.sha }} + extra_toolchains: 'parser_tools;fortran;omp;go;rust;python3' + delta-extension-main: name: Rust builds (using Delta extension) uses: ./.github/workflows/_extension_distribution.yml From 92969bbdc4fe17849e9a6aeb938fd67d73043423 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 15 Nov 2024 10:49:53 +0100 Subject: [PATCH 31/41] disable non-working archs --- .github/workflows/TestCITools.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 443429d..01f3958 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -35,6 +35,7 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;python3' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' # TODO: remove once fixed upstream extension-template-rust: name: Extension template (Rust) @@ -47,6 +48,7 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;rust;python3' + exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw' # TODO: remove once fixed upstream delta-extension-main: name: Rust builds (using Delta extension) From e8b396a513945ed0f4396d4c917d628d7d292ad1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 15 Nov 2024 11:25:28 +0100 Subject: [PATCH 32/41] trigger ci --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f20124b..2992f46 100644 --- a/README.md +++ b/README.md @@ -18,4 +18,4 @@ DuckDB's [Extension Template](https://github.com/duckdb/extension-template/actio | v0.10.0 | v0.10.0 | no | Each branch in this repository targets a specific version of DuckDB. Note that these branches will be continually updated to ensure the build environment is functional for that version of DuckDB. -Also note that at some point, support for versions will be dropped. Currently, we aim to support the latest 2 DuckDB versions, to allow extensions devs to transition to a new DuckDB version. +Also note that at some point, support for versions will be dropped. Currently, we aim to support the latest 2 DuckDB versions, to allow extensions devs to transition to a new DuckDB version. \ No newline at end of file From 902ed0580ebeede4acb16fd76dd291cdae451b5a Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 15 Nov 2024 12:27:46 +0100 Subject: [PATCH 33/41] disable rust threads ci run --- .github/workflows/TestCITools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 01f3958..5c7b244 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -48,7 +48,7 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;rust;python3' - exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw' # TODO: remove once fixed upstream + exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw;wasm_threads' # TODO: remove once fixed upstream delta-extension-main: name: Rust builds (using Delta extension) From a4c105082422fa8b7f729a96651a0dd985a1215b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 15 Nov 2024 15:15:33 +0100 Subject: [PATCH 34/41] Support also Wasm targets --- makefiles/c_api_extensions/c_cpp.Makefile | 33 +++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 6a5934f..95f4aaf 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -68,23 +68,40 @@ ifeq ($(MINGW),1) EXTRA_COPY_STEP_RELEASE=$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./cmake_build/release/Release').mkdir(parents=True, exist_ok=True);import shutil;shutil.copyfile('cmake_build/release/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_RELEASE)')" endif +CMAKE_WRAPPER= +EXTRA_CMAKE_FLAGS ?= + +ifneq ($(DUCKDB_WASM_PLATFORM),) + CMAKE_WRAPPER=emcmake + EXTRA_CMAKE_FLAGS += -DCMAKE_C_FLAGS="$(CMAKE_C_FLAGS) -fPIC" + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_eh') + EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC" + endif + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_eh') + EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC -fwasm-exceptions" + endif + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_threads') + EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC -fwasm-exceptions -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread" + endif +endif + ############################################# ### Build targets ############################################# build_extension_library_debug: check_configure - cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug - cmake --build cmake_build/debug --config Debug + $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug $(EXTRA_CMAKE_FLAGS) + cmake --build cmake_build/debug $(EXTRA_COPY_STEP_DEBUG) - $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure - cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release - cmake --build cmake_build/release --config Release + $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release $(EXTRA_CMAKE_FLAGS) + make -j8 -C cmake_build/release $(EXTRA_COPY_STEP_RELEASE) - $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" ############################################# ### Misc From 3d16a4d8fac83500df5ff9d515f942246d291f57 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 15 Nov 2024 16:03:14 +0100 Subject: [PATCH 35/41] Fixup invocation --- makefiles/c_api_extensions/c_cpp.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 95f4aaf..706a48c 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -91,14 +91,14 @@ endif build_extension_library_debug: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug $(EXTRA_CMAKE_FLAGS) - cmake --build cmake_build/debug + make -C cmake_build/debug $(EXTRA_COPY_STEP_DEBUG) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release $(EXTRA_CMAKE_FLAGS) - make -j8 -C cmake_build/release + make -C cmake_build/release $(EXTRA_COPY_STEP_RELEASE) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" From 83c40136b639a6dcbcec16f1f665f2ecfb211719 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 15 Nov 2024 16:11:25 +0100 Subject: [PATCH 36/41] Actually fix properly for ninja --- makefiles/c_api_extensions/c_cpp.Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 706a48c..7cba487 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -41,8 +41,11 @@ endif ### Ninja ############################################# +MAKE_INVOCATION = make + ifeq ($(GEN),ninja) CMAKE_BUILD_FLAGS += -G "Ninja" + MAKE_INVOCATION = ninja endif ############################################# @@ -91,14 +94,14 @@ endif build_extension_library_debug: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug $(EXTRA_CMAKE_FLAGS) - make -C cmake_build/debug + $(MAKE_INVOCATION) -C cmake_build/debug $(EXTRA_COPY_STEP_DEBUG) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release $(EXTRA_CMAKE_FLAGS) - make -C cmake_build/release + $(MAKE_INVOCATION) -C cmake_build/release $(EXTRA_COPY_STEP_RELEASE) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" @@ -116,4 +119,4 @@ update_duckdb_headers: $(PYTHON_VENV_BIN) -c "import urllib.request;urllib.request.urlretrieve('$(DUCKDB_C_EXTENSION_HEADER_URL)', 'duckdb_capi/duckdb_extension.h')" clean_cmake: - rm -rf cmake_build \ No newline at end of file + rm -rf cmake_build From 271ebe13d522a491c1ecb79b20e6757c567f7ea9 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 17 Nov 2024 13:49:40 +0100 Subject: [PATCH 37/41] Fixup cmake --build invocation for Wasm targets Problem is that emscriten wrapped cmake and naked cmake take slighlty different ways to actually build, so we need to abstract this away --- makefiles/c_api_extensions/c_cpp.Makefile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 7cba487..97c1935 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -72,19 +72,23 @@ ifeq ($(MINGW),1) endif CMAKE_WRAPPER= +CMAKE_BUILD_DEBUG = cmake --build cmake_build/debug --config Debug +CMAKE_BUILD_RELEASE = cmake --build cmake_build/release --config Release EXTRA_CMAKE_FLAGS ?= ifneq ($(DUCKDB_WASM_PLATFORM),) + CMAKE_BUILD_DEBUG = $(MAKE_INVOCATION) -C cmake_build/debug + CMAKE_BUILD_RELEASE = $(MAKE_INVOCATION) -C cmake_build/release CMAKE_WRAPPER=emcmake - EXTRA_CMAKE_FLAGS += -DCMAKE_C_FLAGS="$(CMAKE_C_FLAGS) -fPIC" - ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_eh') - EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC" + CMAKE_BUILD_FLAGS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON + EXTRA_CMAKE_FLAGS += -DDUCKDB_WASM_EXTENSION=1 + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_mvp') endif ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_eh') - EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC -fwasm-exceptions" + CMAKE_CXX_FLAGS += -fwasm-exceptions endif ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_threads') - EXTRA_CMAKE_FLAGS += -DCMAKE_CXX_FLAGS="$(CMAKE_CXX_FLAGS) -fPIC -fwasm-exceptions -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread" + CMAKE_CXX_FLAGS += -fwasm-exceptions -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread endif endif @@ -94,14 +98,14 @@ endif build_extension_library_debug: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug $(EXTRA_CMAKE_FLAGS) - $(MAKE_INVOCATION) -C cmake_build/debug + $(CMAKE_BUILD_DEBUG) $(EXTRA_COPY_STEP_DEBUG) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release $(EXTRA_CMAKE_FLAGS) - $(MAKE_INVOCATION) -C cmake_build/release + $(CMAKE_BUILD_RELEASE) $(EXTRA_COPY_STEP_RELEASE) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" From 0c1e35eee90dd25fe2c3ee48de8880df9b42933e Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 17 Nov 2024 21:50:52 +0100 Subject: [PATCH 38/41] vcpkg: Add handling for Wasm targets in c_api --- makefiles/c_api_extensions/c_cpp.Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 97c1935..70aab6a 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -32,6 +32,9 @@ CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS) ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") CMAKE_BUILD_FLAGS += -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' + ifneq ($(DUCKDB_WASM_PLATFORM),) + CMAKE_BUILD_FLAGS += -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + endif endif ifneq ("${VCPKG_TARGET_TRIPLET}", "") CMAKE_BUILD_FLAGS += -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' From f9500150415a894c69fd8b3742241dfe4f86b9c7 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Sun, 17 Nov 2024 22:18:31 +0100 Subject: [PATCH 39/41] Move from wasm_coi to wasm_threads --- makefiles/c_api_extensions/base.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index e8e5a4b..ecb2e7e 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -255,4 +255,4 @@ wasm_eh: DUCKDB_PLATFORM=wasm_eh make configure release move_wasm_extension wasm_threads: - DUCKDB_PLATFORM=wasm_coi make configure release move_wasm_extension + DUCKDB_PLATFORM=wasm_threads make configure release move_wasm_extension From 78529d2ebcdf81d6f4bd615f7c4e21913c401ae6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Tue, 19 Nov 2024 10:51:10 +0100 Subject: [PATCH 40/41] Enable Wasm for templates and remove unnecessary toolchains --- .github/workflows/TestCITools.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 5c7b244..5672617 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -34,8 +34,7 @@ jobs: duckdb_version: v1.1.3 override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} - extra_toolchains: 'parser_tools;fortran;omp;go;python3' - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads' # TODO: remove once fixed upstream + extra_toolchains: 'python3' extension-template-rust: name: Extension template (Rust) @@ -47,8 +46,8 @@ jobs: duckdb_version: v1.1.3 override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} - extra_toolchains: 'parser_tools;fortran;omp;go;rust;python3' - exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw;wasm_threads' # TODO: remove once fixed upstream + extra_toolchains: 'rust;python3' + exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw' # TODO: remove once fixed upstream delta-extension-main: name: Rust builds (using Delta extension) From b712ab898bc7eb7a969323a2cb2827d077499cd6 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 25 Nov 2024 09:56:34 +0100 Subject: [PATCH 41/41] remove pkg-config brew install --- .github/workflows/_extension_distribution.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 03a2d3d..6d527cb 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -344,7 +344,7 @@ jobs: - name: Install Ninja run: | - brew install ninja autoconf make libtool pkg-config automake autoconf-archive + brew install ninja autoconf make libtool automake autoconf-archive - name: Setup Ccache uses: hendrikmuhs/ccache-action@main