From 7d7cae189bd00735df04c2a5e7031b14e56e112a Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 4 Dec 2024 13:41:59 +0100 Subject: [PATCH 1/8] add support for unstable c api extensions --- makefiles/c_api_extensions/base.Makefile | 22 ++++++++++++++-------- makefiles/c_api_extensions/c_cpp.Makefile | 21 +++++++++++++-------- makefiles/c_api_extensions/rust.Makefile | 4 ++-- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index e8e5a4b..0515d1e 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -2,7 +2,8 @@ # # Inputs # EXTENSION_NAME : name of the extension (lower case) -# MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension supports +# USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) # EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected # DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected # DUCKDB_TEST_VERSION : the version of DuckDB to test with, if left blank will default to latest stable on PyPi @@ -35,9 +36,9 @@ endif ### Main extension parameters ############################################# -# The minimum DuckDB version that this extension supports -ifeq ($(MINIMUM_DUCKDB_VERSION),) - MINIMUM_DUCKDB_VERSION = v0.0.1 +# The target DuckDB version +ifeq ($(TARGET_DUCKDB_VERSION),) + TARGET_DUCKDB_VERSION = v0.0.1 endif EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension @@ -193,14 +194,19 @@ endif ############################################# ### Adding metadata ############################################# +UNSTABLE_C_API_FLAG= +ifeq ($(USE_UNSTABLE_C_API),1) + UNSTABLE_C_API_FLAG+=--abi-type C_STRUCT_UNSTABLE +endif + build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ - -dv $(MINIMUM_DUCKDB_VERSION) \ + -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ - -pf configure/platform.txt + -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" build_extension_with_metadata_release: check_configure link_wasm_release @@ -208,9 +214,9 @@ build_extension_with_metadata_release: check_configure link_wasm_release -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ - -dv $(MINIMUM_DUCKDB_VERSION) \ + -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ - -pf configure/platform.txt + -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" ############################################# diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 6a5934f..7e704fd 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -3,10 +3,11 @@ # Inputs # EXTENSION_NAME : name of the extension (lower case) # EXTENSION_LIB_FILENAME : the library name that is produced by the build -# MINIMUM_DUCKDB_VERSION : full version tag (including v) -# MINIMUM_DUCKDB_VERSION_MAJOR : major version -# MINIMUM_DUCKDB_VERSION_MINOR : minor version -# MINIMUM_DUCKDB_VERSION_PATCH : patch version +# USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) +# TARGET_DUCKDB_VERSION : full version tag (including v) +# TARGET_DUCKDB_VERSION_MAJOR : target major version +# TARGET_DUCKDB_VERSION_MINOR : target minor version +# TARGET_DUCKDB_VERSION_PATCH : target patch version # CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass # VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain # VCPKG_TARGET_TRIPLET : vcpkg triplet to override @@ -20,9 +21,13 @@ # Create build params to pass name and version CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MAJOR=$(MINIMUM_DUCKDB_VERSION_MAJOR) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MINOR=$(MINIMUM_DUCKDB_VERSION_MINOR) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_PATCH=$(MINIMUM_DUCKDB_VERSION_PATCH) +CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MAJOR=$(TARGET_DUCKDB_VERSION_MAJOR) +CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MINOR=$(TARGET_DUCKDB_VERSION_MINOR) +CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_PATCH=$(TARGET_DUCKDB_VERSION_PATCH) + +ifeq ($(USE_UNSTABLE_C_API),1) + CMAKE_VERSION_PARAMS += -DDUCKDB_EXTENSION_API_VERSION_UNSTABLE=$(TARGET_DUCKDB_VERSION) +endif CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS) @@ -89,7 +94,7 @@ build_extension_library_release: check_configure ############################################# ### Misc ############################################# -# TODO: switch this to use the $(MINIMUM_DUCKDB_VERSION) after v1.2.0 is released +# TODO: switch this to use the $(TARGET_DUCKDB_VERSION) after v1.2.0 is released BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h diff --git a/makefiles/c_api_extensions/rust.Makefile b/makefiles/c_api_extensions/rust.Makefile index 7e01edb..797ba4f 100644 --- a/makefiles/c_api_extensions/rust.Makefile +++ b/makefiles/c_api_extensions/rust.Makefile @@ -30,12 +30,12 @@ endif ############################################# build_extension_library_debug: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" From 10b3ee2a7a8531f01420fba1a990b6cd39f74cef Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 17 Dec 2024 14:56:50 +0100 Subject: [PATCH 2/8] add semver parsing --- makefiles/c_api_extensions/base.Makefile | 25 +++++++++++++++------ makefiles/c_api_extensions/c_cpp.Makefile | 27 +++++++++++++++++------ scripts/configure_helper.py | 27 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index 0515d1e..e4cf2e6 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -2,7 +2,7 @@ # # Inputs # EXTENSION_NAME : name of the extension (lower case) -# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension supports +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension supports # USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) # EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected # DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected @@ -89,6 +89,16 @@ extension_version: configure/extension_version.txt configure/extension_version.txt: @ $(VERSION_COMMAND) +############################################# +### Parse DuckDB Semver +############################################# + +# Either autodetect or use the provided value +PARSE_SEMVER_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py -s $(TARGET_DUCKDB_VERSION) + +parse_duckdb_version: + @ $(PARSE_SEMVER_COMMAND) + ############################################# ### Testing ############################################# @@ -201,23 +211,23 @@ endif build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ + -l build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" build_extension_with_metadata_release: check_configure link_wasm_release $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ + -l build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" ############################################# ### Python @@ -231,6 +241,7 @@ configure/venv: $(PYTHON_BIN) -m venv configure/venv $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python + $(PYTHON_VENV_BIN) -m pip install packaging ############################################# ### Configure diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 7e704fd..342c6b4 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -4,10 +4,7 @@ # EXTENSION_NAME : name of the extension (lower case) # EXTENSION_LIB_FILENAME : the library name that is produced by the build # USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) -# TARGET_DUCKDB_VERSION : full version tag (including v) -# TARGET_DUCKDB_VERSION_MAJOR : target major version -# TARGET_DUCKDB_VERSION_MINOR : target minor version -# TARGET_DUCKDB_VERSION_PATCH : target patch version +# TARGET_DUCKDB_VERSION : full version # CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass # VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain # VCPKG_TARGET_TRIPLET : vcpkg triplet to override @@ -19,11 +16,27 @@ ### Base config ############################################# +# Get parsed SemVer for Stable C API +FILE_MAJOR := ./configure/duckdb_version_major.txt +FILE_MINOR := ./configure/duckdb_version_minor.txt +FILE_PATCH := ./configure/duckdb_version_patch.txt +MAJOR_VERSION := $(file < $(FILE_MAJOR)) +MINOR_VERSION := $(file < $(FILE_MINOR)) +PATCH_VERSION := $(file < $(FILE_PATCH)) + # Create build params to pass name and version CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME) -CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MAJOR=$(TARGET_DUCKDB_VERSION_MAJOR) -CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MINOR=$(TARGET_DUCKDB_VERSION_MINOR) -CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_PATCH=$(TARGET_DUCKDB_VERSION_PATCH) + +# Set the parsed semver defines +ifneq ($(MAJOR_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MAJOR=$(MAJOR_VERSION) +endif +ifneq ($(MINOR_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MINOR=$(MINOR_VERSION) +endif +ifneq ($(PATCH_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_PATCH=$(PATCH_VERSION) +endif ifeq ($(USE_UNSTABLE_C_API),1) CMAKE_VERSION_PARAMS += -DDUCKDB_EXTENSION_API_VERSION_UNSTABLE=$(TARGET_DUCKDB_VERSION) diff --git a/scripts/configure_helper.py b/scripts/configure_helper.py index 9685410..4448868 100644 --- a/scripts/configure_helper.py +++ b/scripts/configure_helper.py @@ -11,6 +11,7 @@ def main(): arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true') arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true') + arg_parser.add_argument('-s', '--parse-duckdb-semver', type=str, help='Write the parsed DuckDB version SemVer') args = arg_parser.parse_args() @@ -40,6 +41,32 @@ def main(): print(f"Writing platform {duckdb_platform} to {platform_file}") f.write(duckdb_platform) + # Write parsed semver + if args.parse_duckdb_semver: + from packaging.version import Version, InvalidVersion + major_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_major.txt")) + minor_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_minor.txt")) + patch_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_patch.txt")) + + major_version = "" + minor_version = "" + patch_version = "" + + try: + version = Version(args.parse_duckdb_semver) + major_version = f"{version.major}" + minor_version = f"{version.minor}" + patch_version = f"{version.micro}" + print(f"Written parsed DuckDB semver v{version} to {OUTPUT_DIR}/duckdb_version_.txt") + except InvalidVersion: + print(f"DuckDB version is not a semver, writing empty parsed semver files to {OUTPUT_DIR}/duckdb_version_.txt") + + with open(major_file, 'w') as f: + f.write(major_version) + with open(minor_file, 'w') as f: + f.write(minor_version) + with open(patch_file, 'w') as f: + f.write(patch_version) if __name__ == '__main__': main() \ No newline at end of file From 2aba1a3083ab34849a24f76e7ad39b74e77ab292 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 17 Dec 2024 18:22:40 +0100 Subject: [PATCH 3/8] restore slashes, point ci to dev branch --- .github/workflows/TestCITools.yml | 8 ++++---- makefiles/c_api_extensions/base.Makefile | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 5672617..dd0e136 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -29,9 +29,9 @@ jobs: uses: ./.github/workflows/_extension_distribution.yml with: extension_name: capi_quack - override_repository: duckdb/extension-template-c - override_ref: main - duckdb_version: v1.1.3 + override_repository: samansmink/extension-template-c + override_ref: prepare-for-v1.2.0 + duckdb_version: main override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'python3' @@ -43,7 +43,7 @@ jobs: extension_name: rusty_quack override_repository: duckdb/extension-template-rs override_ref: main - duckdb_version: v1.1.3 + duckdb_version: main override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'rust;python3' diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index fa6b050..8fba23b 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -211,13 +211,13 @@ endif build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ + -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" build_extension_with_metadata_release: check_configure link_wasm_release $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ @@ -227,7 +227,7 @@ build_extension_with_metadata_release: check_configure link_wasm_release -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) - $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" ############################################# ### Python From 527e25f972b308f501c7c3ca5511feac7c9970d1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 18 Dec 2024 11:14:38 +0100 Subject: [PATCH 4/8] add option to test with PyPi prerelease --- makefiles/c_api_extensions/base.Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index 8fba23b..aa9e6e3 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -111,13 +111,17 @@ TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSI TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension # By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version -DUCKDB_INSTALL_VERSION?= +DUCKDB_PIP_INSTALL?= ifneq ($(DUCKDB_TEST_VERSION),) - DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION) + DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_TEST_VERSION) endif -ifneq ($(DUCKDB_GIT_VERSION),) - DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION) +# This allows C API extensions to be tested against a prerelease of DuckDB. This only really makes sense when DuckDB already +# has stabilized the C API for the upcoming release. +ifeq ($(DUCKDB_GIT_VERSION),main) + DUCKDB_PIP_INSTALL=--pre duckdb +else ifneq ($(DUCKDB_GIT_VERSION),) + DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_GIT_VERSION) endif TEST_RELEASE_TARGET=test_extension_release_internal @@ -239,7 +243,7 @@ venv: configure/venv configure/venv: $(PYTHON_BIN) -m venv configure/venv - $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' + $(PYTHON_VENV_BIN) -m pip install $(DUCKDB_PIP_INSTALL) $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python $(PYTHON_VENV_BIN) -m pip install packaging From 490962e1dbcf7975e8ed91194e5b6ce170e1c680 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 21 Jan 2025 15:28:46 +0100 Subject: [PATCH 5/8] small tweaks to capi makefiles --- makefiles/c_api_extensions/base.Makefile | 8 +++++--- makefiles/c_api_extensions/c_cpp.Makefile | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index 348b221..f86a247 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -2,7 +2,7 @@ # # Inputs # EXTENSION_NAME : name of the extension (lower case) -# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension supports +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension targets # USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) # EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected # DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected @@ -111,8 +111,10 @@ TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSI TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension # By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version -DUCKDB_PIP_INSTALL?= -ifneq ($(DUCKDB_TEST_VERSION),) +DUCKDB_PIP_INSTALL?=duckdb +ifeq ($(DUCKDB_TEST_VERSION),main) + DUCKDB_PIP_INSTALL=--pre duckdb +else ifneq ($(DUCKDB_TEST_VERSION),) DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_TEST_VERSION) endif diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 784d0b2..e90e04a 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -4,7 +4,7 @@ # EXTENSION_NAME : name of the extension (lower case) # EXTENSION_LIB_FILENAME : the library name that is produced by the build # USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) -# TARGET_DUCKDB_VERSION : full version +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension targets # CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass # VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain # VCPKG_TARGET_TRIPLET : vcpkg triplet to override @@ -135,7 +135,13 @@ build_extension_library_release: check_configure ### Misc ############################################# # TODO: switch this to use the $(TARGET_DUCKDB_VERSION) after v1.2.0 is released -BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include + +BASE_HEADER_URL= +ifneq ($(TARGET_DUCKDB_VERSION),) + BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/$(TARGET_DUCKDB_VERSION)/src/include +else + BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include +endif DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h From ccd8aa37ac4c649adc7248f5b6114f067d7fb7df Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 21 Jan 2025 17:33:58 +0100 Subject: [PATCH 6/8] restore rust to target v1.1.3 for now --- .github/workflows/TestCITools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 96cd78d..13ebaa7 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -43,7 +43,7 @@ jobs: extension_name: rusty_quack override_repository: duckdb/extension-template-rs override_ref: main - duckdb_version: main + duckdb_version: v1.1.3 override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'rust;python3' From c4ba83cee0d40da1b91d660574a2f5b9b0a9001e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 21 Jan 2025 17:48:17 +0100 Subject: [PATCH 7/8] disable ci running c ext template for now --- .github/workflows/TestCITools.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 13ebaa7..46c5d8b 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -27,6 +27,7 @@ jobs: extension-template-capi: name: Extension template (C API) uses: ./.github/workflows/_extension_distribution.yml + if: false # TODO: re-enable after v1.2 with: extension_name: capi_quack override_repository: samansmink/extension-template-c From 1e250beaa184c5af45f0dab750d33b1a6881f23d Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 21 Jan 2025 17:57:46 +0100 Subject: [PATCH 8/8] fix wasm --- .github/workflows/TestCITools.yml | 6 +++--- makefiles/c_api_extensions/base.Makefile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 46c5d8b..e04aa57 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -27,11 +27,11 @@ jobs: extension-template-capi: name: Extension template (C API) uses: ./.github/workflows/_extension_distribution.yml - if: false # TODO: re-enable after v1.2 + if: false # TODO: re-enable when require changes are merged in template repo with: extension_name: capi_quack - override_repository: samansmink/extension-template-c - override_ref: prepare-for-v1.2.0 + override_repository: duckdb/extension-template-c + override_ref: main duckdb_version: main override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index f86a247..1fc63a2 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -232,8 +232,8 @@ build_extension_with_metadata_debug: check_configure link_wasm_debug build_extension_with_metadata_release: check_configure link_wasm_release $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ - -l build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ - -o build$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ + -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \