add support for unstable c api extensions for c/cpp makefiles

.github/workflows/TestCITools.yml

@@ -27,11 +27,12 @@ jobs:
   extension-template-capi:
     name: Extension template (C API)
     uses: ./.github/workflows/_extension_distribution.yml
+    if: false # TODO: re-enable when require changes are merged in template repo
     with:
       extension_name: capi_quack
       override_repository: duckdb/extension-template-c
       override_ref: main
-      duckdb_version: v1.1.3
+      duckdb_version: main
       override_ci_tools_repository: ${{ github.repository }}
       ci_tools_version: ${{ github.sha }}
       extra_toolchains: 'python3'

makefiles/c_api_extensions/base.Makefile

@@ -2,7 +2,8 @@
 #
 # Inputs
 #   EXTENSION_NAME         : name of the extension (lower case)
-#   MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports
+#   TARGET_DUCKDB_VERSION  : the target version of DuckDB that the extension targets
+# 	USE_UNSTABLE_C_API     : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version)
 #   EXTENSION_VERSION      : the version of the extension, if left blank it will be autodetected
 #   DUCKDB_PLATFORM        : the platform of the extension, if left blank it will be autodetected
 #   DUCKDB_TEST_VERSION    : the version of DuckDB to test with, if left blank will default to latest stable on PyPi
@@ -35,9 +36,9 @@ endif
 ### Main extension parameters
 #############################################
 
-# The minimum DuckDB version that this extension supports
-ifeq ($(MINIMUM_DUCKDB_VERSION),)
-	MINIMUM_DUCKDB_VERSION = v0.0.1
+# The target DuckDB version
+ifeq ($(TARGET_DUCKDB_VERSION),)
+	TARGET_DUCKDB_VERSION = v0.0.1
 endif
 
 EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension
@@ -88,6 +89,16 @@ extension_version: configure/extension_version.txt
 configure/extension_version.txt:
 	@ $(VERSION_COMMAND)
 
+#############################################
+### Parse DuckDB Semver
+#############################################
+
+# Either autodetect or use the provided value
+PARSE_SEMVER_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py -s $(TARGET_DUCKDB_VERSION)
+
+parse_duckdb_version:
+	@ $(PARSE_SEMVER_COMMAND)
+
 #############################################
 ### Testing
 #############################################
@@ -100,14 +111,19 @@ TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSI
 TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension
 
 # By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version
-DUCKDB_INSTALL_VERSION?=
-DUCKDB_INSTALL_OPTIONS?=
-ifneq ($(DUCKDB_TEST_VERSION),)
-	DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION)
-else ifeq ($(DUCKDB_GIT_VERSION),main)
-	DUCKDB_INSTALL_OPTIONS= --pre
+DUCKDB_PIP_INSTALL?=duckdb
+ifeq ($(DUCKDB_TEST_VERSION),main)
+	DUCKDB_PIP_INSTALL=--pre duckdb
+else ifneq ($(DUCKDB_TEST_VERSION),)
+	DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_TEST_VERSION)
+endif
+
+# This allows C API extensions to be tested against a prerelease of DuckDB. This only really makes sense when DuckDB already
+# has stabilized the C API for the upcoming release.
+ifeq ($(DUCKDB_GIT_VERSION),main)
+	DUCKDB_PIP_INSTALL=--pre duckdb
 else ifneq ($(DUCKDB_GIT_VERSION),)
-	DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION)
+	DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_GIT_VERSION)
 endif
 
 TEST_RELEASE_TARGET=test_extension_release_internal
@@ -199,24 +215,29 @@ endif
 #############################################
 ### Adding metadata
 #############################################
+UNSTABLE_C_API_FLAG=
+ifeq ($(USE_UNSTABLE_C_API),1)
+	UNSTABLE_C_API_FLAG+=--abi-type C_STRUCT_UNSTABLE
+endif
+
 build_extension_with_metadata_debug: check_configure link_wasm_debug
 	$(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \
 			-l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \
 			-o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \
 			-n $(EXTENSION_NAME) \
-			-dv $(MINIMUM_DUCKDB_VERSION) \
+			-dv $(TARGET_DUCKDB_VERSION) \
 			-evf configure/extension_version.txt \
-			-pf configure/platform.txt
+			-pf configure/platform.txt $(UNSTABLE_C_API_FLAG)
 	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')"
 
 build_extension_with_metadata_release: check_configure link_wasm_release
 	$(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \
 			-l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \
 			-o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \
 			-n $(EXTENSION_NAME) \
-			-dv $(MINIMUM_DUCKDB_VERSION) \
+			-dv $(TARGET_DUCKDB_VERSION) \
 			-evf configure/extension_version.txt \
-			-pf configure/platform.txt
+			-pf configure/platform.txt $(UNSTABLE_C_API_FLAG)
 	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')"
 
 #############################################
@@ -229,8 +250,9 @@ venv: configure/venv
 
 configure/venv:
 	$(PYTHON_BIN) -m venv configure/venv
-	$(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' $(DUCKDB_INSTALL_OPTIONS)
+	$(PYTHON_VENV_BIN) -m pip install $(DUCKDB_PIP_INSTALL)
 	$(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python
+	$(PYTHON_VENV_BIN) -m pip install packaging
 
 #############################################
 ### Configure

makefiles/c_api_extensions/c_cpp.Makefile

@@ -3,10 +3,8 @@
 # Inputs
 #   EXTENSION_NAME               : name of the extension (lower case)
 #   EXTENSION_LIB_FILENAME       : the library name that is produced by the build
-#	MINIMUM_DUCKDB_VERSION       : full version tag (including v)
-#	MINIMUM_DUCKDB_VERSION_MAJOR : major version
-#	MINIMUM_DUCKDB_VERSION_MINOR : minor version
-#	MINIMUM_DUCKDB_VERSION_PATCH : patch version
+# 	USE_UNSTABLE_C_API           : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version)
+#   TARGET_DUCKDB_VERSION        : the target version of DuckDB that the extension targets
 #	CMAKE_EXTRA_BUILD_FLAGS      : additional CMake flags to pass
 #	VCPKG_TOOLCHAIN_PATH         : path to vcpkg toolchain
 #	VCPKG_TARGET_TRIPLET         : vcpkg triplet to override
@@ -18,11 +16,31 @@
 ### Base config
 #############################################
 
+# Get parsed SemVer for Stable C API
+FILE_MAJOR := ./configure/duckdb_version_major.txt
+FILE_MINOR := ./configure/duckdb_version_minor.txt
+FILE_PATCH := ./configure/duckdb_version_patch.txt
+MAJOR_VERSION := $(file < $(FILE_MAJOR))
+MINOR_VERSION := $(file < $(FILE_MINOR))
+PATCH_VERSION := $(file < $(FILE_PATCH))
+
 # Create build params to pass name and version
 CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME)
-CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MAJOR=$(MINIMUM_DUCKDB_VERSION_MAJOR)
-CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MINOR=$(MINIMUM_DUCKDB_VERSION_MINOR)
-CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_PATCH=$(MINIMUM_DUCKDB_VERSION_PATCH)
+
+# Set the parsed semver defines
+ifneq ($(MAJOR_VERSION),)
+	CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MAJOR=$(MAJOR_VERSION)
+endif
+ifneq ($(MINOR_VERSION),)
+	CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MINOR=$(MINOR_VERSION)
+endif
+ifneq ($(PATCH_VERSION),)
+	CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_PATCH=$(PATCH_VERSION)
+endif
+
+ifeq ($(USE_UNSTABLE_C_API),1)
+	CMAKE_VERSION_PARAMS += -DDUCKDB_EXTENSION_API_VERSION_UNSTABLE=$(TARGET_DUCKDB_VERSION)
+endif
 
 CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS)
 
@@ -116,8 +134,14 @@ build_extension_library_release: check_configure
 #############################################
 ### Misc
 #############################################
-# TODO: switch this to use the $(MINIMUM_DUCKDB_VERSION) after v1.2.0 is released
-BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include
+# TODO: switch this to use the $(TARGET_DUCKDB_VERSION) after v1.2.0 is released
+
+BASE_HEADER_URL=
+ifneq ($(TARGET_DUCKDB_VERSION),)
+	BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/$(TARGET_DUCKDB_VERSION)/src/include
+else
+	BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include
+endif
 DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h
 DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h
 

makefiles/c_api_extensions/rust.Makefile

@@ -30,12 +30,12 @@ endif
 #############################################
 
 build_extension_library_debug: check_configure
-	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO)
+	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO)
 	$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)"
 	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')"
 
 build_extension_library_release: check_configure
-	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO)
+	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO)
 	$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)"
 	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')"
 

scripts/configure_helper.py

@@ -11,6 +11,7 @@ def main():
 
     arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true')
     arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true')
+    arg_parser.add_argument('-s', '--parse-duckdb-semver', type=str, help='Write the parsed DuckDB version SemVer')
 
     args = arg_parser.parse_args()
 
@@ -40,6 +41,32 @@ def main():
             print(f"Writing platform {duckdb_platform} to {platform_file}")
             f.write(duckdb_platform)
 
+    # Write parsed semver
+    if args.parse_duckdb_semver:
+        from packaging.version import Version, InvalidVersion
+        major_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_major.txt"))
+        minor_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_minor.txt"))
+        patch_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_patch.txt"))
+
+        major_version = ""
+        minor_version = ""
+        patch_version = ""
+
+        try:
+            version = Version(args.parse_duckdb_semver)
+            major_version = f"{version.major}"
+            minor_version = f"{version.minor}"
+            patch_version = f"{version.micro}"
+            print(f"Written parsed DuckDB semver v{version} to {OUTPUT_DIR}/duckdb_version_<part>.txt")
+        except InvalidVersion:
+            print(f"DuckDB version is not a semver, writing empty parsed semver files to {OUTPUT_DIR}/duckdb_version_<part>.txt")
+
+        with open(major_file, 'w') as f:
+            f.write(major_version)
+        with open(minor_file, 'w') as f:
+            f.write(minor_version)
+        with open(patch_file, 'w') as f:
+            f.write(patch_version)
 
 if __name__ == '__main__':
     main()