From 2bccd6f3b965f4b3fa924814312fd4b9a0ab0d51 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 13:52:20 +0200
Subject: [PATCH 01/14] revert previous fix and switch to
 ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION to enable rust builds

---
 .github/workflows/_extension_distribution.yml | 266 ++++++++----------
 1 file changed, 112 insertions(+), 154 deletions(-)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index 647f9cf..b0045b7 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -124,6 +124,7 @@ jobs:
   linux:
     name: Linux
     runs-on: ubuntu-latest
+    container: ${{ matrix.container }}
     needs: generate_matrix
     if: ${{ needs.generate_matrix.outputs.linux_matrix != '{}' && needs.generate_matrix.outputs.linux_matrix != '' }}
     strategy:
@@ -131,19 +132,31 @@ jobs:
     env:
       VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+      GEN: ninja
       BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }}
       DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
-      REPO_REF: ${{ inputs.override_ref }}
-      REPO_NAME: ${{ inputs.override_repository != '' && inputs.override_repository || github.repository }}
-      GITHUB_REPOSITORY: ${{ github.repository }}
-      ENABLE_RUST: '0'
-      CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || 'gcc' }}
-      CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || 'g++' }}
-      RUN_TESTS: ${{ matrix.duckdb_arch != 'linux_arm64' && '1' }}
-      AARCH64_CROSS_COMPILE: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }}
-      CONTAINER: ${{ matrix.container }}
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
+      - name: Install required ubuntu packages
+        if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
+        run: |
+          apt-get update -y -qq
+          apt-get install -y -qq software-properties-common
+          add-apt-repository ppa:git-core/ppa
+          apt-get update -y -qq
+          apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
+
+      - name: Install Git 2.18.5
+        if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
+        run: |
+          wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
+          tar xvf v2.18.5.tar.gz
+          cd git-2.18.5
+          make
+          make prefix=/usr install
+          git --version
+
       - uses: actions/checkout@v3
         name: Checkout override repository
         if: ${{inputs.override_repository != ''}}
@@ -160,156 +173,101 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - name: Setup and build extension
-        shell: bash
+      - name: Checkout DuckDB to version
         run: |
-          export CURR_DIR=/home/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/}
-          cat > script.sh << ENDOFLINE
-          VCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}
-          export VCPKG_BUILD=1
-          export DUCKDB_PLATFORM="${DUCKDB_PLATFORM}"
-          export BUILD_SHELL="${BUILD_SHELL}"
-          export VCPKG_TOOLCHAIN_PATH=$CURR_DIR/vcpkg/scripts/buildsystems/vcpkg.cmake
-          set -eo pipefail
-          ###
-          echo "Install required ubuntu packages"
-          cd $CURR_DIR
-          if [ "${DUCKDB_PLATFORM}" != "linux_amd64_gcc4" ]; then
-            apt-get update -y -qq
-            apt-get install -y -qq software-properties-common
-            add-apt-repository ppa:git-core/ppa
-            apt-get update -y -qq
-            apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
-          fi
-          ###
-          echo "Install Git 2.18.5"
-          cd $CURR_DIR
-          if [ "${DUCKDB_PLATFORM}" != "linux_amd64_gcc4" ]; then
-            wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz
-            tar xvf v2.18.5.tar.gz
-            cd git-2.18.5
-            make
-            make prefix=/usr install
-            git --version
-          fi
-          ###
-          echo "Checkout DuckDB to version"
-          cd $CURR_DIR
-          git config --global --add safe.directory $CURR_DIR
-          git config --global --add safe.directory $CURR_DIR/duckdb
           cd duckdb
           git checkout ${{ inputs.duckdb_version }}
-          ###
-          echo "Fix CentOS 7 EOL"
-          cd $CURR_DIR
-          if [[ "${DUCKDB_PLATFORM}" = "linux_amd64_gcc4" ]]; then
-            # Workaround the fact that CentOS 7 is EOL
-            sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-            sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-            yum-config-manager --disable centos-sclo-rh
-            yum install epel-release -y curl
-          fi
-          ###
-          echo "Setup ManyLinux2014"
-          cd $CURR_DIR
-          if [ "${DUCKDB_PLATFORM}" = "linux_amd64_gcc4" ]; then
-            ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl vcpkg
-          else
-            apt-get update -y -qq
-            apt-get install -y -qq software-properties-common
-            add-apt-repository ppa:git-core/ppa
-            apt-get update -y -qq
-            apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config git
-            wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh
-            chmod +x cmake-3.21.3-linux-x86_64.sh
-            ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local
-            cmake --version
-            # echo "Install Python 3.8"
-            # wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz
-            # tar xvf Python-3.8.17.tgz
-            # cd Python-3.8.17
-            # mkdir -p pythonbin
-            # ./configure --with-ensurepip=install
-            # make -j
-            # make install
-            # python3.8 --version
-            # python3.8 -m pip install pip
-            # python3.8 -m pip install requests awscli
-            echo "Install"
-            if [ -n "${AARCH64_CROSS_COMPILE}" ]; then
-              apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
-            fi
-            git checkout ${{ inputs.git_ref }}
-            #python3 --version
-            #python3 -m pip install pip
-            #python3 -m pip install requests awscli
-            echo "Version Check"
-            ldd --version ldd
-            #python3 --version
-            git --version
-            git log -1 --format=%h
-          fi
-          ###
-          echo "Checkout, again"
-          cd $CURR_DIR
-          if [ -n "${REPO_REF}" ]; then
-            git checkout $REPO_REF
-          fi
-          ###
-          echo "Checkout DuckDB to version"
-          cd $CURR_DIR
+
+      - name: Fix CentOS 7 EOL
+        if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
+        run: |
+          # Workaround the fact that CentOS 7 is EOL
+          sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+          sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+          yum-config-manager --disable centos-sclo-rh
+          yum install epel-release -y
+
+      - name: Setup ManyLinux2014
+        if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
+        run: |
+          ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
+
+      - name: Setup Ubuntu
+        if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
+        uses: ./duckdb/.github/actions/ubuntu_18_setup
+        with:
+          aarch64_cross_compile: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }}
+
+      - uses: actions/checkout@v3
+        name: Checkout override repository
+        if: ${{inputs.override_repository != ''}}
+        with:
+          repository: ${{ inputs.override_repository }}
+          ref: ${{ inputs.override_ref }}
+          fetch-depth: 0
+          submodules: 'true'
+
+      - uses: actions/checkout@v3
+        name: Checkout current repository
+        if: ${{inputs.override_repository == ''}}
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Checkout DuckDB to version
+        run: |
           cd duckdb
           git checkout ${{ inputs.duckdb_version }}
-          ###
-          echo "Setup vcpkg"
-          cd $CURR_DIR
-          git clone https://github.com/microsoft/vcpkg.git
-          cd vcpkg
-          git fetch
-          git checkout a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
-          #export CC="${CC}"
-          #export CXX="${CXX}"
-          ./bootstrap-vcpkg.sh
-          ###
-          echo "Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)"
-          cd $CURR_DIR
-          if [ -n "${ENABLE_RUST}" ]; then
-            curl --proto "https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-            #PATH="$HOME/.cargo/bin:$PATH"
-            #. "$HOME/.cargo/env"
-          fi
-          # - name: Setup Ccache
-          #   uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
-          #   continue-on-error: true
-          #   with:
-          #      key: ${{ github.job }}-${{ matrix.duckdb_arch }}
-          ###
-          #echo "Configure OpenSSL for Rust"
-          #cd $CURR_DIR
-          #if [ -n "${ENABLE_RUST}" ]; then
-          OPENSSL_ROOT_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET
-          OPENSSL_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET
-          OPENSSL_INCLUDE_DIR=$CURR_DIR/build/release/vcpkg_installed/$VCPKG_TARGET_TRIPLET/include
-          OPENSSL_USE_STATIC_LIBS=true
-          #fi
-          ###
-          echo "Build extension"
-          cd $CURR_DIR
-          # export VCPKG_TOOLCHAIN_PATH=/home/runner/work/quack/quack/vcpkgs/script/buildsystems/vcpkg.cmake
-          #export CC="${CC}"
-          #export CXX="${CXX}"
-          GEN=ninja make release
-          ###
-          echo "Test extension"
-          cd $CURR_DIR
-          if [ -n "${RUN_TESTS}" ]; then
-            make test
-          fi
-          ENDOFLINE
-          chmod +x script.sh
-          cat script.sh | docker run -i --rm -v /home/runner/work:/home/runner/work --workdir $CURR_DIR $CONTAINER
 
-      - uses: actions/upload-artifact@v3
+      - name: Setup Rust
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64'}}
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Setup Rust for cross compilation
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_arm64'}}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: aarch64-unknown-linux-gnu
+
+      - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64_gcc4' }}
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
+        continue-on-error: true
+        with:
+          key: ${{ github.job }}-${{ matrix.duckdb_arch }}
+
+      - name: Setup vcpkg
+        uses: lukka/run-vcpkg@v11.1
+        with:
+          vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
+
+      - name: Configure OpenSSL for Rust
+        if: ${{ inputs.enable_rust }}
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+      - name: Build extension
+        env:
+          GEN: ninja
+          CC: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }}
+          CXX: ${{ matrix.duckdb_arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }}
+          DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
+        run: |
+          make release
+
+      - name: Test extension
+        if: ${{ matrix.duckdb_arch != 'linux_arm64'}}
+        run: |
+          make test
+
+      - uses: actions/upload-artifact@v2
         with:
           name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
           path: |
@@ -543,4 +501,4 @@ jobs:
           if-no-files-found: error
           name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}
           path: |
-            build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm
+            build/${{ matrix.duckdb_arch }}/extension/${{ inputs.extension_name }}/${{ inputs.extension_name }}.duckdb_extension.wasm
\ No newline at end of file

From b1c8ba4380fec80ac49c93479bac9646ae3e57b8 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 13:58:26 +0200
Subject: [PATCH 02/14] add ci

---
 .github/workflows/TestCITools.yml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/workflows/TestCITools.yml

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
new file mode 100644
index 0000000..341afaf
--- /dev/null
+++ b/.github/workflows/TestCITools.yml
@@ -0,0 +1,31 @@
+#
+# This tests the extension-ci-tools works as expected
+#
+name: Test CI Tools
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  extension-template-main:
+    name: Build the extension template
+    uses: ./.github/workflows/_extension_distribution.yml
+    with:
+      extension_name: quack
+      override_repository: duckdb/extension-template
+      override_ref: main
+      duckdb_version: v1.0.0
+  delta-extension-main:
+    name: Build the Delta extension (to test Rust)
+    uses: ./.github/workflows/_extension_distribution.yml
+    with:
+      extension_name: delta
+      enable_rust: true
+      override_repository: duckdb/duckdb_delta
+      override_ref: main
+      duckdb_version: v1.0.0

From dc60299ff3e67f123d6ceb8a2ecddd56340fbcce Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:13:52 +0200
Subject: [PATCH 03/14] disable unsupported delta architectures

---
 .github/workflows/TestCITools.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 341afaf..36fe5ac 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -20,6 +20,7 @@ jobs:
       override_repository: duckdb/extension-template
       override_ref: main
       duckdb_version: v1.0.0
+
   delta-extension-main:
     name: Build the Delta extension (to test Rust)
     uses: ./.github/workflows/_extension_distribution.yml
@@ -29,3 +30,4 @@ jobs:
       override_repository: duckdb/duckdb_delta
       override_ref: main
       duckdb_version: v1.0.0
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'

From f32bbb6c764974c2defe3ce7d756f322c7935511 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:22:50 +0200
Subject: [PATCH 04/14] add override ref for correct extension-ci-tools to use
 in extension tools CI

---
 .github/workflows/_extension_distribution.yml | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index b0045b7..fd99d3f 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -59,6 +59,11 @@ on:
         required: false
         type: string
         default: ""
+      # The ref for which to use the CI tools
+      override_ci_tools_ref:
+        required: false
+        type: string
+        default: ""
 jobs:
   generate_matrix:
     name: Generate matrix
@@ -214,6 +219,12 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
+      - name: Checkout Extension CI tools
+        if: ${{inputs.override_ci_tools_ref != ''}}
+        run: |
+          cd extension-ci-tools
+          git checkout ${{ inputs.override_ci_tools_ref }}
+
       - name: Checkout DuckDB to version
         run: |
           cd duckdb
@@ -319,6 +330,12 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Checkout Extension CI tools
+        if: ${{inputs.override_ci_tools_ref != ''}}
+        run: |
+          cd extension-ci-tools
+          git checkout ${{ inputs.override_ci_tools_ref }}
+
       - name: Checkout DuckDB to version
         run: |
           cd duckdb
@@ -407,6 +424,12 @@ jobs:
           update-rtools: true
           rtools-version: '42' # linker bug in 43
 
+      - name: Checkout Extension CI tools
+        if: ${{inputs.override_ci_tools_ref != ''}}
+        run: |
+          cd extension-ci-tools
+          git checkout ${{ inputs.override_ci_tools_ref }}
+
       - name: Checkout DuckDB to version
         run: |
           cd duckdb
@@ -466,6 +489,12 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
+      - name: Checkout Extension CI tools
+        if: ${{inputs.override_ci_tools_ref != ''}}
+        run: |
+          cd extension-ci-tools
+          git checkout ${{ inputs.override_ci_tools_ref }}
+
       - name: Checkout DuckDB to version
         run: |
           cd duckdb

From ed09a8b8bfc04f6bc0a8ef248021e1399c306ffd Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:37:47 +0200
Subject: [PATCH 05/14] rename

---
 .github/workflows/TestCITools.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 36fe5ac..28b0505 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   extension-template-main:
-    name: Build the extension template
+    name: Extension template
     uses: ./.github/workflows/_extension_distribution.yml
     with:
       extension_name: quack
@@ -22,7 +22,7 @@ jobs:
       duckdb_version: v1.0.0
 
   delta-extension-main:
-    name: Build the Delta extension (to test Rust)
+    name: Rust builds (using Delta extension)
     uses: ./.github/workflows/_extension_distribution.yml
     with:
       extension_name: delta

From 88dc811186e57aad3f6dd9dc9fab3e36e774c1cd Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:43:14 +0200
Subject: [PATCH 06/14] actually set the ci tools ref override

---
 .github/workflows/TestCITools.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 28b0505..554c7ca 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -20,6 +20,7 @@ jobs:
       override_repository: duckdb/extension-template
       override_ref: main
       duckdb_version: v1.0.0
+      override_ci_tools_ref: ${{ github.ref_name }}
 
   delta-extension-main:
     name: Rust builds (using Delta extension)
@@ -29,5 +30,6 @@ jobs:
       enable_rust: true
       override_repository: duckdb/duckdb_delta
       override_ref: main
+      override_ci_tools_ref: ${{ github.ref_name }}
       duckdb_version: v1.0.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'

From f7bbca30966fa12c214f5eef8e0df5da98356fd2 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:53:34 +0200
Subject: [PATCH 07/14] use GITHUB_SHA env var instead

---
 .github/workflows/TestCITools.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 554c7ca..0821ea7 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -20,7 +20,7 @@ jobs:
       override_repository: duckdb/extension-template
       override_ref: main
       duckdb_version: v1.0.0
-      override_ci_tools_ref: ${{ github.ref_name }}
+      override_ci_tools_ref: ${{ env.GITHUB_SHA }}
 
   delta-extension-main:
     name: Rust builds (using Delta extension)
@@ -30,6 +30,6 @@ jobs:
       enable_rust: true
       override_repository: duckdb/duckdb_delta
       override_ref: main
-      override_ci_tools_ref: ${{ github.ref_name }}
+      override_ci_tools_ref: ${{ env.GITHUB_SHA }}
       duckdb_version: v1.0.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'

From 2331528316c64ae975fffb31cb8360482f480e57 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 14:57:30 +0200
Subject: [PATCH 08/14] use github.sha var instead

---
 .github/workflows/TestCITools.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 0821ea7..6d26136 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -20,7 +20,7 @@ jobs:
       override_repository: duckdb/extension-template
       override_ref: main
       duckdb_version: v1.0.0
-      override_ci_tools_ref: ${{ env.GITHUB_SHA }}
+      override_ci_tools_ref: ${{ github.sha }}
 
   delta-extension-main:
     name: Rust builds (using Delta extension)
@@ -30,6 +30,6 @@ jobs:
       enable_rust: true
       override_repository: duckdb/duckdb_delta
       override_ref: main
-      override_ci_tools_ref: ${{ env.GITHUB_SHA }}
+      override_ci_tools_ref: ${{ github.sha }}
       duckdb_version: v1.0.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'

From 4d1e5697bdcde5a6ff8f3f613877c766acb8ef47 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 15:43:30 +0200
Subject: [PATCH 09/14] try using actions/checkout

---
 .github/workflows/TestCITools.yml             |  6 ++-
 .github/workflows/_extension_distribution.yml | 52 +++++++++++++------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 6d26136..7041c23 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -20,7 +20,8 @@ jobs:
       override_repository: duckdb/extension-template
       override_ref: main
       duckdb_version: v1.0.0
-      override_ci_tools_ref: ${{ github.sha }}
+      override_ci_tools_repository: ${{ github.repository }}
+      override_ci_tools_ref: ${{ github.event.action == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
 
   delta-extension-main:
     name: Rust builds (using Delta extension)
@@ -30,6 +31,7 @@ jobs:
       enable_rust: true
       override_repository: duckdb/duckdb_delta
       override_ref: main
-      override_ci_tools_ref: ${{ github.sha }}
+      override_ci_tools_repository: ${{ github.repository }}
+      override_ci_tools_ref: ${{github.event.action == 'pull_request' && github.event.pull_request.head.sha || github.sha}}
       duckdb_version: v1.0.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index fd99d3f..ab5ed67 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -59,11 +59,17 @@ on:
         required: false
         type: string
         default: ""
-      # The ref for which to use the CI tools
+      # Override the repo for the CI tools (for testing CI tools itself)
+      override_ci_tools_repository:
+        required: false
+        type: string
+        default: ""
+      # Override the ref for the CI tools (for testing CI tools itself)
       override_ci_tools_ref:
         required: false
         type: string
         default: ""
+
 jobs:
   generate_matrix:
     name: Generate matrix
@@ -219,11 +225,14 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - name: Checkout Extension CI tools
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
-        run: |
-          cd extension-ci-tools
-          git checkout ${{ inputs.override_ci_tools_ref }}
+        with:
+          path: 'extension-ci-tools'
+          ref: ${{ inputs.override_ci_tools_ref }}
+          repository: ${{ inputs.override_ci_tools_repository }}
+          fetch-depth: 0
 
       - name: Checkout DuckDB to version
         run: |
@@ -330,11 +339,14 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Checkout Extension CI tools
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
-        run: |
-          cd extension-ci-tools
-          git checkout ${{ inputs.override_ci_tools_ref }}
+        with:
+          path: 'extension-ci-tools'
+          ref: ${{ inputs.override_ci_tools_ref }}
+          repository: ${{ inputs.override_ci_tools_repository }}
+          fetch-depth: 0
 
       - name: Checkout DuckDB to version
         run: |
@@ -424,11 +436,14 @@ jobs:
           update-rtools: true
           rtools-version: '42' # linker bug in 43
 
-      - name: Checkout Extension CI tools
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
-        run: |
-          cd extension-ci-tools
-          git checkout ${{ inputs.override_ci_tools_ref }}
+        with:
+          path: 'extension-ci-tools'
+          ref: ${{ inputs.override_ci_tools_ref }}
+          repository: ${{ inputs.override_ci_tools_repository }}
+          fetch-depth: 0
 
       - name: Checkout DuckDB to version
         run: |
@@ -489,11 +504,14 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - name: Checkout Extension CI tools
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
-        run: |
-          cd extension-ci-tools
-          git checkout ${{ inputs.override_ci_tools_ref }}
+        with:
+          path: 'extension-ci-tools'
+          ref: ${{ inputs.override_ci_tools_ref }}
+          repository: ${{ inputs.override_ci_tools_repository }}
+          fetch-depth: 0
 
       - name: Checkout DuckDB to version
         run: |

From f2ff03faf93b2f5dd5b441174e2f56ed19aa3439 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 16:50:36 +0200
Subject: [PATCH 10/14] use correct checkout version

---
 .github/workflows/TestCITools.yml             | 4 ++--
 .github/workflows/_extension_distribution.yml | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml
index 7041c23..5a3c88d 100644
--- a/.github/workflows/TestCITools.yml
+++ b/.github/workflows/TestCITools.yml
@@ -21,7 +21,7 @@ jobs:
       override_ref: main
       duckdb_version: v1.0.0
       override_ci_tools_repository: ${{ github.repository }}
-      override_ci_tools_ref: ${{ github.event.action == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      override_ci_tools_ref: ${{ github.sha }}
 
   delta-extension-main:
     name: Rust builds (using Delta extension)
@@ -32,6 +32,6 @@ jobs:
       override_repository: duckdb/duckdb_delta
       override_ref: main
       override_ci_tools_repository: ${{ github.repository }}
-      override_ci_tools_ref: ${{github.event.action == 'pull_request' && github.event.pull_request.head.sha || github.sha}}
+      override_ci_tools_ref: ${{ github.sha }}
       duckdb_version: v1.0.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index ab5ed67..4c45ffd 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -225,7 +225,7 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
         with:
@@ -339,7 +339,7 @@ jobs:
         with:
           python-version: '3.11'
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
         with:
@@ -436,7 +436,7 @@ jobs:
           update-rtools: true
           rtools-version: '42' # linker bug in 43
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
         with:
@@ -504,7 +504,7 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
         with:

From 4d9db59ed80cff19c4825c514dc67e58ff0c4d1d Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 16:57:16 +0200
Subject: [PATCH 11/14] remove broken apt repo install

---
 .github/workflows/_extension_distribution.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index 4c45ffd..6f4888d 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -154,8 +154,6 @@ jobs:
         run: |
           apt-get update -y -qq
           apt-get install -y -qq software-properties-common
-          add-apt-repository ppa:git-core/ppa
-          apt-get update -y -qq
           apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
 
       - name: Install Git 2.18.5

From f0a35d47b3323c4e76c1d34de7790e7872e68229 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 17:12:34 +0200
Subject: [PATCH 12/14] remove ubuntu 18 setup action

---
 .github/workflows/_extension_distribution.yml | 31 ++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index 6f4888d..6ace087 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -201,11 +201,34 @@ jobs:
         run: |
           ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
 
-      - name: Setup Ubuntu
+      - name: Install cross compiler for linux_arm64
+        shell: bash
+        if: ${{ matrix.duckdb_arch == 'linux_arm64' }}
+        run: |
+          apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+      - name: Install CMake 3.21
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
-        uses: ./duckdb/.github/actions/ubuntu_18_setup
-        with:
-          aarch64_cross_compile: ${{ matrix.duckdb_arch == 'linux_arm64' && 1 }}
+        shell: bash
+        run: |
+          wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh
+          chmod +x cmake-3.21.3-linux-x86_64.sh
+          ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local
+          cmake --version
+
+      # TODO: do we even need this?
+      - name: Install Python 3.8
+        shell: bash
+        if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
+        run: |
+          wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz
+          tar xvf Python-3.8.17.tgz
+          cd Python-3.8.17
+          mkdir -p pythonbin
+          ./configure --with-ensurepip=install
+          make -j
+          make install
+          python3.8 --version
 
       - uses: actions/checkout@v3
         name: Checkout override repository

From baee499e98b62c95d572e8d58c683d29af7b87c9 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 17:25:37 +0200
Subject: [PATCH 13/14] add pkg-config

---
 .github/workflows/_extension_distribution.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index 6ace087..529ce69 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -154,7 +154,7 @@ jobs:
         run: |
           apt-get update -y -qq
           apt-get install -y -qq software-properties-common
-          apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client
+          apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config
 
       - name: Install Git 2.18.5
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}

From 16f47bcbb35a228be7537b438d219ca695834383 Mon Sep 17 00:00:00 2001
From: Sam Ansmink <samansmink@hotmail.com>
Date: Thu, 11 Jul 2024 17:56:37 +0200
Subject: [PATCH 14/14] small refactor of linux extension CI

---
 .github/workflows/_extension_distribution.yml | 110 +++++++-----------
 1 file changed, 39 insertions(+), 71 deletions(-)

diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml
index 529ce69..30b508e 100644
--- a/.github/workflows/_extension_distribution.yml
+++ b/.github/workflows/_extension_distribution.yml
@@ -149,6 +149,9 @@ jobs:
       ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
+      ###
+      # Setup environment (TODO: this should ideally be part of a container image)
+      ###
       - name: Install required ubuntu packages
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
         run: |
@@ -156,6 +159,12 @@ jobs:
           apt-get install -y -qq software-properties-common
           apt-get install -y -qq --fix-missing ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config
 
+      - name: Install cross compiler for linux_arm64
+        shell: bash
+        if: ${{ matrix.duckdb_arch == 'linux_arm64' }}
+        run: |
+          apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
       - name: Install Git 2.18.5
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
         run: |
@@ -166,46 +175,21 @@ jobs:
           make prefix=/usr install
           git --version
 
-      - uses: actions/checkout@v3
-        name: Checkout override repository
-        if: ${{inputs.override_repository != ''}}
-        with:
-          repository: ${{ inputs.override_repository }}
-          ref: ${{ inputs.override_ref }}
-          fetch-depth: 0
-          submodules: 'true'
+      - name: Setup Rust
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64'}}
+        uses: dtolnay/rust-toolchain@stable
 
-      - uses: actions/checkout@v3
-        name: Checkout current repository
-        if: ${{inputs.override_repository == ''}}
+      - name: Setup Rust for cross compilation
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_arm64'}}
+        uses: dtolnay/rust-toolchain@stable
         with:
-          fetch-depth: 0
-          submodules: 'true'
-
-      - name: Checkout DuckDB to version
-        run: |
-          cd duckdb
-          git checkout ${{ inputs.duckdb_version }}
-
-      - name: Fix CentOS 7 EOL
-        if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
-        run: |
-          # Workaround the fact that CentOS 7 is EOL
-          sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-          sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-          yum-config-manager --disable centos-sclo-rh
-          yum install epel-release -y
-
-      - name: Setup ManyLinux2014
-        if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
-        run: |
-          ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
+          targets: aarch64-unknown-linux-gnu
 
-      - name: Install cross compiler for linux_arm64
-        shell: bash
-        if: ${{ matrix.duckdb_arch == 'linux_arm64' }}
+      - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
+        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64_gcc4' }}
         run: |
-          apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
 
       - name: Install CMake 3.21
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
@@ -216,20 +200,9 @@ jobs:
           ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local
           cmake --version
 
-      # TODO: do we even need this?
-      - name: Install Python 3.8
-        shell: bash
-        if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
-        run: |
-          wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz
-          tar xvf Python-3.8.17.tgz
-          cd Python-3.8.17
-          mkdir -p pythonbin
-          ./configure --with-ensurepip=install
-          make -j
-          make install
-          python3.8 --version
-
+      ###
+      # Checkin out repositories
+      ###
       - uses: actions/checkout@v3
         name: Checkout override repository
         if: ${{inputs.override_repository != ''}}
@@ -246,6 +219,16 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
+      - name: Checkout DuckDB to version
+        run: |
+          cd duckdb
+          git checkout ${{ inputs.duckdb_version }}
+
+      - name: Setup ManyLinux2014
+        if: ${{ matrix.duckdb_arch == 'linux_amd64_gcc4' }}
+        run: |
+          ./duckdb/scripts/setup_manylinux2014.sh general ccache ssh python_alias openssl
+
       - uses: actions/checkout@v3
         name: Checkout Extension CI tools
         if: ${{inputs.override_ci_tools_ref != ''}}
@@ -255,27 +238,9 @@ jobs:
           repository: ${{ inputs.override_ci_tools_repository }}
           fetch-depth: 0
 
-      - name: Checkout DuckDB to version
-        run: |
-          cd duckdb
-          git checkout ${{ inputs.duckdb_version }}
-
-      - name: Setup Rust
-        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64'}}
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: Setup Rust for cross compilation
-        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_arm64'}}
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          targets: aarch64-unknown-linux-gnu
-
-      - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
-        if: ${{ inputs.enable_rust && matrix.duckdb_arch == 'linux_amd64_gcc4' }}
-        run: |
-          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y 
-          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-
+      ###
+      # Runtime configuration before build
+      ###
       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
         continue-on-error: true
@@ -294,6 +259,9 @@ jobs:
           echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV
           echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
 
+      ###
+      # Building & testing
+      ###
       - name: Build extension
         env:
           GEN: ninja