Rely on extension-ci-tools workflow to build linux_amd64_gcc4 extensions

Idea is instead of executing whole workflow job containerized, that means that actions will
be run within the container, that in turn means it's hard to  use older images since they will
not have the required dependecies versions for the actions (in particular node 20).

Solution is to containerize explicitly the relevant tests in the workflow, invoking actions
outside of the container in the regular image that powers the workflow.

Heavy lifting has been done by @samansmink in duckdb/extension-ci-tools#79
and connected PRs.

This PR allows to avoid the need for ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION in Python CI.

.github/workflows/Python.yml

@@ -119,46 +119,91 @@ jobs:
    manylinux-extensions-x64:
     name: Linux Extensions (linux_amd64_gcc4)
     runs-on: ubuntu-latest
-    container: quay.io/pypa/manylinux2014_x86_64
     needs: linux-python3-9
     env:
       GEN: ninja
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        ref: ${{ inputs.git_ref }}
+      - uses: actions/checkout@v4
+        with:
+          path: 'duckdb'
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
 
-    - uses: ./.github/actions/manylinux_2014_setup
-      with:
-        aws-cli: 1
-        ninja-build: 1
-        ccache: 1
-        nodejs: 1
-        ssh: 1
-        python_alias: 1
-        openssl: 1
-
-    - uses: ./.github/actions/build_extensions
-      with:
-        vcpkg_target_triplet: x64-linux
-        post_install: rm build/release/src/libduckdb*
-        deploy_as: linux_amd64_gcc4
-        s3_id: ${{ secrets.S3_ID }}
-        s3_key: ${{ secrets.S3_KEY }}
-        signing_pk: ${{ secrets.DUCKDB_EXTENSION_SIGNING_PK }}
-        run_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
-        run_autoload_tests: ${{ inputs.skip_tests != 'true' && 1 || 0 }}
-        treat_warn_as_error: 0
-        ninja: 1
-
-    - uses: actions/upload-artifact@v3
-      with:
-        name: manylinux-extensions-x64
-        path: |
-          build/release/extension/*/*.duckdb_extension
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
+        with:
+          path: 'extension-ci-tools'
+          ref: 'main'
+          repository: 'duckdb/extension-ci-tools'
+          fetch-depth: 0
+
+      - name: Populate composed extension
+        run: |
+          cat duckdb/.github/config/in_tree_extensions.cmake duckdb/.github/config/out_of_tree_extensions.cmake > extension_config.cmake
+          echo "{\"dependencies\": []}" > vcpkg.json
+          cp duckdb/makefiles/build_extensions Makefile
+
+      - name: Build Docker image
+        shell: bash
+        run: |
+          docker build \
+            --build-arg 'vcpkg_url=https://github.com/microsoft/vcpkg.git' \
+            --build-arg 'vcpkg_commit=a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6' \
+            --build-arg 'extra_toolchains=;python3;' \
+            -t duckdb/linux_amd64_gcc4 \
+            ./extension-ci-tools/docker/linux_amd64_gcc4
+
+      - name: Create env file for docker
+        run: |
+          touch docker_env.txt
+          echo "VCPKG_TARGET_TRIPLET=x64-linux" >> docker_env.txt 
+          echo "BUILD_SHELL=1" >> docker_env.txt
+          echo "OPENSSL_ROOT_DIR=/duckdb_build_dir/build/release/vcpkg_installed/x64-linux" >> docker_env.txt
+          echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/x64-linux" >> docker_env.txt
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt
+          echo "DUCKDB_PLATFORM=linux_amd64_gcc4" >> docker_env.txt
+          echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt
+          echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt
+          echo "TOOLCHAIN_FLAGS=''" >> docker_env.txt
+
+      - name: Generate timestamp for Ccache entry
+        shell: cmake -P {0}
+        id: ccache_timestamp
+        run: |
+          string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+          message("::set-output name=timestamp::${current_date}")
+
+      - name: Create Ccache directory
+        run: |
+          mkdir ccache_dir
+
+      - name: Load Ccache
+        uses: actions/cache@v4
+        with:
+          path: ./ccache_dir
+          key: ccache-extension-distribution-${{ matrix.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }}
+          restore-keys: |
+            ccache-extension-distribution-${{ matrix.duckdb_arch }}-
+
+      - name: Run configure (inside Docker)
+        shell: bash
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/linux_amd64_gcc4 make configure_ci
+
+      - name: Build extension (inside Docker)
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/linux_amd64_gcc4 make release
+
+      - name: Test extension (inside docker)
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/linux_amd64_gcc4 make test_release
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: manylinux-extensions-x64
+          path: |
+            build/release/extension/*/*.duckdb_extension
 
    linux-python3:
     name: Python 3 Linux

makefiles/build_extensions

@@ -0,0 +1,14 @@
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+# Configuration of extension
+EXT_NAME=httpfs
+EXT_CONFIG=${PROJ_DIR}extension_config.cmake
+
+# Include the Makefile from extension-ci-tools
+include extension-ci-tools/makefiles/duckdb_extension.Makefile
+
+configure_ci:
+	cd duckdb && BUILD_ALL_EXT=1 make extension_configuration && cp build/extension_configuration/vcpkg.json ../.
+
+test_release:
+	python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest