diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index df95048..a10eb03 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -92,7 +92,7 @@ jobs:
         run: |
           mkdir ./toolchain
           # TODO: This should use main instead before landing.
-          docker run --rm ghcr.io/opencompl/quidditch/toolchain:${{ github.ref_name }} tar -cC /opt/quidditch-toolchain . \
+          docker run --rm ghcr.io/opencompl/quidditch/toolchain:${{ github.head_ref || github.ref_name }} tar -cC /opt/quidditch-toolchain . \
           | tar -xC ./toolchain
 
       - name: Set up Python
diff --git a/.github/workflows/toolchain.yaml b/.github/workflows/toolchain.yaml
index d7e56b0..652f48f 100644
--- a/.github/workflows/toolchain.yaml
+++ b/.github/workflows/toolchain.yaml
@@ -26,7 +26,7 @@ jobs:
       # Actually build the Docker container
       - uses: actions/checkout@v4
 
-      - uses: docker/setup-buildx-action@v1
+      - uses: docker/setup-buildx-action@v3
 
       - name: GHCR Log-in
         uses: docker/login-action@v3.1.0
diff --git a/runtime/toolchain/Dockerfile b/runtime/toolchain/Dockerfile
index 3f08b61..d38605d 100644
--- a/runtime/toolchain/Dockerfile
+++ b/runtime/toolchain/Dockerfile
@@ -11,8 +11,9 @@ ARG PICOLIBC_VERSION=1.8.6
 ARG SNITCH_CLUSTER_SHA=e02cc9e3f24b92d4607455d5345caba3eb6273b2
 ARG VERILATOR_RELEASE=v4.228
 ARG BENDER_RELEASE=0.27.1
+ARG PRESERVE_SOURCE=false
 
-ENV INSTALL_DIR=/opt/
+ENV INSTALL_DIR=/opt
 ENV CCACHE_DIR=/ccache
 
 WORKDIR $INSTALL_DIR
@@ -65,7 +66,7 @@ RUN cd llvm-project/llvm/build && rm -rf * && \
          -DLLVM_BUILD_STATIC=ON  \
          -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
          -C  /root/phase2.cmake && \
-        ninja install
+        ninja install/strip
 RUN rm -rf llvm-project/llvm/build
 
 RUN cargo install bender --locked --version $BENDER_RELEASE
@@ -87,6 +88,13 @@ RUN cd ./snitch_cluster/target/snitch_cluster && \
     make bin/snitch_cluster.vlt -j$(nproc)
 
 RUN cp ./snitch_cluster/target/snitch_cluster/bin/snitch_cluster.vlt $INSTALL_DIR/quidditch-toolchain/bin/
+RUN strip $INSTALL_DIR/quidditch-toolchain/bin/snitch_cluster.vlt
 RUN rm -rf ./snitch_cluster/target/snitch_cluster/bin  \
     ./snitch_cluster/target/snitch_cluster/generated  \
-    ./snitch_cluster/target/snitch_cluster/work-vlt
+    ./snitch_cluster/target/snitch_cluster/work-vlt \
+    /ccache
+
+RUN $PRESERVE_SOURCE || rm -rf $INSTALL_DIR/verilator  \
+    $INSTALL_DIR/llvm-project  \
+    $INSTALL_DIR/snitch-cluster \
+    $INSTALL_DIR/picolibc