Improve usage of ccache to speed up builds

build

@@ -103,6 +103,8 @@ SIGNDUMMY=
 DO_STATISTICS=
 RUN_SHELL=
 CCACHE=
+#DO_CCACHE_STATISTICS=
+#DO_CCACHE_LOGGING=
 DLNOSIGNATURE=
 BUILD_FLAVOR=
 OBS_PACKAGE=
@@ -244,6 +246,26 @@ Known Parameters:
   --ccache
               Use ccache to speed up rebuilds
 
+  --ccache-host-dir DIRNAME
+              Mount DIRNAME from host to share and persist the cache
+              data to speed up rebuilds (may be backed by NFS+autofs)
+
+  --ccache-host-bin PATH_TO_BINARY
+              If the build root environment packaging does not provide
+              a /usr/bin/ccache, copy the binary from the worker host OS
+              into the build root, to speed up rebuilds. Test this first!
+              Usually the runtime dependencies are minimal, so the same
+              binary should work on any Linux distro from the same era.
+              If PATH_TO_BINARY is "yes", copy host /usr/bin/ccache.
+
+  --ccache-logging
+              optionally enable saving a ccache log of operations done
+              during this build job, beware of overheads and slowdown.
+
+  --ccache-statistics
+              optionally report ccache statistics and log of operations
+              done during this build job, in the end of usual build log.
+
   --icecream N
               Use N parallel build jobs with icecream
 
@@ -457,26 +479,127 @@ toshellscript() {
     echo
 }
 
+provide_ccache() {
+    # If called, ensure a "/.ccache-bin/ccache" is present in the BUILD_ROOT.
+    # It may be a link to "/usr/bin/ccache" installed via packaging, or as a
+    # fallback only, the worker host's copy can be used, from default location
+    # (if CCACHE_HOST_BIN==yes), or from absolute-path location pointed to by
+    # the CCACHE_HOST_BIN value. This routine is called from setupccache(),
+    # and only if CCACHE support is enabled for this build.
+
+    # Fasttrack if already set up
+    [ -x "$BUILD_ROOT/.ccache-bin/ccache" ] && return 0
+
+    # Setup is indeed needed... do some sanity check first
+    [ -n "$BUILD_ROOT" ] && [ -d "$BUILD_ROOT" ] || return
+    mkdir -p $BUILD_ROOT/.ccache-bin
+    rm -f "$BUILD_ROOT/.ccache-bin/ccache" 2>/dev/null || true
+
+    # First try using a binary that may already exist in the build root via
+    # packaging as /usr/bin/ccache. Note that this path can be an absolute
+    # symlink, so can be misleading in the host environment - so we must
+    # chroot before evaluating it.
+    if chroot "$BUILD_ROOT" /usr/bin/test -x /usr/bin/ccache 2>/dev/null || \
+       chroot "$BUILD_ROOT" /bin/test -x /usr/bin/ccache 2>/dev/null ; then
+        echo "Using preferred ccache binary provided in the build root"
+        ln -s "../usr/bin/ccache" "$BUILD_ROOT/.ccache-bin/ccache"
+    else
+        case "$CCACHE_HOST_BIN" in
+            yes|true|on) CCACHE_HOST_BIN="/usr/bin/ccache" ;;
+            /*) ;;
+            *|"") return 1 ;;
+        esac
+
+        if [ -n "$CCACHE_HOST_BIN" ] && [ -x "$CCACHE_HOST_BIN" ] ; then
+            echo "Using fallback ccache binary provided by the build worker: $CCACHE_HOST_BIN"
+            cp -pLf "$CCACHE_HOST_BIN" "$BUILD_ROOT/.ccache-bin/ccache"
+        fi
+    fi
+
+    test -s "$BUILD_ROOT/.ccache-bin/ccache" && test -x "$BUILD_ROOT/.ccache-bin/ccache"
+    # Return exit-code of this test
+}
+
+undo_setupccache() {
+    rm -f "$BUILD_ROOT"/.ccache-bin/{*-,}{gcc,g++,cc,c++,clang,clang++}{,-*}
+    rm -f "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+    rm -f "$BUILD_ROOT"/.build.log.ccache
+}
+
 setupccache() {
-    if test -n "$CCACHE" ; then
-	if mkdir -p $BUILD_ROOT/var/lib/build/ccache/bin; then
-	    for i in $(ls $BUILD_ROOT/usr/bin | grep -E '^(cc|gcc|[cg][+][+]|clang|clang[+][+])([-]?[234][.]?[0-9])*$'); do
-		rm -f $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		test -e $BUILD_ROOT/usr/bin/$i || continue
-		echo '#! /bin/sh' > $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		echo "test -e /usr/bin/$i || exit 1" >> $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		echo 'export PATH=/usr/lib/icecc/bin:/opt/icecream/bin:/usr/bin:$PATH' >> $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		echo "ccache $i \"\$@\"" >> $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		chmod 755 $BUILD_ROOT/var/lib/build/ccache/bin/$i
-		echo "Installed ccache wrapper as $BUILD_ROOT/var/lib/build/ccache/bin/$i"
-	    done
-	fi
-	mkdir -p "$BUILD_ROOT/.ccache"
-	chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache"
-	echo "export CCACHE_DIR=/.ccache" > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
-	echo 'export PATH=/var/lib/build/ccache/bin:$PATH' >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+    # This routine sets up CCACHE wrappers for build roots, if enabled.
+    # This involves a /.ccache-bin/ directory with ccache itself and wrapper
+    # scripts for the compilers found on the build root, and a /.ccache/
+    # directory with the cache that can be private (useless unless you
+    # rebuild in same workers without wiping), or mounted from the host OS
+    # (including a bind-mount of an autofs+NFS mounted directory), so that
+    # the same cache directory can be shared between OBS workers.
+    local WRAPPED_CCACHE=0
+    case "$BUILDTYPE" in
+        *image*) undo_setupccache; return 0 ;;
+    esac
+
+    if test -n "$CCACHE" || test -n "$CCACHE_HOST_DIR" ; then
+        if mkdir -p $BUILD_ROOT/.ccache-bin; then
+            for i in $(ls $BUILD_ROOT/usr/bin | grep -E '^(.+\-)?(cc|gcc|[cg][+][+]|clang|clang[+][+])([-]?[234][.]?[0-9])*$'); do
+                rm -f $BUILD_ROOT/.ccache-bin/$i
+                # Note: a mediated absolute symlink may be used in the
+                # build-root, so the verbatim $BUILD_ROOT/usr/bin/$i
+                # points nowhere valid in the host environment...
+                chroot $BUILD_ROOT /usr/bin/test -e /usr/bin/$i || chroot $BUILD_ROOT /bin/test -e /usr/bin/$i || continue
+                provide_ccache || break
+                WRAPPED_CCACHE=1
+                echo '#! /bin/sh' > $BUILD_ROOT/.ccache-bin/$i
+                echo "test -e /usr/bin/$i || exit 1" >> $BUILD_ROOT/.ccache-bin/$i
+                echo 'export PATH=/usr/lib/icecc/bin:/opt/icecream/bin:/usr/bin:$PATH' >> $BUILD_ROOT/.ccache-bin/$i
+                echo "exec /.ccache-bin/ccache $i \"\$@\"" >> $BUILD_ROOT/.ccache-bin/$i
+                chmod 755 $BUILD_ROOT/.ccache-bin/$i
+                echo "Installed ccache wrapper as $BUILD_ROOT/.ccache-bin/$i"
+            done
+        fi
+    fi
+
+    rm -f "$BUILD_ROOT/.build.log.ccache" || true
+    if test "$WRAPPED_CCACHE" = 1 ; then
+        mkdir -p "$BUILD_ROOT/.ccache"
+        if test -n "$CCACHE_HOST_DIR" ; then
+            if ! test -d "$CCACHE_HOST_DIR" ; then
+                echo "Creating new CCACHE_HOST_DIR='$CCACHE_HOST_DIR' owned by '$ABUILD_UID:$ABUILD_GID' on the host system"
+                mkdir -p "$CCACHE_HOST_DIR" && \
+                chown -R "$ABUILD_UID:$ABUILD_GID" "$CCACHE_HOST_DIR"
+            fi
+            echo "Using host's CCACHE_HOST_DIR='$CCACHE_HOST_DIR' as /.ccache inside BUILD_ROOT"
+            mount -o rbind "$CCACHE_HOST_DIR" "$BUILD_ROOT/.ccache"
+        else
+            chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache"
+        fi
+
+        cat << EOF > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+echo 'Sourcing /etc/profile.d/build_ccache.sh...' >&2
+CCACHE_DIR="/.ccache"
+PATH="/.ccache-bin:\$PATH"
+export PATH CCACHE_DIR
+if test x"\$PROFILE_CCACHE_STATS" = xyes ; then
+    echo "CCache stats before build:"
+    CCACHE_LOGFILE='' ccache -s
+fi >&2
+true
+EOF
+
+        # Note: logging does incur considerable overhead and slowdown
+        # so do not enable it unless investigating your ccache setup.
+        if [ "$DO_CCACHE_LOGGING" = 1 ] ; then
+            touch "$BUILD_ROOT/.build.log.ccache"
+            chown "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.build.log.ccache"
+            chmod 664 "$BUILD_ROOT/.build.log.ccache"
+            echo 'CCACHE_LOGFILE="/.build.log.ccache" ; export CCACHE_LOGFILE' >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+        fi
+
+        if [ "$DO_CCACHE_STATISTICS" = 1 ] ; then
+            chroot "$BUILD_ROOT" /bin/sh -c "PROFILE_CCACHE_STATS=yes ; . /etc/profile.d/build_ccache.sh"
+        fi
     else
-	rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/{gcc,g++,cc,c++,clang,clang++}
+        undo_setupccache
     fi
 }
 
@@ -917,6 +1040,24 @@ while test -n "$1"; do
       -ccache)
 	CCACHE=true
       ;;
+      -ccache-host-dir)
+	needarg
+	CCACHE=true
+	CCACHE_HOST_DIR="$ARG"
+	shift
+      ;;
+      -ccache-host-bin)
+	needarg
+	CCACHE=true
+	CCACHE_HOST_BIN="$ARG"
+	shift
+      ;;
+      -ccache-logging)
+	DO_CCACHE_LOGGING=1
+      ;;
+      -ccache-statistics)
+	DO_CCACHE_STATISTICS=1
+      ;;
       -statistics)
 	DO_STATISTICS=1
       ;;
@@ -1023,6 +1164,12 @@ test -n "$BUILD_RPM_BUILD_STAGE" && validate_param "--stage" "$BUILD_RPM_BUILD_S
 test "$BUILD_ROOT" != /var/tmp/build-root && validate_param "--root" "$BUILD_ROOT" BUILD_ROOT
 test "$CONFIG_DIR" != "$BUILD_DIR/configs" && validate_param "--configdir" "$CONFIG_DIR" CONFIG_DIR
 
+# Beside CLI, these can come from envvars (sourced by worker configs)
+# Note the env can also enforce DO_CCACHE_STATISTICS=1 to post the logs
+if test -n "$CCACHE_HOST_DIR" || test -n "$CCACHE_HOST_BIN" ; then
+    CCACHE=true
+fi
+
 # validate the buildroot
 validate_buildroot "$BUILD_ROOT"
 
@@ -1492,4 +1639,24 @@ echo
 echo "$HOST finished \"build $RECIPEFILE\" at `date --utc`."
 echo
 
+if [ "$DO_CCACHE_STATISTICS" = 1 ] ; then
+    if [ -s "$BUILD_ROOT/.build.log.ccache" ] ; then
+        echo "Dumping current run's CCACHE log:"
+        echo "======="
+        cat "$BUILD_ROOT/.build.log.ccache"
+        echo "======="
+    fi
+
+    if [ -x "$BUILD_ROOT"/.ccache-bin/ccache ] && [ -s "$BUILD_ROOT"/etc/profile.d/build_ccache.sh ] ; then
+        echo
+        echo "CCACHE stats after this run:"
+        echo
+        chroot "$BUILD_ROOT" /bin/sh -c "PROFILE_CCACHE_STATS=no ; . /etc/profile.d/build_ccache.sh && CCACHE_LOGFILE='' /.ccache-bin/ccache -s"
+    fi
+
+    echo
+    echo "$HOST finished \"build $RECIPEFILE\" augmented by CCACHE at `date --utc`."
+    echo
+fi
+
 cleanup_and_exit "$exitcode"

build-recipe-dsc

@@ -41,6 +41,10 @@ recipe_prepare_dsc() {
     rm -rf "$BUILD_ROOT$TOPDIR/BUILD"
     mkdir -p "$BUILD_ROOT$TOPDIR/SOURCES.DEB"
     chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT$TOPDIR"
+    if [ -s "$BUILD_ROOT"/etc/profile.d/build_ccache.sh ]; then
+	grep CCACHE_BASEDIR "$BUILD_ROOT"/etc/profile.d/build_ccache.sh > /dev/null || \
+	echo "CCACHE_BASEDIR='$TOPDIR/BUILD' ; export CCACHE_BASEDIR" >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+    fi
     DEB_TRANSFORM=
     DEB_SOURCEDIR="$TOPDIR/SOURCES"
     DEB_DSCFILE="$RECIPEFILE"

build-recipe-preinstallimage

@@ -49,6 +49,7 @@ recipe_build_preinstallimage() {
 	.srcfiles*) continue ;;
 	.pkgs) continue ;;
 	.rpm-cache) continue ;;
+	.ccache*) continue ;;
 	installed-pkg) continue ;;
 	proc|sys) continue ;;
       esac

build-recipe-simpleimage

@@ -72,6 +72,7 @@ recipe_build_simpleimage() {
 	.pkgs) continue ;;
 	.rpm-cache) continue ;;
 	.tmp) continue ;;
+	.ccache*) continue ;;
 	installed-pkg) continue ;;
 	proc|sys) continue ;;
       esac

build-recipe-spec

@@ -157,11 +157,23 @@ recipe_build_spec() {
 
     # su involves a shell which would require even more
     # complicated quoting to bypass than this
+    rm -f $BUILD_ROOT/.build.command
+    touch $BUILD_ROOT/.build.command
+    PKG_BUILDDIR="$(chroot $BUILD_ROOT su -c 'rpmspec -q --srpm --qf="`rpmspec -q --srpm --eval="'"$TOPDIR/BUILD"'/%{name}-%{version}" '"$TOPDIR/SOURCES/$RECIPEFILE"'`" '"$TOPDIR/SOURCES/$RECIPEFILE" - $BUILD_USER < /dev/null | head -1)" || PKG_BUILDDIR=""
+    if [ -n "$PKG_BUILDDIR" ]; then
+        if [ -s "$BUILD_ROOT"/etc/profile.d/build_ccache.sh ]; then
+            grep CCACHE_BASEDIR "$BUILD_ROOT"/etc/profile.d/build_ccache.sh > /dev/null || \
+            echo "CCACHE_BASEDIR='$PKG_BUILDDIR' ; export CCACHE_BASEDIR" >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+            echo "=== Amended CCACHE config for SPEC build root:"
+            cat "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+#            echo ". /etc/profile.d/build_ccache.sh ;" >> $BUILD_ROOT/.build.command
+        fi
+    fi
     toshellscript $rpmbuild \
 	    "${definesnstuff[@]}" \
 	    "${rpmbopts[@]}" \
 	    "$TOPDIR/SOURCES/$RECIPEFILE" \
-	    > $BUILD_ROOT/.build.command
+	    >> $BUILD_ROOT/.build.command
     chmod 755 $BUILD_ROOT/.build.command
     check_exit
     if test -n "$RUN_SHELL"; then

build-vm

@@ -845,6 +845,8 @@ vm_first_stage() {
     echo "VM_WATCHDOG='$VM_WATCHDOG'" >> $BUILD_ROOT/.build/build.data
     echo "BUILDENGINE='$BUILDENGINE'" >> $BUILD_ROOT/.build/build.data
     echo "CCACHE='$CCACHE'" >> $BUILD_ROOT/.build/build.data
+    echo "CCACHE_HOST_DIR='$CCACHE_HOST_DIR'" >> $BUILD_ROOT/.build/build.data
+    echo "CCACHE_HOST_BIN='$CCACHE_HOST_BIN'" >> $BUILD_ROOT/.build/build.data
     echo "ABUILD_TARGET='$ABUILD_TARGET'" >> $BUILD_ROOT/.build/build.data
     echo "BUILD_FLAVOR='$BUILD_FLAVOR'" >> $BUILD_ROOT/.build/build.data
     echo "OBS_PACKAGE='$OBS_PACKAGE'" >> $BUILD_ROOT/.build/build.data

configs/debian.conf

@@ -9,7 +9,7 @@ Runscripts: base-files initscripts
 
 VMinstall: util-linux binutils libblkid1 libuuid1 libdevmapper1.02 mount
 
-Required: autoconf automake binutils bzip2 gcc gettext libc6
+Required: autoconf automake binutils bzip2 gcc gettext libc6 ccache
 Required: libtool libncurses5 perl zlib1g dpkg
 
 Support: build-essential fakeroot
@@ -21,7 +21,7 @@ Support: net-tools util-linux
 Support: patch procps psmisc rcs strace
 Support: texinfo unzip vim ncurses-base sysvinit
 
-Keep: binutils cpp cracklib file findutils gawk gcc gcc-ada gcc-c++
+Keep: binutils cpp cracklib file findutils gawk gcc gcc-ada gcc-c++ ccache
 Keep: gzip libada libstdc++ libunwind
 Keep: libunwind-devel libzio make mktemp pam-devel pam-modules
 Keep: patch perl rcs timezone

init_buildsystem

@@ -156,6 +156,7 @@ cleanup_and_exit() {
 	umount -n "$BUILD_ROOT/proc" 2> /dev/null || true
 	umount -n "$BUILD_ROOT/dev/pts" 2> /dev/null || true
 	umount -n "$BUILD_ROOT/mnt" 2> /dev/null || true
+	umount -n "$BUILD_ROOT/.ccache" 2> /dev/null || true
     fi
     exit $1
 }
@@ -167,6 +168,7 @@ clean_build_root() {
 	umount -n "$BUILD_ROOT/dev/pts" 2> /dev/null || true
 	umount -n "$BUILD_ROOT/dev/shm" 2> /dev/null || true
 	umount -n "$BUILD_ROOT/mnt" 2> /dev/null || true
+	umount -n "$BUILD_ROOT/.ccache" 2> /dev/null || true
 	rm -rf -- "$BUILD_ROOT"/* 2> /dev/null || true
 	chattr -a -A -i -R -- "$BUILD_ROOT" 2> /dev/null || true
 	rm -rf -- "$BUILD_ROOT"/*
@@ -486,6 +488,7 @@ if test -e "$BUILD_IS_RUNNING" ; then
     umount -n $BUILD_ROOT/proc 2> /dev/null
     umount -n $BUILD_ROOT/dev/pts 2> /dev/null
     umount -n $BUILD_ROOT/mnt 2> /dev/null
+    umount -n $BUILD_ROOT/.ccache 2> /dev/null || true
     echo "Your build system is broken!! Shall I execute"
     echo
     echo "    rm -rf -- $BUILD_ROOT/*"