Add support for hashed mode to Poudriere

src/etc/poudriere.conf.sample

@@ -383,3 +383,16 @@ DISTFILES_CACHE=/usr/ports/distfiles
 # be fetched.
 # Default: everything
 #PACKAGE_FETCH_WHITELIST="gcc* rust llvm*"
+
+# Have pkg create the repo such that each package is named with the short hash
+# of its file contents in the package filename, with symlinks to the traditional
+# package filenames. The packagesite.yaml file will point to the hashed version
+# of these files. By using hashed pkg filenames, this allows users to lazily
+# cache packages without conflicting with the existing packages, or serving stale
+# packages from a cache. Once the packages are synced the much
+# smaller meta files can then be synced. Allowing a near atomic update of the repo.
+# On caching CDNs this means a need to purge 2-5 files instead of all pkgs that
+# have been updated.
+# The symlinks are only required for the local poudriere for resovling dependencies,
+# they do not need to be uploaded to the CDN.
+#PKG_HASH="no"

src/man/poudriere-bulk.8

@@ -28,7 +28,7 @@
 .\"
 .\" Note: The date here should be updated whenever a non-trivial
 .\" change is made to the manual page.
-.Dd July 5, 2022
+.Dd September 26, 2022
 .Dt POUDRIERE-BULK 8
 .Os
 .Sh NAME
@@ -38,7 +38,7 @@
 .Nm
 .Fl a
 .Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
 .Op Fl B Ar name
 .Op Fl b Ar branch
 .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -48,7 +48,7 @@
 .Nm
 .Fl f Ar file Op Fl f Ar file2 Ar ...
 .Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
 .Op Fl B Ar name
 .Op Fl b Ar branch
 .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -57,7 +57,7 @@
 .Op Fl z Ar set
 .Nm
 .Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
 .Op Fl B Ar name
 .Op Fl b Ar branch
 .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -232,6 +232,8 @@ Fetch only from the original
 Skip
 .Fx
 mirrors.
+.It Fl H
+Create a repository where the package filenames contain the short hash of the contents.
 .It Fl I
 Advanced interactive mode.
 .Pp

src/share/poudriere/bulk.sh

@@ -47,6 +47,8 @@ Options:
                    -f file.  Implies -c for -a.
     -c          -- Clean all the previously built binary packages and logs.
     -F          -- Only fetch from original master_site (skip FreeBSD mirrors)
+    -H          -- Create a repository where the package filenames contain the
+                   short hash of the contents.
     -I          -- Advanced Interactive mode. Leaves jail running with ports
                    installed after test.
     -i          -- Interactive mode. Enter jail for interactive testing and
@@ -97,7 +99,7 @@ if [ $# -eq 0 ]; then
 	usage
 fi
 
-while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
+while getopts "ab:B:CcFf:HiIj:J:knNO:p:RrSTtvwz:" FLAG; do
 	case "${FLAG}" in
 		a)
 			ALL=1
@@ -126,6 +128,9 @@ while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
 			fi
 			LISTPKGS="${LISTPKGS:+${LISTPKGS} }${OPTARG}"
 			;;
+		H)
+			PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }--hash --symlink"
+			;;
 		I)
 			INTERACTIVE_MODE=2
 			;;

src/share/poudriere/common.sh

@@ -6473,6 +6473,10 @@ delete_old_pkg() {
 		if [ -L "${pkg}" ]; then
 			is_sym=1
 		fi
+		if [ -d "${pkg}" ] && [ "${pkgfile}" = "Hashed" ]; then
+			msg_debug "Ignoring directory"
+			return 0;
+		fi
 		if [ "${is_sym}" -eq 1 ] && [ ! -e "${pkg}" ]; then
 			msg "Deleting ${COLOR_PORT}${pkgfile}${COLOR_RESET}: dead symlink"
 			delete_pkg "${pkg}"
@@ -9468,12 +9472,16 @@ clean_restricted() {
 }
 
 build_repo() {
-	local origin pkg_repo_list_files
+	local origin pkg_repo_list_files hashcmd
 
 	msg "Creating pkg repository"
 	if [ ${DRY_RUN} -eq 1 ]; then
 		return 0
 	fi
+	if [ ${PKG_HASH} != "no" ]; then
+		hashcmd="--hash --symlink"
+		PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }$hashcmd"
+	fi
 	bset status "pkgrepo:"
 	ensure_pkg_installed force_extract || \
 	    err 1 "Unable to extract pkg."
@@ -9493,12 +9501,20 @@ build_repo() {
 		install -m 0400 "${PKG_REPO_META_FILE}" \
 		    "${MASTERMNT:?}/tmp/pkgmeta"
 	fi
+
+	# Remount rw
+	# mount_nullfs does not support mount -u
+	umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+	    umount -f ${MASTERMNT}/packages
+	mount_packages
+
 	mkdir -p ${MASTERMNT}/tmp/packages
 	if [ -n "${PKG_REPO_SIGNING_KEY}" ]; then
 		msg "Signing repository with key: ${PKG_REPO_SIGNING_KEY}"
 		install -m 0400 "${PKG_REPO_SIGNING_KEY}" \
 			"${MASTERMNT:?}/tmp/repo.key"
 		injail ${PKG_BIN:?} repo \
+			${PKG_REPO_FLAGS} \
 			${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
 			-o /tmp/packages \
 			${PKG_META} \
@@ -9515,6 +9531,7 @@ build_repo() {
 		# using SSH with DNSSEC as older hosts don't support
 		# it.
 		${MASTERMNT:?}${PKG_BIN:?} repo \
+		    ${PKG_REPO_FLAGS} \
 		    ${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
 		    -o "${MASTERMNT:?}/tmp/packages" ${PKG_META_MASTERMNT} \
 		    "${MASTERMNT:?}/packages" \
@@ -9527,6 +9544,7 @@ build_repo() {
 			;;
 		esac
 		JNETNAME="n" injail ${PKG_BIN:?} repo \
+		    ${PKG_REPO_FLAGS} \
 		    ${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
 		    -o /tmp/packages ${PKG_META} /packages \
 		    ${SIGNING_COMMAND:+signing_command: ${SIGNING_COMMAND}} ||
@@ -9542,6 +9560,11 @@ build_repo() {
 			sign_pkg pubkey "${PACKAGES:?}/Latest/pkg.${PKG_EXT}"
 		fi
 	fi
+
+	# Remount ro
+	umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+	    umount -f ${MASTERMNT}/packages
+	mount_packages -o ro
 }
 
 calculate_size_in_mb() {
@@ -10109,6 +10132,7 @@ esac
 : ${FLAVOR_DEFAULT_ALL:=no}
 : ${NULLFS_PATHS:="/rescue /usr/share /usr/tests /usr/lib32"}
 : ${PACKAGE_FETCH_URL:="pkg+http://pkg.FreeBSD.org/\${ABI}"}
+: ${PKG_HASH:=no}
 
 : ${POUDRIERE_TMPDIR:=$(command mktemp -dt poudriere)}
 : ${SHASH_VAR_PATH_DEFAULT:=${POUDRIERE_TMPDIR}}

src/share/poudriere/include/pkg.sh

@@ -400,6 +400,10 @@ delete_pkg() {
 	local pkg="$1"
 
 	clear_pkg_cache "${pkg}"
+
+	# If ${pkg} is a symlink, delete the target as well
+	[ -L "${pkg}" ] && unlink $(realpath "${pkg}")
+
 	# Delete the package and the depsfile since this package is being deleted,
 	# which will force it to be recreated
 	unlink "${pkg}"
@@ -417,6 +421,8 @@ delete_pkg_xargs() {
 	# Delete the package and the depsfile since this package is being deleted,
 	# which will force it to be recreated
 	{
+		# If ${pkg} is a symlink, delete the target as well
+		[ -L "${pkg}" ] && echo $(realpath "${pkg}")
 		echo "${pkg}"
 		echo "${pkg_cache_dir}"
 	} >> "${listfile}"

src/share/poudriere/pkgclean.sh

@@ -289,6 +289,8 @@ check_should_delete_pkg() {
 	*".${PKG_EXT}")
 		if should_delete "${file}"; then
 			echo "${file}" >> "${BADFILES_LIST:?}"
+			# If the pkg is a symlink to a hashed package, remove the hashed version as well
+			[ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
 		fi
 		;;
 	*.txz)
@@ -305,6 +307,8 @@ check_should_delete_pkg() {
 	*)
 		msg_verbose "Found incorrect format file: ${file}"
 		echo "${file}" >> "${BADFILES_LIST:?}"
+		# If the pkg is a symlink to a hashed package, remove the hashed version as well
+		[ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
 		;;
 	esac
 }