automatic dependency checking for python 3 modules

Manual.md

@@ -1639,6 +1639,13 @@ written in Python) or just single Python file ones that live in `/usr/bin`.
 If `python_version` is set to `ignore`, python-containing shebangs will not be rewritten.
 Use this only if a package should not be using a system version of python.
 
+- `python_extras`: Python module extras to consider when verifying Python module dependencies.
+Can be used to ensure additional dependency sets are checked. Example: `python_extras="all"`.
+
+- `nopyprovides`: if set, don't create `provides` entries for Python modules in the package.
+
+- `nopyverifydeps`: if set, don't verify Python module dependencies.
+
 Also, a set of useful variables are defined to use in the templates:
 
 | Variable    | Value                            |

common/environment/build-style/python3-module.sh

@@ -1,3 +1,6 @@
 lib32disabled=yes
+if [ -z "$nopyprovides" ] || [ -z "$noverifypydeps" ]; then
+	hostmakedepends+=" python3-packaging-bootstrap"
+fi
 makedepends+=" python3"
 build_helper+=" python3"

common/environment/build-style/python3-pep517.sh

@@ -1,3 +1,6 @@
-hostmakedepends+=" python3-build python3-installer"
 lib32disabled=yes
+hostmakedepends+=" python3-build python3-installer"
+if [ -z "$nopyprovides" ] || [ -z "$noverifypydeps" ]; then
+	hostmakedepends+=" python3-packaging-bootstrap"
+fi
 build_helper+=" python3"

common/environment/setup-subpkg/subpkg.sh

@@ -11,6 +11,9 @@ unset -v nostrip nostrip_files
 # hooks/post-install/14-fix-permissions
 unset -v nocheckperms nofixperms
 
+# hooks/pre-pkg/04-generate-provides
+unset -v nopyprovides
+
 # hooks/pre-pkg/04-generate-runtime-deps
 unset -v noverifyrdeps skiprdeps allow_unknown_shlibs shlib_requires
 
@@ -20,6 +23,9 @@ unset -v lib32depends lib32disabled lib32files lib32mode lib32symlinks
 # hooks/pre-pkg/06-shlib-provides
 unset -v noshlibprovides shlib_provides
 
+# hooks/pre-pkg/06-verify-python-deps
+unset -v noverifypydeps python_extras
+
 # xbps-triggers: system-accounts
 unset -v system_accounts system_groups
 

common/hooks/do-pkg/00-gen-pkg.sh

@@ -34,21 +34,23 @@ genpkg() {
 	cd $pkgdir
 
 	_preserve=${preserve:+-p}
-	if [ -s ${PKGDESTDIR}/rdeps ]; then
-		_deps="$(<${PKGDESTDIR}/rdeps)"
+	if [ -s ${XBPS_STATEDIR}/${pkgname}-rdeps ]; then
+		_deps="$(<${XBPS_STATEDIR}/${pkgname}-rdeps)"
 	fi
-	if [ -s ${PKGDESTDIR}/shlib-provides ]; then
-		_shprovides="$(<${PKGDESTDIR}/shlib-provides)"
+	if [ -s ${XBPS_STATEDIR}/${pkgname}-provides ]; then
+		_provides="$(<${XBPS_STATEDIR}/${pkgname}-provides)"
 	fi
-	if [ -s ${PKGDESTDIR}/shlib-requires ]; then
-		_shrequires="$(<${PKGDESTDIR}/shlib-requires)"
+	if [ -s ${XBPS_STATEDIR}/${pkgname}-shlib-provides ]; then
+		_shprovides="$(<${XBPS_STATEDIR}/${pkgname}-shlib-provides)"
+	fi
+	if [ -s ${XBPS_STATEDIR}/${pkgname}-shlib-requires ]; then
+		_shrequires="$(<${XBPS_STATEDIR}/${pkgname}-shlib-requires)"
 	fi
 	if [ -s ${XBPS_STATEDIR}/gitrev ]; then
 		_gitrevs="$(<${XBPS_STATEDIR}/gitrev)"
 	fi
 
 	# Stripping whitespaces
-	local _provides="$(echo $provides)"
 	local _conflicts="$(echo $conflicts)"
 	local _replaces="$(echo $replaces)"
 	local _reverts="$(echo $reverts)"

common/hooks/post-install/04-create-xbps-metadata-scripts.sh

@@ -274,7 +274,7 @@ _EOF
 	if [ -d ${PKGDESTDIR}/usr/lib/python* ]; then
 		pycompile_version="$(find ${PKGDESTDIR}/usr/lib/python* -prune -type d | grep -o '[[:digit:]]\.[[:digit:]]\+$')"
 		if [ -z "${pycompile_module}" ]; then
-			pycompile_module="$(find ${PKGDESTDIR}/usr/lib/python*/site-packages -mindepth 1 -maxdepth 1 '!' -name '*.egg-info' '!' -name '*.dist-info' '!' -name '*.so' '!' -name '*.pth' -printf '%f ')"
+			pycompile_module="$(find ${PKGDESTDIR}/usr/lib/python*/site-packages* -mindepth 1 -maxdepth 1 '!' -name '*.egg-info' '!' -name '*.dist-info' '!' -name '*.so' '!' -name '*.pth' -printf '%f ')"
 		fi
 	fi
 

common/hooks/post-install/06-strip-and-debug-pkgs.sh

@@ -51,7 +51,7 @@ create_debug_pkg() {
 		msg_red "$pkgver: failed to create debug pkg\n"
 		return 1
 	fi
-	printf "${pkgver} " >> ${_destdir}/rdeps
+	printf "${pkgver} " >> ${XBPS_STATEDIR}/${pkgname}-dbg-rdeps
 	rmdir --ignore-fail-on-non-empty "${PKGDESTDIR}/usr/lib" 2>/dev/null
 	return 0
 }

common/hooks/post-install/98-shlib-provides.sh

@@ -50,8 +50,8 @@ collect_sonames() {
 		echo "$f" >> ${_tmpfile}
 	done
 	if [ -s "${_tmpfile}" ]; then
-		tr '\n' ' ' < "${_tmpfile}" > ${_destdir}/shlib-provides
-		echo >> ${_destdir}/shlib-provides
+		tr '\n' ' ' < "${_tmpfile}" > "${XBPS_STATEDIR}/${pkgname}-shlib-provides"
+		echo >> "${XBPS_STATEDIR}/${pkgname}-shlib-provides"
 		if [ "$_mainpkg" ]; then
 			cp "${_tmpfile}" "${_shlib_dir}/${pkgname}.soname"
 		fi

common/hooks/pre-pkg/04-generate-provides.sh

@@ -0,0 +1,29 @@
+# vim: set ts=4 sw=4 et:
+#
+# This hook executes the following tasks:
+#	- Generates provides file with provides entries for xbps-create(1)
+
+generate_python_provides() {
+    local py3_bin="${XBPS_MASTERDIR}/usr/bin/python3"
+
+    if [ -z "$nopyprovides" ] && [ -d "${PKGDESTDIR}/${py3_sitelib}" ] && [ -x "${py3_bin}" ]; then
+        PYTHONPATH="${XBPS_MASTERDIR}/${py3_sitelib}-bootstrap" "${py3_bin}" \
+            "${XBPS_COMMONDIR}"/scripts/parse-py-metadata.py \
+            -S "${PKGDESTDIR}/${py3_sitelib}" -v "${pkgver}" provides
+    fi
+}
+
+hook() {
+    local -a _provides=()
+
+    # include explicit values from the template
+    read -r -a _provides <<< "$provides"
+
+    # get the canonical python package names for each python module
+    mapfile -t _provides -O "${#_provides[@]}" < <( generate_python_provides )
+
+    if [ "${#_provides[@]}" -gt 0 ]; then
+        echo "   ${_provides[*]}"
+        echo "${_provides[*]}" > "${XBPS_STATEDIR}/${pkgname}-provides"
+    fi
+}

common/hooks/pre-pkg/04-generate-runtime-deps.sh

@@ -41,7 +41,7 @@ store_pkgdestdir_rundeps() {
                     _curdep="${_curdep}>=0"
                 fi
                 printf "%s " "${_curdep}"
-            done > "${PKGDESTDIR}/rdeps"
+            done > "${XBPS_STATEDIR}/${pkgname}-rdeps"
         fi
 }
 
@@ -149,6 +149,6 @@ hook() {
     store_pkgdestdir_rundeps
 
     if [ -n "${sorequires}" ]; then
-        echo "${sorequires}" | xargs -n1 | sort | xargs > ${PKGDESTDIR}/shlib-requires
+        echo "${sorequires}" | xargs -n1 | sort | xargs > ${XBPS_STATEDIR}/${pkgname}-shlib-requires
     fi
 }

common/hooks/pre-pkg/05-generate-32bit-runtime-deps.sh

@@ -19,13 +19,13 @@ hook() {
 	# 32bit dependencies.
 	trap - ERR
 
-	: > ${destdir32}/rdeps
+	: > ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 
-	if [ -s "$PKGDESTDIR/rdeps" ]; then
+	if [ -s "${XBPS_STATEDIR}/${pkgname}-rdeps" ]; then
 		if [ -n "$lib32depends" ]; then
 			_deps="${lib32depends}"
 		else
-			_deps="$(<${PKGDESTDIR}/rdeps)"
+			_deps="$(<${XBPS_STATEDIR}/${pkgname}-rdeps)"
 		fi
 		for f in ${_deps}; do
 			unset found pkgn pkgv _shprovides
@@ -43,7 +43,7 @@ hook() {
 			# If dependency is a development pkg switch it to 32bit.
 			if [[ $pkgn == *-devel ]]; then
 				echo "   RDEP: $f -> ${pkgn}-32bit${pkgv} (development)"
-				printf "${pkgn}-32bit${pkgv} " >> ${destdir32}/rdeps
+				printf "${pkgn}-32bit${pkgv} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 				continue
 			fi
 			# If dependency does not have "shlib-provides" do not
@@ -60,28 +60,28 @@ hook() {
 				_shprovides="$($XBPS_QUERY_CMD -R --property=shlib-provides "$pkgn")"
 				if [ -n "${_shprovides}" ]; then
 					echo "   RDEP: $f -> ${pkgn}-32bit${pkgv} (shlib-provides)"
-					printf "${pkgn}-32bit${pkgv} " >> ${destdir32}/rdeps
+					printf "${pkgn}-32bit${pkgv} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 				else
 					echo "   RDEP: $f -> ${pkgn}${pkgv} (no shlib-provides)"
-					printf "${pkgn}${pkgv} " >> ${destdir32}/rdeps
+					printf "${pkgn}${pkgv} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 				fi
 			else
-				if [ -s ${XBPS_DESTDIR}/${pkgn}-${version}/shlib-provides ]; then
+				if [ -s "${XBPS_STATEDIR}/${pkgn}-shlib-provides" ]; then
 					# Dependency is a subpkg; check if it provides any shlib
 					# and convert to 32bit if true.
 					echo "   RDEP: $f -> ${pkgn}-32bit${pkgv} (subpkg, shlib-provides)"
-					printf "${pkgn}-32bit${pkgv} " >> ${destdir32}/rdeps
+					printf "${pkgn}-32bit${pkgv} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 				else
 					echo "   RDEP: $f -> ${pkgn}${pkgv} (subpkg, no shlib-provides)"
-					printf "${pkgn}${pkgv} " >> ${destdir32}/rdeps
+					printf "${pkgn}${pkgv} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 				fi
 			fi
 		done
 	fi
 	# If it's a development pkg add a dependency to the 64bit pkg.
 	if [[ $pkgn == *-devel ]]; then
 		echo "   RDEP: ${pkgver}"
-		printf "${pkgver} " >> ${destdir32}/rdeps
+		printf "${pkgver} " >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 	fi
-	printf "\n" >> ${destdir32}/rdeps
+	printf "\n" >> ${XBPS_STATEDIR}/${pkgname}-32bit-rdeps
 }

common/hooks/pre-pkg/06-verify-python-deps.sh

@@ -0,0 +1,17 @@
+# vim: set ts=4 sw=4 et:
+#
+# This hook executes the following tasks:
+#	- Verifies python module dependencies from dist-info's METADATA and egg-info's PKG-INFO
+
+hook() {
+    local py3_bin="${XBPS_MASTERDIR}/usr/bin/python3"
+
+    if [ -z "$nopyprovides" ] && [ -d "${PKGDESTDIR}/${py3_sitelib}" ] && [ -x "${py3_bin}" ]; then
+            PYTHONPATH="${XBPS_MASTERDIR}/${py3_sitelib}-bootstrap" "${py3_bin}" \
+                "${XBPS_COMMONDIR}"/scripts/parse-py-metadata.py \
+                ${NOCOLORS:+-C} ${XBPS_STRICT:+-s} -S "${PKGDESTDIR}/${py3_sitelib}" -v "${pkgver}" \
+                depends -e "${python_extras}" \
+                -V <( $XBPS_QUERY_XCMD -R -p provides -s "py3:" ) -D "${XBPS_STATEDIR}/${pkgname}-rdeps" \
+                || msg_error "$pkgver: failed to verify python module dependencies\n"
+    fi
+}

common/hooks/pre-pkg/99-pkglint.sh

@@ -170,11 +170,11 @@ hook() {
 	fi
 
 	# Check for missing shlibs and SONAME bumps.
-	if [ ! -s "${PKGDESTDIR}/shlib-provides" ]; then
+	if [ ! -s "${XBPS_STATEDIR}/${pkgname}-shlib-provides" ]; then
 		return 0
 	fi
 
-	for filename in $(<${PKGDESTDIR}/shlib-provides); do
+	for filename in $(<"${XBPS_STATEDIR}/${pkgname}-shlib-provides"); do
 		rev=${filename#*.so.}
 		libname=${filename%.so*}
 		_shlib=$(echo "$libname"|sed -E 's|\+|\\+|g')

common/hooks/pre-pkg/999-collected-rdeps.sh

@@ -1,7 +1,7 @@
 # This hook displays resolved dependencies for a pkg.
 
 hook() {
-	if [ -e $PKGDESTDIR/rdeps ]; then
-		echo "   $(cat $PKGDESTDIR/rdeps)"
+	if [ -e "${XBPS_STATEDIR}/${pkgname}-rdeps" ]; then
+		echo "   $(cat "${XBPS_STATEDIR}/${pkgname}-rdeps")"
 	fi
 }