Implement xlocate to sync with all the repos defined

leahneukirchen · Mar 20, 2022 · 4047c61 · 4047c61
1 parent eedea0e
commit 4047c61
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 43 deletions.
diff --git a/README b/README
@@ -75,11 +75,9 @@ COMMANDS
         - use ':' to lint all templates staged in the git index
 
      xlocate
-        [-g | -S | pattern] – locate files in all XBPS packages
-          -g  Update a git based xlocate database, useful for local
-              repositories
-          -S  Sync with the official git based xlocate database, which is
-              recommended before using the tool
+        [-S | pattern] – locate files in all XBPS packages
+          -S  Sync with all the repos defined in /etc/xbps.d or
+              /usr/share/xbps.d
 
      xlog pkg
         – open commit log for XBPS template

diff --git a/xlocate b/xlocate
@@ -1,5 +1,6 @@
-#!/bin/sh
-# xlocate [-g | -S | PATTERN] - locate files in all XBPS packages
+#!/bin/bash
+# xlocate [-S | PATTERN] - locate files in all XBPS packages
+# By default tries to pull from precompiled git repo and uses xbps-query as a fallback
 
 : ${XDG_CACHE_HOME:=~/.cache}
 : ${XDG_CONFIG_HOME:=~/.config}
@@ -10,33 +11,75 @@ if [ -f "${XLOCATE_CONF}" ]; then
 fi
 
 : ${XLOCATE_GIT:=$XDG_CACHE_HOME/xlocate.git}
-: ${XLOCATE_REPO:=https://alpha.de.repo.voidlinux.org/xlocate/xlocate.git}
+
+REPO_CONFS=($(sort -u <(ls /etc/xbps.d) <(ls /usr/share/xbps.d) | xargs -I{} sh -c 'grep -l "^\s*repository=" /etc/xbps.d/{} 2>/dev/null || grep -l "^\s*repository=" /usr/share/xbps.d/{}'))
+REPOS=($(printf '%s\n' "${REPO_CONFS[@]}" | xargs -I{} bash -c 'source {} && echo $repository'))
+REPOS_DBPATH=($(printf '%s\n' "${REPOS[@]}" | awk '{if(index($0, ":")) {gsub(/[\.:\/]/, "_", $0); gsub(/^/, "/var/db/xbps/", $0);} print}'))
+XLOCATE_REPOS=($(printf '%s\n' "${REPOS[@]}" | sed 's/^\(.*\)\/.*$/\1\/xlocate\/xlocate.git/'))
 
 if command -v pv >/dev/null; then
 	PROGRESS="pv -l"
 else
 	PROGRESS=cat
 fi
 
+# drops all commits matching and below
+drop_commits() {
+	git rev-parse HEAD &> /dev/null \
+	&& GIT_SEQUENCE_EDITOR="sed -i '/$1/,\$d'" git rebase -i --root \
+	|| true
+}
+
 xupdategit() {
 	set -e
 	DIR=$(mktemp -dt xlocate.XXXXXX)
 	DIR=$(/usr/bin/realpath -e "$DIR")
-	git init -q $DIR
 	cd $DIR
-	xbps-query -M -Ro '*' | $PROGRESS | awk '
-		$0 ~ ": " {
-			s = index($0, ": ")
-			pkg = substr($0, 1, s-1)
-			file = substr($0, s+2)
-			sub(" *\\([^)]*\\)$", "", file)
-			print file >>pkg
-			}'
-	printf '%s\n' ./* |
-		LC_ALL= xargs -d'\n' -I'{}' -n1 -P "$(nproc)" -r -- \
-			sort -o {} {}
-	git add ./*
-	git -c user.name=xupdategit -c user.email=xupdategit@none commit -q -m 'xupdategit'
+	git init -q
+	git checkout -b master -q
+	if [[ -d "$XLOCATE_GIT" ]]; then
+		git pull "$XLOCATE_GIT" master &>/dev/null
+		newer_option="-newer $XLOCATE_GIT/refs/heads/master"
+	fi
+
+	for ((i=0; i<${#REPO_CONFS[@]}; i++)); do
+		if [[ -n $rebuild || -n $(find "${REPOS_DBPATH[i]}" -maxdepth 1 -type f -name '*-repodata' $newer_option) ]]; then
+			conf_file_name=$(basename ${REPO_CONFS[i]} .conf) remote_name=$conf_file_name commit_name=$conf_file_name
+
+			[[ ! -n $rebuild ]] && drop_commits $commit_name
+
+			if git ls-remote ${XLOCATE_REPOS[i]} &>/dev/null; then
+				git remote remove $remote_name 2>/dev/null || true
+				git remote add $remote_name ${XLOCATE_REPOS[i]}
+				branch_name=$(git remote show $remote_name | sed -n '/HEAD branch/s/.*: //p')
+				git fetch $remote_name $branch_name
+
+				# only checkout file we're not having from higher priority repo
+				git rev-parse HEAD &> /dev/null \
+				&& git diff-tree -r --diff-filter=A HEAD $remote_name/$branch_name --name-only --no-commit-id | while IFS= read -r file; do git checkout $remote_name/$branch_name -- ":/$file"; done \
+				|| git checkout $remote_name/$branch_name -- '*'
+			else
+				xbps-query --repository=${REPOS[i]} -Mio '*' | awk '
+					$0 ~ ": " {
+					s = index($0, ": ")
+					pkg = substr($0, 1, s-1)
+					file = substr($0, s+2)
+					sub(" *\\([^)]*\\)$", "", file)
+					print file >>pkg
+				}'
+
+				# only keep file we're not having from higher priority repo
+				git diff --diff-filter=M --name-only | while IFS= read -r file; do git restore -- ":/$file"; done
+			fi
+			git add ./*
+			git -c user.name=xupdategit -c user.email=xupdategit@none commit -q -m "xupdategit: $commit_name"
+			printf '%s\n' ./* |
+				LC_ALL= xargs -d'\n' -I'{}' -n1 -P "$(nproc)" -r -- \
+					sort -o {} {}
+			rebuild=1
+		fi
+	done
+
 	git repack -ad
 	rm -rf "$XLOCATE_GIT" .git/COMMIT_EDITMSG .git/description \
 		.git/index .git/hooks .git/logs
@@ -45,38 +88,23 @@ xupdategit() {
 	rm -rf "$DIR"
 }
 
-xsyncgit() {
-	if ! [ -d "$XLOCATE_GIT" ]; then
-		[ -n "${XLOCATE_GIT%/*}" ] && mkdir -p "${XLOCATE_GIT%/*}"
-		git clone --bare "$XLOCATE_REPO" "$XLOCATE_GIT"
-	fi
-	git -C "$XLOCATE_GIT" fetch -u -f "$XLOCATE_REPO" master:master
-}
-
 case "$1" in
--g)
-	xupdategit
-	exit $?;;
 -S)
-	xsyncgit
+	xupdategit
 	exit $?;;
 '')
-	echo "Usage: xlocate [-g | -S | PATTERN]" 1>&2
+	echo "Usage: xlocate [-S | PATTERN]" 1>&2
 	exit 1;;
 esac
 
 if [ -d "$XLOCATE_GIT" ]; then
-	if [ -f "$XLOCATE_GIT/refs/heads/master" ]; then
+	if [[ -f "$XLOCATE_GIT/refs/heads/master" ]]; then
 		BASE="$XLOCATE_GIT/refs/heads/master"
-	elif [ -f "$XLOCATE_GIT/FETCH_HEAD" ]; then
+	elif [[ -f "$XLOCATE_GIT/FETCH_HEAD" ]]; then
 		BASE="$XLOCATE_GIT/FETCH_HEAD"
 	fi
-	if [ -z "$BASE" ] || find /var/db/xbps/ -name '*repodata' -newer "$BASE" | grep -q .; then
-		if grep -q origin "$XLOCATE_GIT/config"; then
-			echo "xlocate: database outdated, please run xlocate -S." 1>&2
-		else
-			echo "xlocate: database outdated, please run xlocate -g." 1>&2
-		fi
+	if [[ -z "$BASE" || -n $(find "${REPOS_DBPATH[@]}" -maxdepth 1 -type f -name '*-repodata' -newer "$BASE") ]]; then
+		echo "xlocate: database outdated, please run xlocate -S." 1>&2
 	fi
 	git -c grep.lineNumber=false --git-dir="$XLOCATE_GIT" grep -- "$@" @ |
 		sed 's/^@://; s/:/\t/' | grep .