Look in more directories for upgrade binaries

This makes major upgrades compatible with images from other distros.

Issue: PGO-864

Author: cbandy

internal/controller/pgupgrade/jobs.go

@@ -21,6 +21,7 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/feature"
 	"github.com/crunchydata/postgres-operator/internal/initialize"
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/postgres"
 	"github.com/crunchydata/postgres-operator/internal/shell"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
@@ -86,13 +87,25 @@ func upgradeCommand(spec *v1beta1.PGUpgradeSettings, fetchKeyCommand string) []s
 		`id; [[ "$(id -nu)" == 'postgres' && "$(id -ng)" == 'postgres' ]]`,
 
 		`section 'Step 2 of 7: Finding data and tools...'`,
-
-		// Expect Postgres executables at the Red Hat paths.
-		`old_bin="/usr/pgsql-${old_version}/bin" && [[ -x "${old_bin}/postgres" ]]`,
-		`new_bin="/usr/pgsql-${new_version}/bin" && [[ -x "${new_bin}/initdb" ]]`,
 		`old_data="${data_volume}/pg${old_version}" && [[ -d "${old_data}" ]]`,
 		`new_data="${data_volume}/pg${new_version}"`,
 
+		// Search for Postgres executables matching the old and new versions.
+		// Use `command -v` to look through all of PATH, then trim the executable name from the absolute path.
+		`old_bin=$(` + postgres.ShellPath(oldVersion) + ` && command -v postgres)`,
+		`old_bin="${old_bin%/postgres}"`,
+		`new_bin=$(` + postgres.ShellPath(newVersion) + ` && command -v pg_upgrade)`,
+		`new_bin="${new_bin%/pg_upgrade}"`,
+
+		// The executables found might not be the versions we need, so do a cursory check before writing to disk.
+		// pg_upgrade checks every executable thoroughly since PostgreSQL v14.
+		//
+		// https://git.postgresql.org/gitweb/?p=postgresql.git;hb=refs/tags/REL_10_0;f=src/bin/pg_upgrade/exec.c#l355
+		// https://git.postgresql.org/gitweb/?p=postgresql.git;hb=refs/tags/REL_14_0;f=src/bin/pg_upgrade/exec.c#l358
+		// https://git.postgresql.org/gitweb/?p=postgresql.git;hb=refs/tags/REL_18_0;f=src/bin/pg_upgrade/exec.c#l370
+		`(set -x && [[ "$("${old_bin}/postgres" --version)" =~ ") ${old_version}"($|[^0-9]) ]])`,
+		`(set -x && [[ "$("${new_bin}/initdb" --version)"   =~ ") ${new_version}"($|[^0-9]) ]])`,
+
 		// pg_upgrade writes its files in "${new_data}/pg_upgrade_output.d" since PostgreSQL v15.
 		// Change to a writable working directory to be compatible with PostgreSQL v14 and earlier.
 		//
@@ -115,6 +128,9 @@ func upgradeCommand(spec *v1beta1.PGUpgradeSettings, fetchKeyCommand string) []s
 		`value=$(LC_ALL=C PGDATA="${old_data}" "${old_bin}/postgres" -C shared_preload_libraries)`,
 		`echo >> "${new_data}/postgresql.conf" "shared_preload_libraries = '${value//$'\''/$'\'\''}'"`,
 
+		// NOTE: The default for --new-bindir is the directory of pg_upgrade since PostgreSQL v13.
+		//
+		// https://www.postgresql.org/docs/release/13#id-1.11.6.28.5.11
 		`section 'Step 5 of 7: Checking for potential issues...'`,
 		`"${new_bin}/pg_upgrade" --check` + argMethod + argJobs + ` \`,
 		`--old-bindir="${old_bin}" --old-datadir="${old_data}" \`,
@@ -253,7 +269,7 @@ func removeDataCommand(upgrade *v1beta1.PGUpgrade) []string {
 		`delete() (set -x && rm -rf -- "$@")`,
 
 		`old_data="${data_volume}/pg${old_version}"`,
-		`control=$(LC_ALL=C /usr/pgsql-${old_version}/bin/pg_controldata "${old_data}")`,
+		`control=$(` + postgres.ShellPath(oldVersion) + ` && LC_ALL=C pg_controldata "${old_data}")`,
 		`read -r state <<< "${control##*cluster state:}"`,
 
 		// We expect exactly one state for a replica that has been stopped.

internal/controller/pgupgrade/jobs_test.go

@@ -213,10 +213,14 @@ spec:
           export LD_PRELOAD='libnss_wrapper.so' NSS_WRAPPER_GROUP NSS_WRAPPER_PASSWD
           id; [[ "$(id -nu)" == 'postgres' && "$(id -ng)" == 'postgres' ]]
           section 'Step 2 of 7: Finding data and tools...'
-          old_bin="/usr/pgsql-${old_version}/bin" && [[ -x "${old_bin}/postgres" ]]
-          new_bin="/usr/pgsql-${new_version}/bin" && [[ -x "${new_bin}/initdb" ]]
           old_data="${data_volume}/pg${old_version}" && [[ -d "${old_data}" ]]
           new_data="${data_volume}/pg${new_version}"
+          old_bin=$(PATH="/usr/lib/postgresql/19/bin:/usr/libexec/postgresql19:/usr/pgsql-19/bin${PATH+:${PATH}}" && command -v postgres)
+          old_bin="${old_bin%/postgres}"
+          new_bin=$(PATH="/usr/lib/postgresql/25/bin:/usr/libexec/postgresql25:/usr/pgsql-25/bin${PATH+:${PATH}}" && command -v pg_upgrade)
+          new_bin="${new_bin%/pg_upgrade}"
+          (set -x && [[ "$("${old_bin}/postgres" --version)" =~ ") ${old_version}"($|[^0-9]) ]])
+          (set -x && [[ "$("${new_bin}/initdb" --version)"   =~ ") ${new_version}"($|[^0-9]) ]])
           cd "${data_volume}"
           section 'Step 3 of 7: Initializing new data directory...'
           PGDATA="${new_data}" "${new_bin}/initdb" --allow-group-access --data-checksums
@@ -352,7 +356,7 @@ spec:
           printf 'Removing PostgreSQL %s data...\n\n' "$@"
           delete() (set -x && rm -rf -- "$@")
           old_data="${data_volume}/pg${old_version}"
-          control=$(LC_ALL=C /usr/pgsql-${old_version}/bin/pg_controldata "${old_data}")
+          control=$(PATH="/usr/lib/postgresql/19/bin:/usr/libexec/postgresql19:/usr/pgsql-19/bin${PATH+:${PATH}}" && LC_ALL=C pg_controldata "${old_data}")
           read -r state <<< "${control##*cluster state:}"
           [[ "${state}" == 'shut down in recovery' ]] || { printf >&2 'Unexpected state! %q\n' "${state}"; exit 1; }
           delete "${old_data}/pg_wal/"

internal/postgres/config.go

@@ -264,6 +264,17 @@ func Environment(cluster *v1beta1.PostgresCluster) []corev1.EnvVar {
 	}
 }
 
+// ShellPath returns a POSIX shell command that prepends typical Postgres executable paths to the PATH variable.
+func ShellPath(postgresVersion int32) string {
+	return fmt.Sprintf(`PATH="`+
+		strings.Join([]string{
+			`/usr/lib/postgresql/%[1]d/bin`, // Debian
+			`/usr/libexec/postgresql%[1]d`,  // Alpine
+			`/usr/pgsql-%[1]d/bin`,          // Red Hat
+		}, ":")+
+		`${PATH+:${PATH}}"`, postgresVersion)
+}
+
 // reloadCommand returns an entrypoint that convinces PostgreSQL to reload
 // certificate files when they change. The process will appear as name in `ps`
 // and `top`.

internal/postgres/config_test.go

@@ -547,6 +547,37 @@ func TestBashSafeLink(t *testing.T) {
 	})
 }
 
+func TestShellPath(t *testing.T) {
+	t.Parallel()
+
+	script := ShellPath(11)
+
+	assert.Assert(t, cmp.Contains(script, `/usr/lib/postgresql/11/bin`))
+	assert.Assert(t, cmp.Contains(script, `/usr/libexec/postgresql11`))
+	assert.Assert(t, cmp.Contains(script, `/usr/pgsql-11/bin`))
+
+	t.Run("ShellCheckPOSIX", func(t *testing.T) {
+		shellcheck := require.ShellCheck(t)
+
+		dir := t.TempDir()
+		file := filepath.Join(dir, "script.sh")
+		assert.NilError(t, os.WriteFile(file, []byte(script), 0o600))
+
+		// Expect ShellCheck for "sh" to be happy.
+		// - https://www.shellcheck.net/wiki/SC2148
+		cmd := exec.CommandContext(t.Context(), shellcheck, "--enable=all", "--shell=sh", file)
+		output, err := cmd.CombinedOutput()
+		assert.NilError(t, err, "%q\n%s", cmd.Args, output)
+	})
+
+	t.Run("PrettyYAML", func(t *testing.T) {
+		b, err := yaml.Marshal(script)
+		assert.NilError(t, err)
+		assert.Assert(t, !strings.Contains(string(b), `\n`), "expected literal flow scalar, got:\n%s", b)
+		assert.Equal(t, 1, strings.Count(string(b), "\n"), "expected one trailing newline, got:\n%s", b)
+	})
+}
+
 func TestStartupCommand(t *testing.T) {
 	shellcheck := require.ShellCheck(t)
 	t.Parallel()