From 467435aeeaf83485430ad6e55c7213a2df1279ab Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 05:49:00 +0000 Subject: [PATCH 1/3] fix(xsnap): force rebuild if build config changes --- .dockerignore | 1 + packages/xsnap/.gitignore | 1 + packages/xsnap/build.env | 2 +- packages/xsnap/src/build.js | 24 ++++++++++++++++++++++-- packages/xsnap/xsnap-native | 2 +- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6e6aa7d3e00..97f2dd62b43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ packages/stat-logger **/_agstate .vagrant endo-sha.txt +packages/xsnap/build.config.env # When changing/adding entries here, make sure to search the whole project for # `@@AGORIC_DOCKER_SUBMODULES@@` packages/xsnap/moddable diff --git a/packages/xsnap/.gitignore b/packages/xsnap/.gitignore index 04f5262f86f..2cee34ad100 100644 --- a/packages/xsnap/.gitignore +++ b/packages/xsnap/.gitignore @@ -1,4 +1,5 @@ build +build.config.env dist test/fixture-snap-pool/ test/fixture-snap-shot.xss diff --git a/packages/xsnap/build.env b/packages/xsnap/build.env index cf11308c6fc..b816a3002cd 100644 --- a/packages/xsnap/build.env +++ b/packages/xsnap/build.env @@ -1,4 +1,4 @@ MODDABLE_URL=https://github.com/agoric-labs/moddable.git MODDABLE_COMMIT_HASH=f6c5951fc055e4ca592b9166b9ae3cbb9cca6bf0 XSNAP_NATIVE_URL=https://github.com/agoric-labs/xsnap-pub -XSNAP_NATIVE_COMMIT_HASH=2d8ccb76b8508e490d9e03972bb4c64f402d5135 +XSNAP_NATIVE_COMMIT_HASH=eef9b67da5517ed18ff9e0073b842db20924eae3 diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 3bd1d4c0454..418355405e0 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -220,6 +220,7 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => { * existsSync: typeof import('fs').existsSync, * rmdirSync: typeof import('fs').rmdirSync, * readFile: typeof import('fs').promises.readFile, + * writeFile: typeof import('fs').promises.writeFile, * }, * os: { * type: typeof import('os').type, @@ -230,6 +231,21 @@ const makeXsnap = async ({ spawn, fs, os }) => { const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); + const configEnvs = [ + `XSNAP_VERSION=${pkg.version}`, + `CC=cc "-D__has_builtin(x)=1"`, + ]; + + const configEnvFile = asset('../build.config.env'); + const existingConfigEnvs = fs.existsSync(configEnvFile) + ? await fs.readFile(configEnvFile, 'utf-8') + : ''; + + const expectedConfigEnvs = configEnvs.concat('').join('\n'); + if (existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { + await fs.writeFile(configEnvFile, expectedConfigEnvs); + } + const platform = ModdableSDK.platforms[os.type()]; if (!platform) { throw Error(`Unsupported OS found: ${os.type()}`); @@ -241,8 +257,10 @@ const makeXsnap = async ({ spawn, fs, os }) => { [ `MODDABLE=${ModdableSDK.MODDABLE}`, `GOAL=${goal}`, - `XSNAP_VERSION=${pkg.version}`, - `CC=cc "-D__has_builtin(x)=1"`, + // Any other configuration variables that affect the build output + // should be placed in `configEnvs` to force a rebuild if they change + ...configEnvs, + `EXTRA_DEPS=${configEnvFile}`, '-f', 'xsnap-worker.mk', ], @@ -263,6 +281,7 @@ const makeXsnap = async ({ spawn, fs, os }) => { * existsSync: typeof import('fs').existsSync, * rmdirSync: typeof import('fs').rmdirSync, * readFile: typeof import('fs').promises.readFile, + * writeFile: typeof import('fs').promises.writeFile, * }, * os: { * type: typeof import('os').type, @@ -344,6 +363,7 @@ const run = () => spawn: childProcessTop.spawn, fs: { readFile: fsTop.promises.readFile, + writeFile: fsTop.promises.writeFile, existsSync: fsTop.existsSync, rmdirSync: fsTop.rmdirSync, }, diff --git a/packages/xsnap/xsnap-native b/packages/xsnap/xsnap-native index 2d8ccb76b85..eef9b67da55 160000 --- a/packages/xsnap/xsnap-native +++ b/packages/xsnap/xsnap-native @@ -1 +1 @@ -Subproject commit 2d8ccb76b8508e490d9e03972bb4c64f402d5135 +Subproject commit eef9b67da5517ed18ff9e0073b842db20924eae3 From a22772e03e9141dfc651d7bb8cadf30f5bfe2dea Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 05:50:08 +0000 Subject: [PATCH 2/3] fix(agd): check xsnap was rebuilt --- packages/xsnap/package.json | 2 ++ packages/xsnap/scripts/get_xsnap_version.sh | 14 ++++++++++++++ repoconfig.sh | 3 +-- scripts/agd-builder.sh | 15 +++------------ 4 files changed, 20 insertions(+), 14 deletions(-) create mode 100755 packages/xsnap/scripts/get_xsnap_version.sh diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 3d67a418912..fd157a7be7d 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -16,6 +16,7 @@ "build:env": "node src/build.js --show-env > build.env", "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env", "build": "yarn build:bin && yarn build:env", + "check-version": "xsnap_version=$(./scripts/get_xsnap_version.sh); if test \"${npm_package_version}\" != \"${xsnap_version}\"; then echo \"xsnap version mismatch; expected '${npm_package_version}', got '${xsnap_version}'\"; exit 1; fi", "postinstall": "npm run build:from-env", "clean": "rm -rf xsnap-native/xsnap/build", "lint": "run-s --continue-on-error lint:*", @@ -56,6 +57,7 @@ "moddable/xs/makefiles", "moddable/xs/platforms/*.h", "moddable/xs/sources", + "scripts", "src", "xsnap-native/xsnap/makefiles", "xsnap-native/xsnap/sources" diff --git a/packages/xsnap/scripts/get_xsnap_version.sh b/packages/xsnap/scripts/get_xsnap_version.sh new file mode 100755 index 00000000000..065c3724ab2 --- /dev/null +++ b/packages/xsnap/scripts/get_xsnap_version.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +# the xsnap binary lives in a platform-specific directory +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) platform=lin ;; + Darwin*) platform=mac ;; + *) platform=win ;; +esac + +# extract the xsnap package version from the long version printed by xsnap-worker +"./xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -v | sed -e 's/^xsnap \([^ ]*\) (XS [^)]*)$/\1/g' diff --git a/repoconfig.sh b/repoconfig.sh index 14194defb79..231093a2c66 100644 --- a/repoconfig.sh +++ b/repoconfig.sh @@ -4,7 +4,6 @@ NODEJS_VERSION=v16 GOLANG_VERSION=1.20.3 GOLANG_DIR=golang/cosmos GOLANG_DAEMON=$GOLANG_DIR/build/agd -XSNAP_VERSION=agoric-upgrade-10 # Args are major, minor and patch version numbers golang_version_check() { @@ -12,7 +11,7 @@ golang_version_check() { [ "$1" -eq 1 ] && [ "$2" -eq 20 ] && [ "$3" -ge 2 ] && return 0 [ "$1" -eq 1 ] && [ "$2" -ge 21 ] && return 0 [ "$1" -ge 2 ] && return 0 - } 2>/dev/null + } 2> /dev/null echo 1>&2 "need go version 1.20.2+, 1.21+, or 2+" return 1 } diff --git a/scripts/agd-builder.sh b/scripts/agd-builder.sh index e55a4173c3e..39a1648dc44 100755 --- a/scripts/agd-builder.sh +++ b/scripts/agd-builder.sh @@ -219,21 +219,12 @@ $do_not_build || ( echo "At least $src is newer than gyp bindings" (cd "$GOLANG_DIR" && lazy_yarn build:gyp) } + + # check the built xsnap version against the package version it should be using + (cd "${thisdir}/../packages/xsnap" && npm run -s check-version) || exit 1 fi ) -# the xsnap binary lives in a platform-specific directory -unameOut="$(uname -s)" -case "${unameOut}" in - Linux*) platform=lin ;; - Darwin*) platform=mac ;; - *) platform=win ;; -esac - -# check the xsnap version against our baked-in notion of what version we should be using -xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n) -[[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}" - if $only_build; then echo "Build complete." 1>&2 exit 0 From 2b53896d5d47ce84630be9215803cf9a302fcc6c Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 08:08:30 +0000 Subject: [PATCH 3/3] feat(xsnap): force rebuild if binary version mismatch --- packages/xsnap/src/build.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 418355405e0..23032349121 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -226,8 +226,10 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => { * type: typeof import('os').type, * } * }} io + * @param {object} [options] + * @param {boolean} [options.forceBuild] */ -const makeXsnap = async ({ spawn, fs, os }) => { +const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => { const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); @@ -242,7 +244,7 @@ const makeXsnap = async ({ spawn, fs, os }) => { : ''; const expectedConfigEnvs = configEnvs.concat('').join('\n'); - if (existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { + if (forceBuild || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { await fs.writeFile(configEnvFile, expectedConfigEnvs); } @@ -347,7 +349,18 @@ async function main(args, { env, stdout, spawn, fs, os }) { if (!showEnv) { if (hasSource) { - await makeXsnap({ spawn, fs, os }); + // Force a rebuild if for some reason the binary is out of date + // Since the make checks may not always detect that situation + let forceBuild = !hasBin; + if (hasBin) { + const npm = makeCLI('npm', { spawn }); + await npm + .run(['run', '-s', 'check-version'], { cwd: asset('..') }) + .catch(() => { + forceBuild = true; + }); + } + await makeXsnap({ spawn, fs, os }, { forceBuild }); } else if (!hasBin) { throw new Error( 'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',