diff --git a/.dockerignore b/.dockerignore index 3647c99be6d..5036bb4f5db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,7 +17,16 @@ packages/stat-logger **/swingset-kernel-state **/_agstate .vagrant + # When changing/adding entries here, make sure to search the whole project for # `@@AGORIC_DOCKER_SUBMODULES@@` +# +# We avoid copying these into a docker build context, because we're +# also not copying the .git directories. If someone runs "docker +# build" from a non-clean agoric-sdk tree, the build context would +# have moddable/ source files but no moddable/.git, and that would be +# confused with an unpacked NPM tarball. See +# packages/xsnap/src/build.js for details. + packages/xsnap/moddable packages/xsnap/xsnap-native diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bdfb8ab54e8..107435e91f1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,9 +53,6 @@ jobs: run: echo "GIT_REVISION=$(git rev-parse HEAD)" >> $GITHUB_ENV - name: Save GIT_COMMIT run: echo "GIT_COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - name: Save commit hash, url of submodules to environment - run: | - node packages/xsnap/src/build.js --show-env >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -84,14 +81,8 @@ jobs: platforms: ${{ matrix.platform }} push: true tags: '${{ env.REGISTRY }}/agoric/agoric-sdk:${{ env.BUILD_TAG }}' - # When changing/adding entries here, make sure to search the whole - # project for `@@AGORIC_DOCKER_SUBMODULES@@` build-args: | GIT_COMMIT=${{env.GIT_COMMIT}} - MODDABLE_COMMIT_HASH=${{env.MODDABLE_COMMIT_HASH}} - MODDABLE_URL=${{env.MODDABLE_URL}} - XSNAP_NATIVE_COMMIT_HASH=${{env.XSNAP_NATIVE_COMMIT_HASH}} - XSNAP_NATIVE_URL=${{env.XSNAP_NATIVE_URL}} GIT_REVISION=${{env.GIT_REVISION}} - name: Build and Push setup uses: docker/build-push-action@v4 diff --git a/packages/deployment/Dockerfile.sdk b/packages/deployment/Dockerfile.sdk index 71b0ad65307..d016e907bbe 100644 --- a/packages/deployment/Dockerfile.sdk +++ b/packages/deployment/Dockerfile.sdk @@ -34,35 +34,26 @@ RUN set -eux; \ # The js build container FROM node:18-bullseye AS build-js -# When changing/adding entries here, make sure to search the whole project for -# `@@AGORIC_DOCKER_SUBMODULES@@` -ARG MODDABLE_COMMIT_HASH -ARG MODDABLE_URL -ARG XSNAP_NATIVE_COMMIT_HASH -ARG XSNAP_NATIVE_URL - WORKDIR /usr/src/agoric-sdk COPY . . # add retry for qemu arm64 network fetching and mui issues with qemu -RUN bash -c "for i in {1..3}; do yarn install --network-timeout 1000000 && exit 0 || (echo retrying; sleep 15;) done; exit 1" +RUN \ + XSNAP_IS_IN_DOCKER=1 \ + bash -c "for i in {1..3}; do yarn install --network-timeout 1000000 && exit 0 || (echo retrying; sleep 15;) done; exit 1" # Need to build the Node.js node extension that uses our above Golang shared library. COPY --from=cosmos-go /usr/src/agoric-sdk/golang/cosmos/build golang/cosmos/build/ RUN cd golang/cosmos && yarn build:gyp -# Check out the specified submodule versions. -# When changing/adding entries here, make sure to search the whole project for -# `@@AGORIC_DOCKER_SUBMODULES@@` +# XSNAP_IS_IN_DOCKER tells xsnap that it needs to git-clone the +# submodules. See packages/xsnap/src/build.js for details RUN \ - MODDABLE_COMMIT_HASH="$MODDABLE_COMMIT_HASH" \ - MODDABLE_URL="$MODDABLE_URL" \ - XSNAP_NATIVE_COMMIT_HASH="$XSNAP_NATIVE_COMMIT_HASH" \ - XSNAP_NATIVE_URL="$XSNAP_NATIVE_URL" \ + XSNAP_IS_IN_DOCKER=1 \ yarn build # Remove dev dependencies. -RUN rm -rf packages/xsnap/moddable packages/xsnap/xsnap-native/build/tmp +RUN rm -rf packages/xsnap/moddable packages/xsnap/xsnap-native/xsnap/build/tmp # FIXME: This causes bundling differences. https://github.com/endojs/endo/issues/919 # RUN yarn install --frozen-lockfile --production --network-timeout 100000 diff --git a/packages/deployment/Makefile b/packages/deployment/Makefile index 0f9316a9929..45f07c5b0ab 100644 --- a/packages/deployment/Makefile +++ b/packages/deployment/Makefile @@ -25,8 +25,7 @@ docker-build: docker-build-sdk docker-build-solo \ docker-build-setup docker-build-ssh-node docker-build-sdk: - bargs=`node ../xsnap/src/build.js --show-env | sed -e 's/^/ --build-arg=/'`; \ - docker build $$bargs --build-arg=GIT_REVISION=$(GIT_REVISION) \ + docker build --build-arg=GIT_REVISION=$(GIT_REVISION) \ -t $(REPOSITORY_SDK):$(TAG) --file=Dockerfile.sdk ../.. docker tag $(REPOSITORY_SDK):$(TAG) $(REPOSITORY_SDK):latest diff --git a/packages/xsnap/build.env b/packages/xsnap/build.env deleted file mode 100644 index cf11308c6fc..00000000000 --- a/packages/xsnap/build.env +++ /dev/null @@ -1,4 +0,0 @@ -MODDABLE_URL=https://github.com/agoric-labs/moddable.git -MODDABLE_COMMIT_HASH=f6c5951fc055e4ca592b9166b9ae3cbb9cca6bf0 -XSNAP_NATIVE_URL=https://github.com/agoric-labs/xsnap-pub -XSNAP_NATIVE_COMMIT_HASH=2d8ccb76b8508e490d9e03972bb4c64f402d5135 diff --git a/packages/xsnap/build.json b/packages/xsnap/build.json new file mode 100644 index 00000000000..9d73788bae2 --- /dev/null +++ b/packages/xsnap/build.json @@ -0,0 +1,10 @@ +{ + "moddable": { + "url": "https://github.com/agoric-labs/moddable.git", + "hash": "f6c5951fc055e4ca592b9166b9ae3cbb9cca6bf0" + }, + "xsnap_native": { + "url": "https://github.com/agoric-labs/xsnap-pub", + "hash": "2d8ccb76b8508e490d9e03972bb4c64f402d5135" + } +} diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index a7ef324985e..bffa3a2f906 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -12,11 +12,8 @@ }, "scripts": { "repl": "node src/xsrepl.js", - "build:bin": "if test -d ./test; then node src/build.js; else yarn build:from-env; fi", - "build:env": "test -d ./test && node src/build.js --show-env > build.env", - "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env", - "build": "yarn build:bin && yarn build:env", - "postinstall": "yarn build:from-env", + "build": "node src/build.js", + "postinstall": "yarn build", "clean": "rm -rf xsnap-native/xsnap/build", "lint": "run-s --continue-on-error lint:*", "lint:js": "eslint 'src/**/*.js' 'test/**/*.js' api.js", @@ -48,8 +45,20 @@ "files": [ "LICENSE*", "api.js", - "build.env", - "src" + "build.json", + "src", + "moddable/licenses", + "moddable/readme.md", + "moddable/tools", + "moddable/xs", + "moddable/modules/data/text", + "moddable/modules/data/base64", + "xsnap-native/README.md", + "xsnap-native/xsnap/documentation", + "xsnap-native/xsnap/makefiles", + "xsnap-native/xsnap/readme.md", + "xsnap-native/xsnap/sources", + "xsnap-native/xsnap/xsbug-node" ], "publishConfig": { "access": "public" diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 2fcba467a21..c5c55588e83 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -1,12 +1,75 @@ #!/usr/bin/env node /* global process */ /* eslint-disable @jessie.js/no-nested-await -- test/build code */ +/* eslint-disable no-await-in-loop -- test/build code */ +/* eslint-disable no-lonely-if -- makes the logic easier to read */ import * as childProcessTop from 'child_process'; import fsTop from 'fs'; import osTop from 'os'; +import { join } from 'path'; const { freeze } = Object; +// This package builds 'xsnap' program at install time. At runtime, +// this package's API helps you launch an instance of that program and +// then talk to it (over pipes). +// +// 'xsnap' is built from sources in `./xsnap-native/`, which link +// against a library built from the sources in `./moddable/`. +// +// When built from a git clone of the agoric-sdk, these subdirectories +// are populated as git submodules, so they will be clones of specific +// commit IDs of repos from the "agoric-labs" organization. These +// repos contain Agoric-specific forks of the upstream Moddable +// code. We may observe two cases: +// +// A: the subdirectories do not exist, which means we've cloned +// agoric-sdk but we have not yet run 'git submodule update +// --init' + +// B: they do exist, and they have .git subdirectories, which means +// we've initialized the submodules at least once +// +// When built from an NPM-registry -hosted tarball, these +// subdirectories are filled with source code from the distribution +// tarball (copied into the tarball by virtue of 'files' entries in +// our package.json). This yields a third case: +// +// C: they exist, but they lack a .git subdirectory +// +// When built in a docker build context, the subdirectories will +// initially be missing (if they existed in the original checkout at +// all, they were subsequently excluded by agoric-sdk/.dockerignore), +// however we'll also be missing the top-level .git metadata which +// would allow a "git submodule update --init" to work. This +// environment is distinguished by $XSNAP_IS_IN_DOCKER=1 being set by +// packages/deployment/Dockerfile.sdk, and indicates that we must read +// the URLs and commit hashes from build.json, and then do a "git +// clone", to get the same sources that we would normally get from the +// submodules. +// +// D: the subdirectories do not exist, and $XSNAP_IS_IN_DOCKER is true +// +// In a docker build context, on the second or subsequent times that +// build.js is run, we'll see both $XSNAP_IS_IN_DOCKER=1 and the +// subdirectories existing. +// +// E: the subdirectories exist, and $XSNAP_IS_IN_DOCKER is true +// +// In cases A and B, we want to run `git submodule update --init` on +// each directory, to act upon any change in the desired commit ID +// (e.g. if the developer switched git branches since the last +// build). In case C, we shouldn't do anything (and in fact do not +// need the 'git' executable at all). In case D, we must read the URLs +// from build.json and then do a "git clone". In case E, we should do +// nothing, and leave the sources alone. +// +// A: git submodule update --init +// B: git submodule update --init +// C: do nothing +// D: read build.json, then git clone/checkout for each subdirectory +// E: do nothing + /** @param {string} path */ const asset = path => new URL(path, import.meta.url).pathname; @@ -81,10 +144,10 @@ function makeCLI(command, { spawn }) { /** * @param {string} path - * @param {string} repoUrl - * @param {{ git: ReturnType }} io + * @param {{ fs: { existsSync: typeof import('fs').existsSync }, + * git: ReturnType }} io */ -const makeSubmodule = (path, repoUrl, { git }) => { +const makeSubmodule = (path, { fs, git }) => { /** @param {string} text */ const parseStatus = text => text @@ -109,11 +172,15 @@ const makeSubmodule = (path, repoUrl, { git }) => { return freeze({ path, - clone: async () => git.run(['clone', repoUrl, path]), - /** @param {string} commitHash */ - checkout: async commitHash => - git.run(['checkout', commitHash], { cwd: path }), - init: async () => git.run(['submodule', 'update', '--init', '--checkout']), + clone: async repoUrl => git.run(['clone', repoUrl, path]), + /** @param {string} hash */ + checkout: async hash => git.run(['checkout', hash], { cwd: path }), + init: async () => + git.run(['submodule', 'update', '--init', '--checkout', path]), + update: async () => + git.run(['submodule', 'update', '--init', '--checkout', path]), + exists: () => fs.existsSync(path), + hasDotGit: () => fs.existsSync(join(path, '.git')), status: async () => git.pipe(['submodule', 'status', path]).then(parseStatus), /** @param {string} leaf */ @@ -139,6 +206,16 @@ const makeSubmodule = (path, repoUrl, { git }) => { }); }; +const readBuildJSON = async (envRecordFile, { fs }) => { + const data = await fs.readFile(envRecordFile); + return JSON.parse(data); +}; + +const writeBuildJSON = async (envRecordFile, build, { fs }) => { + const data = JSON.stringify(build, undefined, 2); + await fs.writeFile(envRecordFile, `${data}\n`); +}; + /** * @param {string[]} args * @param {{ @@ -146,78 +223,86 @@ const makeSubmodule = (path, repoUrl, { git }) => { * stdout: typeof process.stdout, * spawn: typeof import('child_process').spawn, * fs: { + * readFile: typeof import('fs').promises.readFile, + * writeFile: typeof import('fs').promises.writeFile, * existsSync: typeof import('fs').existsSync, * rmdirSync: typeof import('fs').rmdirSync, - * readFile: typeof import('fs').promises.readFile, * }, * os: { * type: typeof import('os').type, * } * }} io */ -async function main(args, { env, stdout, spawn, fs, os }) { +async function main(args, { env, spawn, fs, os }) { const git = makeCLI('git', { spawn }); - // When changing/adding entries here, make sure to search the whole project - // for `@@AGORIC_DOCKER_SUBMODULES@@` + const inDocker = env.XSNAP_IS_IN_DOCKER; // TODO: we assume it's "1" + const submodules = [ { - url: env.MODDABLE_URL || 'https://github.com/agoric-labs/moddable.git', path: ModdableSDK.MODDABLE, - commitHash: env.MODDABLE_COMMIT_HASH, - envPrefix: 'MODDABLE_', + key: 'moddable', }, { - url: - env.XSNAP_NATIVE_URL || 'https://github.com/agoric-labs/xsnap-pub.git', path: asset('../xsnap-native'), - commitHash: env.XSNAP_NATIVE_COMMIT_HASH, - envPrefix: 'XSNAP_NATIVE_', + key: 'xsnap_native', }, ]; - if (args.includes('--show-env')) { - for (const submodule of submodules) { - const { path, envPrefix, commitHash } = submodule; - if (!commitHash) { - // We need to glean the commitHash and url from Git. - const sm = makeSubmodule(path, '?', { git }); - // eslint-disable-next-line no-await-in-loop - const [[{ hash }], url] = await Promise.all([ - sm.status(), - sm.config('url'), - ]); - submodule.commitHash = hash; - submodule.url = url; - } - stdout.write(`${envPrefix}URL=${submodule.url}\n`); - stdout.write(`${envPrefix}COMMIT_HASH=${submodule.commitHash}\n`); - } - return; - } + // didGit is true if we consulted git and need to update build.json + // with the results. We assume all subdirs are using git, or none are. + let didGit = false; + const buildJSONFile = 'build.json'; + let buildJSON = {}; - for (const { url, path, commitHash } of submodules) { - const submodule = makeSubmodule(path, url, { git }); + for (const { path, key } of submodules) { + const submodule = makeSubmodule(path, { fs, git }); - // Allow overriding of the checked-out version of the submodule. - if (commitHash) { - // Do the moral equivalent of submodule update when explicitly overriding. - try { - fs.rmdirSync(submodule.path); - } catch (_e) { - // ignore - } - if (!fs.existsSync(submodule.path)) { - // eslint-disable-next-line no-await-in-loop - await submodule.clone(); + if (inDocker) { + if (submodule.exists()) { + console.log(`${key} case E: in docker, have submodule/ : do nothing`); + } else { + console.log(`${key} case D: in docker, missing submodule/ : do clone`); + buildJSON = await readBuildJSON(buildJSONFile, { fs }); // must exist + const data = buildJSON[key]; + await submodule.clone(data.url); // requires git + await submodule.checkout(data.hash); } - // eslint-disable-next-line no-await-in-loop - await submodule.checkout(commitHash); } else { - // eslint-disable-next-line no-await-in-loop - await submodule.init(); + // not inDocker + if (submodule.exists()) { + if (submodule.hasDotGit()) { + console.log(`${key} case B: have submodule/.git : do update`); + didGit = true; + await submodule.update(); // requires git + } else { + console.log(`${key} case C: have submodule/, not .git : do nothing`); + } + } else { + console.log(`${key} case A: no submodule/ directory : do init`); + didGit = true; + await submodule.update(); // requires git + } + } + + if (didGit) { + // record the submodule's source URL and git commit hash into + // build.json for later auditing + + // We need to glean the commitHash and url from Git. + const sm = makeSubmodule(path, { fs, git }); + const [[{ hash }], url] = await Promise.all([ + sm.status(), + sm.config('url'), + ]); + buildJSON[key] = { url, hash }; } } + if (didGit) { + await writeBuildJSON(buildJSONFile, buildJSON, { fs }); + } + + // now compile xsnap const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); @@ -252,6 +337,7 @@ main(process.argv.slice(2), { spawn: childProcessTop.spawn, fs: { readFile: fsTop.promises.readFile, + writeFile: fsTop.promises.writeFile, existsSync: fsTop.existsSync, rmdirSync: fsTop.rmdirSync, },