From 3f431da92f7fa2173e8dcdd3288abd9487a54f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1n=20Jakub=20Nani=C5=A1ta?= Date: Mon, 4 Dec 2023 13:02:32 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=9A=20OmniGraph=E2=84=A2=20Better=20do?= =?UTF-8?q?cker=20setup=20(#56)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 35 ++++++++-------- README.md | 40 +++++++++++++++++++ docker-compose.templates.yaml | 31 +++++++++++++- ...docker-compose.yaml => docker-compose.yaml | 36 +++++------------ package.json | 2 +- packages/test-evm-node/hardhat.config.ts | 21 ++++++++++ packages/test-evm-node/package.json | 21 ++++++++++ packages/test-evm-node/tsconfig.json | 9 +++++ packages/ua-utils-evm-hardhat-test/README.md | 31 +------------- .../docker-compose.templates.yaml | 35 ---------------- .../hardhat.config.ts | 9 ++--- .../ua-utils-evm-hardhat-test/package.json | 3 +- turbo.json | 6 +++ yarn.lock | 4 +- 14 files changed, 164 insertions(+), 119 deletions(-) rename packages/ua-utils-evm-hardhat-test/docker-compose.yaml => docker-compose.yaml (60%) create mode 100644 packages/test-evm-node/hardhat.config.ts create mode 100644 packages/test-evm-node/package.json create mode 100644 packages/test-evm-node/tsconfig.json delete mode 100644 packages/ua-utils-evm-hardhat-test/docker-compose.templates.yaml diff --git a/Dockerfile b/Dockerfile index 098d4c005..06a98d820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,16 @@ FROM node:18.16.0 as base ENV YARN_CACHE_FOLDER=/tmp/yarn_cache +# We'll need a mock NPM_TOKEN to execute any yarn commands +ENV NPM_TOKEN= + # Update the system packages RUN apt-get update RUN apt-get install -y \ # Get the envsubst command (see below) - gettext-base + gettext-base \ + # Get the json utilities + jq # .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.- # / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ @@ -28,11 +33,6 @@ RUN apt-get install -y \ # `-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' FROM base as builder -# The name of the package we're building here -# -# e.g. @layerzerolabs/ua-utils-evm-hardhat -ARG PACKAGE - WORKDIR /app # We'll only use this turbo to prune the workspace, we don't care what version we use here @@ -47,7 +47,14 @@ COPY . . # # See more here https://turbo.build/repo/docs/reference/command-line-reference/prune # And here https://turbo.build/repo/docs/handbook/deploying-with-docker -RUN turbo prune $PACKAGE --docker +# +# There's an open issue on turbo github to support pruning +# without scope, in the meantime we'll use yarn workspaces info +# in combination with jq to get a full list of the workspace packages +# and prefix them with --scope +# +# See https://github.com/vercel/turbo/issues/4074 +RUN turbo prune $(yarn workspaces --silent info | jq -r 'keys | map("--scope " + .) | join(" ")') --docker # .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.- # / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ @@ -99,17 +106,14 @@ RUN \ # / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ # `-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' # -# Image that builds the package +# Image that prepares the project for development # # .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.- # / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ # `-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' -FROM dependencies as build +FROM dependencies as development -# The name of the package we're building here -# -# e.g. @layerzerolabs/ua-utils-evm-hardhat -ARG PACKAGE +ENV NPM_TOKEN= WORKDIR /app @@ -117,7 +121,4 @@ WORKDIR /app COPY tsconfig.json ./ # Now we grab the full source code from the builder step -COPY --from=builder /app/out/full/ . - -# And finally we build the package -RUN yarn build --filter=$PACKAGE... \ No newline at end of file +COPY --from=builder /app/out/full/ . \ No newline at end of file diff --git a/README.md b/README.md index a74236a5f..2317ec142 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,46 @@ This project is built using `turborepo`. The above commands are just aliases to yarn dev --filter=create-lz-oapp... ``` +### Running tests + +The tests are by default executed in a containerized environment that sets up two `hardhat` nodes accessible from within the environment: + +- `http://network-britney:8545` +- `http://network-vengaboys:8545` + +You can run the whole test suite within this environment by running: + +```bash +yarn test +``` + +#### Refining tested packages + +To only run a specific test suite, you can define `DOCKER_COMPOSE_RUN_TESTS_TURBO_ARGS` environment variable before running the tests. This variable will be passed to the underlying `turbo` command and can contain any arguments that this command understands, for example: + +```bash +# To only run tests for @layerzerolabs/ua-utils-evm-hardhat-test package +DOCKER_COMPOSE_RUN_TESTS_TURBO_ARGS=--filter=ua-utils-evm-hardhat-test yarn test +``` + +#### Rebuilding containers + +`docker compose` will by default reuse images for containers it has already built. If by any chance you are seeing code changes not being reflected in your test runs, you can force docker to rebuild the images by defining `DOCKER_COMPOSE_RUN_TESTS_ARGS` environment variable. This variable will be passed to the underlying `docker compose run` command and can contain any arguments that this command understands, for example: + +```bash +DOCKER_COMPOSE_RUN_TESTS_ARGS=--build yarn test +``` + +#### Container logs + +To monitor the container logs you'll need to run: + +```bash +docker compose logs -f +``` + +This allows you to monitor logs coming from e.g. the `hardhat` nodes + ### Troubleshooting #### Problems with committing diff --git a/docker-compose.templates.yaml b/docker-compose.templates.yaml index b662d5a91..7c51d4727 100644 --- a/docker-compose.templates.yaml +++ b/docker-compose.templates.yaml @@ -18,5 +18,34 @@ services: build: context: . args: - PACKAGE: + # In order not to embed the NPM_TOKEN to the image, we'll pass it as an build argument + # + # This works in conjunction with the adjacent Dockerfile that: + # + # - interpolates .npmrc with this argument + # - installs the depenendencies + # - deletes the interpolated file + # + # This all needs to be done within one layer so that there is no way of reverse engineering + # the Dockerfile and getting the NPM_TOKEN out NPM_TOKEN: $NPM_TOKEN + # We'll provide a single testing MNEMONIC for the project so that the test EVM nodes + # account are in sync with the hardhat accounts + environment: + - MNEMONIC='test test test test test test test test test test test test' + + # EVM node for testing purposes + evm-node: + extends: + service: project + command: ["npx", "turbo", "start", "--filter", "test-evm-node"] + healthcheck: + interval: 2s + retries: 10 + test: ["CMD", "curl", "-f", "http://0.0.0.0:8545/"] + + # Service that can build this package + tests: + extends: + service: project + working_dir: /app/packages/ua-utils-evm-hardhat-test diff --git a/packages/ua-utils-evm-hardhat-test/docker-compose.yaml b/docker-compose.yaml similarity index 60% rename from packages/ua-utils-evm-hardhat-test/docker-compose.yaml rename to docker-compose.yaml index cb522ca4f..eb940dfff 100644 --- a/packages/ua-utils-evm-hardhat-test/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,46 +18,28 @@ services: network-vengaboys: extends: file: docker-compose.templates.yaml - service: node + service: evm-node network-britney: extends: file: docker-compose.templates.yaml - service: node + service: evm-node # ~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~ # - # Bootstrap script that deploys all the networks + # The actual tests # # .oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo. - bootstrap: + tests: extends: file: docker-compose.templates.yaml - service: package + service: project depends_on: network-vengaboys: condition: service_healthy network-britney: condition: service_healthy - # Can't say I love the fact that the deploy scripts are inlined here - # but at least it's all in this file that bootstraps the whole environment - command: - - /bin/bash - - -c - - | - npx hardhat --network vengaboys deploy --reset - npx hardhat --network britney deploy --reset - - # ~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~^~ - # - # The actual tests - # - # .oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo. - tests: - extends: - file: docker-compose.templates.yaml - service: package - depends_on: - bootstrap: - condition: service_completed_successfully - command: ["npx", "hardhat", "test"] + volumes: + - ./node_modules/.cache/turbo:/app/node_modules/.cache/turbo + - ./packages:/app/packages + command: "npx turbo test $DOCKER_COMPOSE_RUN_TESTS_TURBO_ARGS" diff --git a/package.json b/package.json index a654bcff6..e99bf0337 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "dev": "npx turbo run dev", "lint": "npx turbo run lint", "prepare": "husky install", - "test": "npx turbo run test" + "test": "docker compose run --rm $DOCKER_COMPOSE_RUN_TESTS_ARGS tests" }, "lint-staged": { "**/*": "npx prettier --write --ignore-unknown" diff --git a/packages/test-evm-node/hardhat.config.ts b/packages/test-evm-node/hardhat.config.ts new file mode 100644 index 000000000..550fd3249 --- /dev/null +++ b/packages/test-evm-node/hardhat.config.ts @@ -0,0 +1,21 @@ +import assert from 'assert' +import { HardhatUserConfig } from 'hardhat/types' + +const MNEMONIC = process.env.MNEMONIC +assert(MNEMONIC, `Missing MNEMONIC environment variable`) + +/** + * This is a dummy hardhat config that enables us to test + * hardhat functionality without mocking too much + */ +const config: HardhatUserConfig = { + networks: { + hardhat: { + accounts: { + mnemonic: MNEMONIC, + }, + }, + }, +} + +export default config diff --git a/packages/test-evm-node/package.json b/packages/test-evm-node/package.json new file mode 100644 index 000000000..a7c4b2145 --- /dev/null +++ b/packages/test-evm-node/package.json @@ -0,0 +1,21 @@ +{ + "name": "@layerzerolabs/test-evm-node", + "version": "0.0.1", + "private": true, + "description": "EVM node for E2E testing purposes", + "repository": { + "type": "git", + "url": "git+https://github.com/LayerZero-Labs/lz-utils.git", + "directory": "packages/test-evm-node" + }, + "license": "MIT", + "scripts": { + "lint": "npx eslint '**/*.{js,ts,json}'", + "start": "npx hardhat node --hostname 0.0.0.0" + }, + "devDependencies": { + "hardhat": "^2.19.0", + "ts-node": "^10.9.1", + "typescript": "^5.2.2" + } +} \ No newline at end of file diff --git a/packages/test-evm-node/tsconfig.json b/packages/test-evm-node/tsconfig.json new file mode 100644 index 000000000..99636b366 --- /dev/null +++ b/packages/test-evm-node/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "exclude": ["dist", "node_modules"], + "include": ["src", "*.config.ts"], + "compilerOptions": { + "module": "commonjs", + "types": ["node"] + } +} diff --git a/packages/ua-utils-evm-hardhat-test/README.md b/packages/ua-utils-evm-hardhat-test/README.md index 55ba88d04..3e71b8c6a 100644 --- a/packages/ua-utils-evm-hardhat-test/README.md +++ b/packages/ua-utils-evm-hardhat-test/README.md @@ -8,33 +8,6 @@ ## Development -This package provides integration tests for `@layerzerolabs/utils-evm-hardhat` executed within a containerized setup. To run the test suite, simply run: +This package provides integration tests for `@layerzerolabs/utils-evm-hardhat` executed within a containerized setup. These tests are dependent on two networks that are bootstrapped in the root `docker-compose.yaml` and will only run when executed within that environment. -```bash -# You can use the alias command from this package directory -yarn test - -# Or use turbo and run from project root -yarn test --filter=utils-evm-hardhat-test - -# Or just use the actual test command from this package directory -docker compose run --rm tests -``` - -In case you're running the tests from the project root, it might sometimes be useful to rebuild the containers -(for example when adding/removing dependencies) and as a lazy developer, you might not be happy about `cd`ing to the package directory -and run the command from there. For that usecase the `$DOCKER_COMPOSE_RUN_TESTS_ARGS` environment variable has been added: - -```bash -# To rebuild the containers before running tests from the project root -DOCKER_COMPOSE_RUN_TESTS_ARGS=--build yarn test --filter=utils-evm-hardhat-test -``` - -To monitor the container logs, you'll need to `cd` into the package directory and run: - -```bash -docker compose logs -f -``` - -This allows you to see any logs coming from the containers, including detailed logs from the `hardhat` nodes -. +Please see the root [`README.md`](../../README.md) for more information. diff --git a/packages/ua-utils-evm-hardhat-test/docker-compose.templates.yaml b/packages/ua-utils-evm-hardhat-test/docker-compose.templates.yaml deleted file mode 100644 index eafb845dd..000000000 --- a/packages/ua-utils-evm-hardhat-test/docker-compose.templates.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.- -# / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ -# `-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' -# -# Reusable services for docker compose -# -# .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.-. .-.- -# / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ \ / / \ -# `-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' `-`-' -version: "3.9" - -services: - # Service that can build this package - package: - build: - args: - PACKAGE: "@layerzerolabs/ua-utils-evm-hardhat-test" - extends: - file: ../../docker-compose.templates.yaml - service: project - working_dir: /app/packages/ua-utils-evm-hardhat-test - volumes: - - ./:/app/packages/ua-utils-evm-hardhat-test - - # Hardhat node started on port 8545 - node: - extends: - service: package - expose: - - 8545 - command: ["npx", "hardhat", "node", "--hostname", "0.0.0.0", "--no-deploy"] - healthcheck: - interval: 2s - retries: 10 - test: ["CMD", "curl", "-f", "http://0.0.0.0:8545/"] diff --git a/packages/ua-utils-evm-hardhat-test/hardhat.config.ts b/packages/ua-utils-evm-hardhat-test/hardhat.config.ts index b4dd9ad33..2cb8c52e3 100644 --- a/packages/ua-utils-evm-hardhat-test/hardhat.config.ts +++ b/packages/ua-utils-evm-hardhat-test/hardhat.config.ts @@ -1,10 +1,12 @@ import 'hardhat-deploy' import 'hardhat-deploy-ethers' +import assert from 'assert' import { withLayerZeroArtifacts } from '@layerzerolabs/utils-evm-hardhat' import { EndpointId } from '@layerzerolabs/lz-definitions' import { HardhatUserConfig } from 'hardhat/types' -const MNEMONIC = 'test test test test test test test test test test test test' +const MNEMONIC = process.env.MNEMONIC +assert(MNEMONIC, `Missing MNEMONIC environment variable`) /** * This is a dummy hardhat config that enables us to test @@ -15,11 +17,6 @@ const config: HardhatUserConfig = { version: '0.8.19', }, networks: { - hardhat: { - accounts: { - mnemonic: MNEMONIC, - }, - }, vengaboys: { eid: EndpointId.ETHEREUM_MAINNET, url: 'http://network-vengaboys:8545', diff --git a/packages/ua-utils-evm-hardhat-test/package.json b/packages/ua-utils-evm-hardhat-test/package.json index 897db9c9c..7ef69914a 100644 --- a/packages/ua-utils-evm-hardhat-test/package.json +++ b/packages/ua-utils-evm-hardhat-test/package.json @@ -11,7 +11,8 @@ "license": "MIT", "scripts": { "lint": "npx eslint '**/*.{js,ts,json}'", - "test": "docker compose run --rm $DOCKER_COMPOSE_RUN_TESTS_ARGS tests" + "pretest": "npx hardhat --network vengaboys deploy --reset && npx hardhat --network britney deploy --reset", + "test": "npx hardhat test" }, "devDependencies": { "@ethersproject/abstract-signer": "^5.7.0", diff --git a/turbo.json b/turbo.json index 69101a56b..782f32509 100644 --- a/turbo.json +++ b/turbo.json @@ -21,7 +21,13 @@ "lint": { "cache": false }, + "start": { + "outputs": [], + "dependsOn": ["build"], + "persistent": true + }, "test": { + "cache": false, "outputs": [], "dependsOn": ["^build"] } diff --git a/yarn.lock b/yarn.lock index edfe2da02..1809a93f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11392,7 +11392,7 @@ turbo-darwin-64@1.5.6: turbo-darwin-arm64@1.5.6: version "1.5.6" - resolved "https://registry.npmjs.org/turbo-darwin-arm64/-/turbo-darwin-arm64-1.5.6.tgz" + resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.5.6.tgz#bed2bba126d53d7bbfd45f95e239fc43fd5bccbf" integrity sha512-c/aXgW9JuXT2bJSKf01pdSDQKnrdcdj3WFKmKiVldb9We6eqFzI0fLHBK97k5LM/OesmRMfCMQ2Cv2DU8RqBAA== turbo-linux-64@1.5.6: @@ -11417,7 +11417,7 @@ turbo-windows-arm64@1.5.6: turbo@1.5.6: version "1.5.6" - resolved "https://registry.npmjs.org/turbo/-/turbo-1.5.6.tgz" + resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.5.6.tgz#7dac8db14b2452afa45c57c050ff19989d4ab074" integrity sha512-xJO/fhiMo4lI62iGR9OgUfJTC9tnnuoMwNC52IfvvBDEPlA8RWGMS8SFpDVG9bNCXvVRrtUTNJXMe6pJWBiOTA== optionalDependencies: turbo-darwin-64 "1.5.6"