Merge pull request #904 from polywrap/prealpha-dev

Prep 0.0.1-prealpha.84

.eslintrc.js

@@ -2,6 +2,7 @@ module.exports = {
   root: true,
   ignorePatterns: [
     "**/w3/**/*.*",
+    "**/infra-modules/**/*.*",
     "**/build/**/*.*",
     "**/__tests__/**/*.*",
     "**/node_modules/**/*.*",

.github/workflows/js-ci.yaml

@@ -191,7 +191,7 @@ jobs:
 
 # NOTE: Mac & Windows CI is not currently possible, since GitHub Actions does not
 # use "Docker Enterprise". Docker Enterprise is required when running linux based
-# containers, which is what we run within our toolchain for test-envs and build-envs.
+# containers, which is what we run within our toolchain for infra and build-images.
 # In order to enable this, we can move to Circle CI, which supports Docker Enterprise.
 
 #  JS-CI-Mac:

.github/workflows/rs-ci.yaml

@@ -95,5 +95,5 @@ jobs:
 
 # NOTE: Mac & Windows CI is not currently possible, since GitHub Actions does not
 # use "Docker Enterprise". Docker Enterprise is required when running linux based
-# containers, which is what we run within our toolchain for test-envs and build-envs.
+# containers, which is what we run within our toolchain for infra and build-images.
 # In order to enable this, we can move to Circle CI, which supports Docker Enterprise.

CHANGELOG.md

@@ -1,3 +1,12 @@
+# Web3API 0.0.1-prealpha.84
+## Features
+* [PR-328](https://github.com/polywrap/monorepo/pull/328) `@web3api/infra`: A modular infrastructure pipeline has been added to the CLI, available via the `w3 infra ...` command. This command allows for custom infra modules to be defined and combined, able to support any 3rd party services your development requires. This command supersedes the old `w3 test-env ...` command.
+* [PR-898](https://github.com/polywrap/monorepo/pull/898) `@web3api/cli`: The `wasm/rust` default build image has been updated to include the `wasm-snip` utility, which helps remove dead code from the wasm modules in a post-processing step. This has reduce the average Rust-based wasm module's size by ~85%.
+* [PR-885](https://github.com/polywrap/monorepo/pull/885) `@web3api/test-env-js`: A `buildApi` helper function has been added.
+
+## Breaking Changes
+* [PR-328](https://github.com/polywrap/monorepo/pull/328) `@web3api/cli`: The `w3 test-env ...` command has been removed and replaced by the `w3 infra ...` command.
+
 # Web3API 0.0.1-prealpha.83
 ## Features
 * [PR-870](https://github.com/polywrap/monorepo/pull/870) `@web3api/cli`: The `web3api.deploy.yaml` manifest file now supports the use of environment variables.

VERSION

@@ -1 +1 @@
-0.0.1-prealpha.83
+0.0.1-prealpha.84

package.json

@@ -10,7 +10,7 @@
   "workspaces": {
     "packages": [
       "./packages/apis/**",
-      "./packages/cli/**",
+      "./packages/cli",
       "./packages/core-interfaces/**",
       "./packages/js/**",
       "./packages/manifest-schemas/**",
@@ -36,7 +36,7 @@
     "lint:ci": "yarn lint",
     "test": "lerna run test --no-private --ignore @web3api/client-js --concurrency 1 && lerna run test --scope @web3api/client-js",
     "test:ci": "lerna run test:ci --no-private --ignore @web3api/client-js --concurrency 1 && lerna run test:ci --scope @web3api/client-js",
-    "test:core": "lerna run test:ci --no-private --ignore @web3api/*-plugin-js --ignore @web3api/cli* --ignore @web3api/client-js && lerna run test:ci --no-private --scope @web3api/templates",
+    "test:core": "lerna run test:ci --no-private --ignore @web3api/*-plugin-js --ignore @web3api/cli* --ignore @web3api/client-js --concurrency 1",
     "test:plugins": "lerna run test:ci --no-private --scope @web3api/*-plugin-js --concurrency 1",
     "test:client": "lerna run test:ci --no-private --scope @web3api/client-js --concurrency 1",
     "test:cli": "lerna run test:ci --no-private --scope @web3api/cli --concurrency 1",

packages/cli/copyfiles.ts

@@ -0,0 +1,21 @@
+import fse from "fs-extra";
+
+const source = process.argv[2];
+const dest = process.argv[3];
+
+const ignore = [
+  ".env",
+  "node_modules",
+  ".idea",
+  ".log",
+  ".vscode",
+  ".w3",
+  ".rs.bk",
+  ".lock",
+];
+
+fse.copySync(source, dest, {
+  filter: function (path) {
+    return !ignore.some((ignorePath) => path.endsWith(ignorePath));
+  },
+});

packages/cli/lang/en.json

@@ -13,6 +13,44 @@
   "commands_build_options_t": "Use the development server's ENS instance",
   "commands_build_options_w": "Automatically rebuild when changes are made (default: false)",
   "commands_build_options_v": "Verbose output (default: false)",
+  "commands_infra_description": "Manage infrastructure for your Web3API",
+  "commands_infra_actions_subtitle": "Infra allows you to execute the following commands:",
+  "commands_infra_options_options": "options",
+  "commands_infra_options_manifest": "manifest",
+  "commands_infra_options_i_node": "node",
+  "commands_infra_options_h": "Show usage information",
+  "commands_infra_options_m": "Use only specified modules",
+  "commands_infra_options_v": "Verbose output (default: false)",
+  "commands_infra_options_d": "Run in detached mode",
+  "commands_infra_moduleName": "module-name",
+  "commands_infra_manifestPath": "manifest-path",
+  "commands_infra_manifestPathDescription": "Infra Manifest path",
+  "commands_infra_command_up": "Start Web3API infrastructure",
+  "commands_infra_command_down": "Stop Web3API infrastructure",
+  "commands_infra_command_vars": "Show Web3API infrastructure's required .env variables",
+  "commands_infra_command_config": "Validate and display Web3API infrastructure's bundled docker-compose manifest",
+  "commands_infra_error_never": "This should never happen...",
+  "commands_infra_error_noCommand": "No command given",
+  "commands_infra_error_unrecognizedCommand": "Unrecognized command: {command}",
+  "commands_infra_error_noModulesMatch": "{modules} did not match any module names",
+  "commands_infra_error_noModulesDeclared": "No modules declared",
+  "commands_infra_modulesUsed_text": "Using infra modules",
+  "commands_infra_options_command": "command",
+  "commands_infra_options_start": "Startup the test env",
+  "commands_infra_options_stop": "Shutdown the test env",
+  "commands_infra_options_env_vars": "Show environment variables",
+  "commands_infra_shutdown_error": "Failed to shutdown test environment",
+  "commands_infra_shutdown_text": "Shutting down test environment...",
+  "commands_infra_shutdown_warning": "Warning shutting down test environment",
+  "commands_infra_startup_error": "Failed to start test environment",
+  "commands_infra_startup_text": "Starting test environment...",
+  "commands_infra_startup_warning": "Warning starting test environment",
+  "commands_infra_vars_text": "Extracting environment variables",
+  "commands_infra_vars_warning": "Warning extracting environment variables",
+  "commands_infra_vars_error": "Error extracting environment variables",
+  "commands_infra_manifest_text": "Aggregating manifest",
+  "commands_infra_manifest_warning": "Warning aggregating manifest",
+  "commands_infra_manifest_error": "Error aggregating manifest",
   "commands_build_uriViewers": "URI Viewers",
   "commands_deploy_description": "Deploys/Publishes a Web3API",
   "commands_deploy_options_options": "options",
@@ -140,7 +178,7 @@
   "commands_query_error_noApi": "API needs to be initialized",
   "commands_query_error_readFail": "Failed to read query {query}",
   "commands_query_error_noRecipeScriptFound": "Recipe script not found at path: {path}",
-  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 test-env up'",
+  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 infra up --modules=eth-ens-ipfs'",
   "commands_testEnv_description": "Manage a test environment for Web3API",
   "commands_testEnv_error_never": "This should never happen...",
   "commands_testEnv_error_noCommand": "No command given",
@@ -226,5 +264,6 @@
   "lib_watcher_alreadyWatching": "Watcher session is already in progress. Directory: {dir}",
   "lib_wasm_rust_invalidModule": "Module paths must point to Cargo.toml files. Found: {path}",
   "lib_docker_invalidImageId": "Invalid docker image ID returned: {imageId}",
-  "lib_docker_noInstall": "Docker executable not found in PATH"
+  "lib_docker_noInstall": "Docker executable not found in PATH",
+  "lib_infra_unrecognizedModule": "Unrecognized modules: {modules}. Default modules: {defaultModules}"
 }

packages/cli/lang/es.json

@@ -15,6 +15,44 @@
   "commands_build_options_w": "Automatically rebuild when changes are made (default: false)",
   "commands_build_options_v": "Verbose output (default: false)",
   "commands_build_uriViewers": "URI Viewers",
+  "commands_infra_description": "Manage infrastructure for your Web3API",
+  "commands_infra_actions_subtitle": "Infra allows you to execute the following commands:",
+  "commands_infra_options_options": "options",
+  "commands_infra_options_manifest": "manifest",
+  "commands_infra_options_i_node": "node",
+  "commands_infra_options_h": "Show usage information",
+  "commands_infra_options_m": "Use only specified modules",
+  "commands_infra_options_v": "Verbose output (default: false)",
+  "commands_infra_options_d": "Run in detached mode",
+  "commands_infra_moduleName": "module-name",
+  "commands_infra_manifestPath": "manifest-path",
+  "commands_infra_manifestPathDescription": "Infra Manifest path",
+  "commands_infra_command_up": "Start Web3API infrastructure",
+  "commands_infra_command_down": "Stop Web3API infrastructure",
+  "commands_infra_command_vars": "Show Web3API infrastructure's required .env variables",
+  "commands_infra_command_config": "Validate and display Web3API infrastructure's bundled docker-compose manifest",
+  "commands_infra_error_never": "This should never happen...",
+  "commands_infra_error_noCommand": "No command given",
+  "commands_infra_error_unrecognizedCommand": "Unrecognized command: {command}",
+  "commands_infra_error_noModulesMatch": "{modules} did not match any module names",
+  "commands_infra_error_noModulesDeclared": "No modules declared",
+  "commands_infra_modulesUsed_text": "Using infra modules",
+  "commands_infra_options_command": "command",
+  "commands_infra_options_start": "Startup the test env",
+  "commands_infra_options_stop": "Shutdown the test env",
+  "commands_infra_options_env_vars": "Show environment variables",
+  "commands_infra_shutdown_error": "Failed to shutdown test environment",
+  "commands_infra_shutdown_text": "Shutting down test environment...",
+  "commands_infra_shutdown_warning": "Warning shutting down test environment",
+  "commands_infra_startup_error": "Failed to start test environment",
+  "commands_infra_startup_text": "Starting test environment...",
+  "commands_infra_startup_warning": "Warning starting test environment",
+  "commands_infra_vars_text": "Extracting environment variables",
+  "commands_infra_vars_warning": "Warning extracting environment variables",
+  "commands_infra_vars_error": "Error extracting environment variables",
+  "commands_infra_manifest_text": "Aggregating manifest",
+  "commands_infra_manifest_warning": "Warning aggregating manifest",
+  "commands_infra_manifest_error": "Error aggregating manifest",
   "commands_deploy_description": "Deploys/Publishes a Web3API",
   "commands_deploy_options_options": "options",
   "commands_deploy_options_o_path": "path",
@@ -141,7 +179,7 @@
   "commands_query_error_noApi": "API needs to be initialized",
   "commands_query_error_readFail": "Failed to read query {query}",
   "commands_query_error_noRecipeScriptFound": "Recipe script not found at path: {path}",
-  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 test-env up'",
+  "commands_query_error_noTestEnvFound": "w3 test-env not found, please run 'w3 infra up --modules=eth-ens-ipfs'",
   "commands_testEnv_description": "Manage a test environment for Web3API",
   "commands_testEnv_error_never": "This should never happen...",
   "commands_testEnv_error_noCommand": "No command given",
@@ -227,5 +265,6 @@
   "lib_watcher_alreadyWatching": "Watcher session is already in progress. Directory: {dir}",
   "lib_wasm_rust_invalidModule": "Module paths must point to Cargo.toml files. Found: {path}",
   "lib_docker_invalidImageId": "Invalid docker image ID returned: {imageId}",
-  "lib_docker_noInstall": "Docker executable not found in PATH"
+  "lib_docker_noInstall": "Docker executable not found in PATH",
+  "lib_infra_unrecognizedModule": "Unrecognized modules: {modules}. Default modules: {defaultModules}"
 }

packages/cli/package.json

@@ -16,11 +16,12 @@
     "w3": "bin/w3"
   },
   "scripts": {
-    "build": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:envs && yarn build:deployers",
-    "build:envs": "copyfiles ./src/lib/build-envs/**/**/* ./build/lib/build-envs/ -u 3",
-    "build:deployers": "copyfiles ./src/lib/deployers/**/web3api.deploy.ext.json ./build/lib/deployers -u 3",
+    "build": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:build-images && yarn build:deploy-modules && yarn build:infra-modules",
+    "build:build-images": "copyfiles ./src/lib/defaults/build-images/**/**/* ./build/lib/defaults/build-images/ -u 4",
+    "build:deploy-modules": "copyfiles ./src/lib/defaults/deploy-modules/**/web3api.deploy.ext.json ./build/lib/defaults/deploy-modules -u 4",
+    "build:infra-modules": "ts-node ./copyfiles ./src/lib/defaults/infra-modules ./build/lib/defaults/infra-modules",
     "prebuild": "ts-node ./scripts/generateIntlTypes.ts",
-    "build:fast": "rimraf ./build && tsc --project tsconfig.build.json && yarn build:envs",
+    "build:fast": "rimraf ./build && tsc --project tsconfig.build.json",
     "lint": "eslint --color -c ../../.eslintrc.js .",
     "test": "cross-env TEST=true jest --passWithNoTests --runInBand --verbose",
     "test:ci": "cross-env TEST=true jest --passWithNoTests --runInBand --verbose",
@@ -30,11 +31,11 @@
     "colors": "1.4.0"
   },
   "dependencies": {
-    "@ensdomains/ensjs": "2.0.1",
     "@formatjs/intl": "1.8.2",
+    "@ethersproject/wallet": "5.6.2",
+    "@ethersproject/providers": "5.6.8",
     "@web3api/asyncify-js": "0.0.1-prealpha.83",
     "@web3api/client-js": "0.0.1-prealpha.83",
-    "@web3api/client-test-env": "0.0.1-prealpha.83",
     "@web3api/core-js": "0.0.1-prealpha.83",
     "@web3api/ens-plugin-js": "0.0.1-prealpha.83",
     "@web3api/ethereum-plugin-js": "0.0.1-prealpha.83",
@@ -43,18 +44,19 @@
     "@web3api/schema-bind": "0.0.1-prealpha.83",
     "@web3api/schema-compose": "0.0.1-prealpha.83",
     "@web3api/schema-parse": "0.0.1-prealpha.83",
+    "@web3api/test-env-js": "0.0.1-prealpha.83",
     "assemblyscript": "0.19.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "chokidar": "3.5.1",
     "commander": "9.2.0",
     "content-hash": "2.5.2",
     "copyfiles": "2.4.1",
-    "ethers": "5.0.7",
     "fs-extra": "9.0.1",
     "gluegun": "4.6.1",
     "graphql-tag": "2.11.0",
     "ipfs-http-client": "48.1.3",
+    "docker-compose": "0.23.17",
     "js-yaml": "3.14.0",
     "jsonschema": "1.4.0",
     "mustache": "4.0.1",
@@ -72,7 +74,6 @@
     "@types/node": "12.12.26",
     "@types/prettier": "2.6.0",
     "@types/rimraf": "3.0.0",
-    "@web3api/test-env-js": "0.0.1-prealpha.83",
     "cross-env": "7.0.3",
     "dir-compare": "3.3.0",
     "eslint-plugin-formatjs": "2.12.7",

packages/cli/src/__tests__/e2e/build.spec.ts

@@ -60,16 +60,16 @@ describe("e2e tests for build command", () => {
     expect(output).toBe("")
   });
 
-  test("Adds uuid-v4 suffix to build-env image if no build manifest specified", async () => {
+  test("Adds uuid-v4 suffix to build image if no build manifest specified", async () => {
     const projectRoot = getTestCaseDir(0);
     const project = new Web3ApiProject({
-      rootCacheDir: projectRoot,
+      rootDir: projectRoot,
       web3apiManifestPath: path.join(projectRoot, "web3api.yaml")
     });
 
-    await project.cacheDefaultBuildManifestFiles();
+    await project.cacheDefaultBuildImage();
 
-    const cacheBuildEnvPath = path.join(projectRoot, ".w3/web3api/build/env")
+    const cacheBuildEnvPath = path.join(projectRoot, ".w3/web3api/build/image")
     const cachedBuildManifest = await loadBuildManifest(
       path.join(cacheBuildEnvPath, "web3api.build.yaml")
     );