From b74150eb1a2054f66c8bb801502923acea2f65bf Mon Sep 17 00:00:00 2001 From: Natalie Weizenbaum Date: Mon, 28 Oct 2024 17:58:29 -0700 Subject: [PATCH] Add package metadata files and get tests running --- .eslintignore | 3 + .eslintrc | 14 +++++ .github/dependabot.yml | 10 ++++ .github/workflows/ci.yml | 94 +++++++++++++++++++++++++++++ .gitignore | 15 +++++ .prettierrc.js | 3 + CODE_OF_CONDUCT.md | 10 ++++ CONTRIBUTING.md | 38 ++++++++++++ LICENSE | 20 +++++++ README.md | 124 +++++++++++++++++++++++++++++++++++++++ jest.config.ts | 7 +++ lib/index.test.ts | 4 +- lib/index.ts | 2 - package.json | 41 +++++++++++++ tsconfig.build.json | 8 +++ tsconfig.json | 15 +++++ 16 files changed, 404 insertions(+), 4 deletions(-) create mode 100644 .eslintignore create mode 100644 .eslintrc create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .prettierrc.js create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 jest.config.ts create mode 100644 package.json create mode 100644 tsconfig.build.json create mode 100644 tsconfig.json diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..f1073fd --- /dev/null +++ b/.eslintignore @@ -0,0 +1,3 @@ +build/ +dist/ +**/*.js diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..3b5b912 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,14 @@ +{ + "extends": "./node_modules/gts/", + "rules": { + "@typescript-eslint/explicit-function-return-type": [ + "error", + {"allowExpressions": true} + ], + "func-style": ["error", "declaration"], + "prefer-const": ["error", {"destructuring": "all"}], + // It would be nice to sort import declaration order as well, but that's not + // autofixable and it's not worth the effort of handling manually. + "sort-imports": ["error", {"ignoreDeclarationSort": true}], + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..782a0ad --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..979d981 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI + +defaults: + run: {shell: bash} + +env: + PROTOC_VERSION: 3.x + +on: + push: + branches: [main, feature.*] + tags: ['**'] + pull_request: + +jobs: + static_analysis: + name: Static analysis + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + check-latest: true + - run: npm install + - run: npm run check + + tests: + name: 'Tests | Node ${{ matrix.node-version }} | ${{ matrix.os }}' + runs-on: ${{ matrix.os }}-latest + + strategy: + matrix: + os: [ubuntu, macos, windows] + node-version: ['lts/*', 'lts/-1', 'lts/-2'] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + check-latest: true + - run: npm install + - run: npm run test + + deploy: + name: Deploy + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/') && github.repository == 'sass/sync-message-channel'" + needs: [static_analysis, tests, sass_spec] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + check-latest: true + registry-url: 'https://registry.npmjs.org' + - run: npm install + - run: npm publish + env: + NODE_AUTH_TOKEN: '${{ secrets.NPM_TOKEN }}' + + typedoc: + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/') && github.repository == 'sass/sync-message-channel'" + needs: [deploy] + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + permissions: + pages: write + id-token: write + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + check-latest: true + registry-url: 'https://registry.npmjs.org' + - run: npm install + - run: npm run doc + + - name: Upload static files as artifact + uses: actions/upload-pages-artifact@v3 + with: {path: docs} + + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f345cd1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.DS_Store +build +dist +node_modules +npm-debug.log* +package-lock.json + +# Editors +.idea +.vscode +*.njsproj +*.ntvs* +*.sln +*.suo +*.sw? diff --git a/.prettierrc.js b/.prettierrc.js new file mode 100644 index 0000000..c5166c2 --- /dev/null +++ b/.prettierrc.js @@ -0,0 +1,3 @@ +module.exports = { + ...require('gts/.prettierrc.json'), +}; diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..dfc4c84 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,10 @@ +Sass is more than a technology; Sass is driven by the community of individuals +that power its development and use every day. As a community, we want to embrace +the very differences that have made our collaboration so powerful, and work +together to provide the best environment for learning, growing, and sharing of +ideas. It is imperative that we keep Sass a fun, welcoming, challenging, and +fair place to play. + +[The full community guidelines can be found on the Sass website.][link] + +[link]: http://sass-lang.com/community-guidelines diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..96c5c3b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,38 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +* [Contributor License Agreement](#contributor-license-agreement) +* [Code Reviews](#code-reviews) +* [Large Language Models](#large-language-models) + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code Reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Large Language Models + +Do not submit any code or prose written or modified by large language models or +"artificial intelligence" such as GitHub Copilot or ChatGPT to this project. +These tools produce code that looks plausible, which means that not only is it +likely to contain bugs those bugs are likely to be difficult to notice on +review. In addition, because these models were trained indiscriminately and +non-consensually on open-source code with a variety of licenses, it's not +obvious that we have the moral or legal right to redistribute code they +generate. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..63ff5ce --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2024, Google LLC + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b6c2b9d --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# `sync-message-port` + +This package exposes a utility class that encapsulates the ability to send and +receive messages with arbitrary structure across Node.js worker boundaries. It +can be used as the building block for synchronous versions of APIs that are +traditionally only available asynchronously in the Node.js ecosystem by running +the asynchronous APIs in a worker and accessing their results synchronously from +the main thread. + +See [the `sync-process` package] for an example of `sync-message-channel` in +action. + +[the `sync-process` package]: https://github.com/sass/sync-process + +## Usage + +1. Use `SyncMessagePort.createChanenl()` to create a message channel that's set + up to be compatible iwth `SyncMessagePort`s. A normal `MessageChannel` won't + work! + +2. You can send this `MessageChannel`'s ports across worker boundaries just like + any other `MessagePort`. Send one to the worker you want to communicate with + synchronously. + +3. Once you're ready to start sending and receiving messages, wrap *both* ports + in `new SyncMessagePort()`, even if one is only ever going to be sending + messages and not receiving them. + +4. Use `SyncMessagePort.postMessage()` to send messages and + `SyncMessagePort.receiveMessage()` to receive them synchronously. + +```js +import {Worker} from 'node:worker_threads'; +import {SyncMessagePort} from 'sync-message-channel'; +// or +// const {SyncMessagePort} = require('sync-message-port'); + +// Channels must be created using this function. A MessageChannel created by +// hand won't work. +const channel = SyncMessagePort.createChannel(); +const localPort = new SyncMessagePort(channel.port1); + +const worker = new Worker(` + import {workerData} = require('node:worker_threads'); + import {SyncMessagePort} from 'sync-message-channel'; + + const remotePort = new SyncMessagePort(workerData.port); + + setTimeout(() => { + remotePort.postMessage("hello from worker!"); + }, 2000); +`, { + workerData: {port: channel.port2}, + transferList: [channel.port2], + eval: true, +}); + +// Note that because workers report errors asynchronously, this won't report an +// error if the worker fails to load because the main thread will be +// synchronously waiting for its first message. +worker.on('error', console.error); + +console.log(localPort.receiveMessage()); +``` + +## Why synchrony? + +Although JavaScript in general and Node.js in particular are typically designed +to embrace asynchrony, there are a number of reasons why a synchronous API may +be preferable or even necessary. + +### No a/synchronous polymorphism + +Although `async`/`await` and the `Promise` API has substantially improved the +usability of writing asynchronous code in JavaScript, it doesn't address one +core issue: there's no way to write code that's *polymorphic* over asynchrony. +Put in simpler terms, there's no language-level way to write a complex function +that takes a callback and to run that functions synchronously if the callback is +synchronous and asynchronously otherwise. The only option is to write the +function twice. + +This poses a real, practical problem when interacting with libraries. Suppose +you have a library that takes a callback option—for example, an HTML +sanitization library that takes a callback to determine how to handle a given +``. The library doesn't need to do any IO itself, so it's written +synchronously. But what if your callback wants to make an HTTP request to +determine how to handle a tag? You're stuck unless you can make that request +synchronous. This library makes that possible. + +### Performance considerations + +Asynchrony is generally more performant in situations where there's a large +amount of concurrent IO happening. But when performance is CPU-bound, it's often +substantially worse due to the overhead of bouncing back and forth between the +event loop and user code. + +As a real-world example, the Sass compiler API supports both synchronous and +asynchronous code paths to work around the polymorphism problem described above. +The logic of these paths is exactly the same—the only difference is that the +asynchronous path's functions all return `Promise`s instead of synchronous +values. Compiling with the asynchronous path often takes 2-3x longer than with +the synchronous path. This means that being able to run plugins synchronously +can provide a substantial overall performance gain, even if the plugins +themselves lose the benefit of concurrency. + +## How does it work? + +This uses [`Atomics`] and [`SharedArrayBuffer`] under the covers to signal +across threads when messages are available, and +[`worker_threads.receiveMessageOnPort()`] to actually retrieve messages. + +[`Atomics`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Atomics +[`SharedArrayBuffer`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer +[`Worker.receiveMessageOnPort()`]: https://nodejs.org/api/worker_threads.html#workerreceivemessageonportport + +### Can I use this in a browser? + +Unfortunately, no. Browsers don't support any equivalent of +`worker_threads.receiveMessageOnPort()`, even within worker threads. You could +make a similar package that can transmit only binary data (or data that can be +encoded as binary) using only `SharedArrayBuffer`, but that's outside the scope +of this package. + +Disclaimer: this is not an official Google product. diff --git a/jest.config.ts b/jest.config.ts new file mode 100644 index 0000000..a2539f4 --- /dev/null +++ b/jest.config.ts @@ -0,0 +1,7 @@ +const config = { + roots: ['/lib/'], + preset: 'ts-jest', + testEnvironment: 'node', +}; + +export default config; diff --git a/lib/index.test.ts b/lib/index.test.ts index 4c1e167..4099827 100644 --- a/lib/index.test.ts +++ b/lib/index.test.ts @@ -6,7 +6,7 @@ import * as fs from 'fs'; import * as p from 'path'; import {MessagePort, Worker} from 'worker_threads'; -import {SyncMessagePort} from './sync-message-port'; +import {SyncMessagePort} from './index'; describe('SyncMessagePort', () => { describe('sends a message', () => { @@ -152,7 +152,7 @@ function spawnWorker(source: string, port: MessagePort): Worker { file, ` const {SyncMessagePort} = require(${JSON.stringify( - p.join(p.dirname(__filename), 'sync-message-port') + p.join(p.dirname(__filename), 'index') )}); const {workerData} = require('worker_threads'); diff --git a/lib/index.ts b/lib/index.ts index 7970244..1c17978 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -11,8 +11,6 @@ import { receiveMessageOnPort, } from 'worker_threads'; -// TODO(nex3): Make this its own package. - /** * An enum of possible states for the shared buffer that two `SyncMessagePort`s * use to communicate. diff --git a/package.json b/package.json new file mode 100644 index 0000000..6820ac1 --- /dev/null +++ b/package.json @@ -0,0 +1,41 @@ +{ + "name": "sync-message-channel", + "version": "1.0.0", + "description": "A Node.js communication port that can pass messages synchronously between workers", + "repository": "sass/sync-message-channel", + "author": "Google Inc.", + "license": "MIT", + "exports": { + "types": "./dist/types/index.d.ts", + "default": "./dist/lib/index.js" + }, + "main": "dist/lib/index.js", + "types": "dist/types/index.d.ts", + "files": [ + "dist/**/*" + ], + "engines": { + "node": ">=16.0.0" + }, + "scripts": { + "init": "ts-node ./tool/init.ts", + "check": "npm-run-all check:gts check:tsc", + "check:gts": "gts check", + "check:tsc": "tsc --noEmit", + "clean": "gts clean", + "compile": "tsc -p tsconfig.build.json", + "fix": "gts fix", + "test": "jest" + }, + "devDependencies": { + "@types/jest": "^29.4.0", + "@types/node": "^22.0.0", + "gts": "^5.0.0", + "jest": "^29.4.1", + "minipass": "7.1.2", + "npm-run-all": "^4.1.5", + "ts-jest": "^29.0.5", + "ts-node": "^10.2.1", + "typescript": "^5.0.2" + } +} diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 0000000..5ca9532 --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "jest.config.js", + "lib/src/vendor/dart-sass/**", + "**/*.test.ts" + ] +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b5d630b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "./node_modules/gts/tsconfig-google.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "lib": ["DOM"] + }, + "include": [ + "*.ts", + "bin/*.ts", + "lib/**/*.ts", + "tool/**/*.ts", + "test/**/*.ts" + ] +}