From e525dbcc07f73d3d2a45a577fdc3bcf338b67a04 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 17 Aug 2023 15:48:17 +1000 Subject: [PATCH 001/149] chore: copied over websocket domain from Polykey/websocket --- src/ErrorWebSocket.ts | 44 +++ src/WebSocketClient.ts | 257 ++++++++++++ src/WebSocketServer.ts | 290 +++++++++++++- src/WebSocketStream.ts | 359 ++++++++++++++++- src/errors.ts | 138 +++++++ src/events.ts | 43 ++ src/ids/index.ts | 383 ++++++++++++++++++ src/ids/types.ts | 131 +++++++ src/types.ts | 18 +- src/utils.ts | 0 src/utils/errors.ts | 31 ++ src/utils/index.ts | 3 + src/utils/sysexits.ts | 91 +++++ src/utils/utils.ts | 30 ++ tests/WebSocket.test.ts | 850 ++++++++++++++++++++++++++++++++++++++++ tests/testClient.ts | 31 ++ tests/testServer.ts | 36 ++ 17 files changed, 2732 insertions(+), 3 deletions(-) create mode 100644 src/ErrorWebSocket.ts create mode 100644 src/errors.ts create mode 100644 src/events.ts create mode 100644 src/ids/index.ts create mode 100644 src/ids/types.ts delete mode 100644 src/utils.ts create mode 100644 src/utils/errors.ts create mode 100644 src/utils/index.ts create mode 100644 src/utils/sysexits.ts create mode 100644 src/utils/utils.ts create mode 100644 tests/WebSocket.test.ts create mode 100644 tests/testClient.ts create mode 100644 tests/testServer.ts diff --git a/src/ErrorWebSocket.ts b/src/ErrorWebSocket.ts new file mode 100644 index 00000000..1a53e540 --- /dev/null +++ b/src/ErrorWebSocket.ts @@ -0,0 +1,44 @@ +import type { Class } from '@matrixai/errors'; +import { AbstractError } from '@matrixai/errors'; +import sysexits from './utils/sysexits'; + +class ErrorWebSocket extends AbstractError { + static description: string = 'Polykey error'; + exitCode: number = sysexits.GENERAL; + + public static fromJSON>( + this: T, + json: any, + ): InstanceType { + if ( + typeof json !== 'object' || + json.type !== this.name || + typeof json.data !== 'object' || + typeof json.data.message !== 'string' || + isNaN(Date.parse(json.data.timestamp)) || + typeof json.data.description !== 'string' || + typeof json.data.data !== 'object' || + typeof json.data.exitCode !== 'number' || + ('stack' in json.data && typeof json.data.stack !== 'string') + ) { + throw new TypeError(`Cannot decode JSON to ${this.name}`); + } + const e = new this(json.data.message, { + timestamp: new Date(json.data.timestamp), + data: json.data.data, + cause: json.data.cause, + }); + e.exitCode = json.data.exitCode; + e.stack = json.data.stack; + return e; + } + + public toJSON(): any { + const json = super.toJSON(); + json.data.description = this.description; + json.data.exitCode = this.exitCode; + return json; + } +} + +export default ErrorWebSocket; diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 3cf9665a..c370cf38 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,5 +1,262 @@ +import type { TLSSocket } from 'tls'; +import type { ContextTimed } from '@matrixai/contexts'; +import type { NodeId, NodeIdEncoded } from '../ids'; +import { createDestroy } from '@matrixai/async-init'; +import Logger from '@matrixai/logger'; +import WebSocket from 'ws'; +import { Validator } from 'ip-num'; +import { Timer } from '@matrixai/timer'; +import WebSocketStream from './WebSocketStream'; +import * as webSocketUtils from './utils'; +import * as webSocketErrors from './errors'; +import { promise } from './utils'; +import * as nodesUtils from '../nodes/utils'; + +interface WebSocketClient extends createDestroy.CreateDestroy {} +@createDestroy.CreateDestroy() class WebSocketClient { + /** + * @param obj + * @param obj.host - Target host address to connect to + * @param obj.port - Target port to connect to + * @param obj.expectedNodeIds - Expected NodeIds you are trying to connect to. Will validate the cert chain of the + * sever. If none of these NodeIDs are found the connection will be rejected. + * @param obj.connectionTimeoutTime - Timeout time used when attempting the connection. + * Default is Infinity milliseconds. + * @param obj.pingIntervalTime - Time between pings for checking connection health and keep alive. + * Default is 1,000 milliseconds. + * @param obj.pingTimeoutTimeTime - Time before connection is cleaned up after no ping responses. + * Default is 10,000 milliseconds. + * @param obj.logger + */ + static async createWebSocketClient({ + host, + port, + expectedNodeIds, + connectionTimeoutTime = Infinity, + pingIntervalTime = 1_000, + pingTimeoutTimeTime = 10_000, + logger = new Logger(this.name), + }: { + host: string; + port: number; + expectedNodeIds: Array; + connectionTimeoutTime?: number; + pingIntervalTime?: number; + pingTimeoutTimeTime?: number; + logger?: Logger; + }): Promise { + logger.info(`Creating ${this.name}`); + const clientClient = new this( + logger, + host, + port, + expectedNodeIds, + connectionTimeoutTime, + pingIntervalTime, + pingTimeoutTimeTime, + ); + logger.info(`Created ${this.name}`); + return clientClient; + } + + protected host: string; + protected activeConnections: Set = new Set(); + + constructor( + protected logger: Logger, + host: string, + protected port: number, + protected expectedNodeIds: Array, + protected connectionTimeoutTime: number, + protected pingIntervalTime: number, + protected pingTimeoutTimeTime: number, + ) { + if (Validator.isValidIPv4String(host)[0]) { + this.host = host; + } else if (Validator.isValidIPv6String(host)[0]) { + this.host = `[${host}]`; + } else { + throw new webSocketErrors.ErrorClientInvalidHost(); + } + } + + public async destroy(force: boolean = false) { + this.logger.info(`Destroying ${this.constructor.name}`); + if (force) { + for (const activeConnection of this.activeConnections) { + activeConnection.cancel( + new webSocketErrors.ErrorClientEndingConnections( + 'Destroying WebSocketClient', + ), + ); + } + } + for (const activeConnection of this.activeConnections) { + // Ignore errors here, we only care that it finishes + await activeConnection.endedProm.catch(() => {}); + } + this.logger.info(`Destroyed ${this.constructor.name}`); + } + + @createDestroy.ready(new webSocketErrors.ErrorClientDestroyed()) + public async stopConnections() { + for (const activeConnection of this.activeConnections) { + activeConnection.cancel( + new webSocketErrors.ErrorClientEndingConnections(), + ); + } + for (const activeConnection of this.activeConnections) { + // Ignore errors here, we only care that it finished + await activeConnection.endedProm.catch(() => {}); + } + } + + @createDestroy.ready(new webSocketErrors.ErrorClientDestroyed()) + public async startConnection( + ctx: Partial = {}, + ): Promise { + // Setting up abort/cancellation logic + const abortRaceProm = promise(); + // Ignore unhandled rejection + abortRaceProm.p.catch(() => {}); + const timer = + ctx.timer ?? + new Timer({ + delay: this.connectionTimeoutTime, + }); + void timer.then( + () => { + abortRaceProm.rejectP( + new webSocketErrors.ErrorClientConnectionTimedOut(), + ); + }, + () => {}, // Ignore cancellation errors + ); + const { signal } = ctx; + let abortHandler: () => void | undefined; + if (signal != null) { + abortHandler = () => { + abortRaceProm.rejectP(signal.reason); + }; + if (signal.aborted) abortHandler(); + else signal.addEventListener('abort', abortHandler); + } + const cleanUp = () => { + // Cancel timer if it was internally created + if (ctx.timer == null) timer.cancel(); + signal?.removeEventListener('abort', abortHandler); + }; + const address = `wss://${this.host}:${this.port}`; + this.logger.info(`Connecting to ${address}`); + const connectProm = promise(); + const authenticateProm = promise<{ + nodeId: NodeIdEncoded; + localHost: string; + localPort: number; + remoteHost: string; + remotePort: number; + }>(); + const ws = new WebSocket(address, { + rejectUnauthorized: false, + }); + // Handle connection failure + const openErrorHandler = (e) => { + connectProm.rejectP( + new webSocketErrors.ErrorClientConnectionFailed(undefined, { + cause: e, + }), + ); + }; + ws.once('error', openErrorHandler); + // Authenticate server's certificates + ws.once('upgrade', async (request) => { + const tlsSocket = request.socket as TLSSocket; + const peerCert = tlsSocket.getPeerCertificate(true); + try { + const nodeId = await webSocketUtils.verifyServerCertificateChain( + this.expectedNodeIds, + webSocketUtils.detailedToCertChain(peerCert), + ); + authenticateProm.resolveP({ + nodeId: nodesUtils.encodeNodeId(nodeId), + localHost: request.connection.localAddress ?? '', + localPort: request.connection.localPort ?? 0, + remoteHost: request.connection.remoteAddress ?? '', + remotePort: request.connection.remotePort ?? 0, + }); + } catch (e) { + authenticateProm.rejectP(e); + } + }); + ws.once('open', () => { + this.logger.info('starting connection'); + connectProm.resolveP(); + }); + const earlyCloseProm = promise(); + ws.once('close', () => { + earlyCloseProm.resolveP(); + }); + // There are 3 resolve conditions here. + // 1. Connection established and authenticated + // 2. connection error or authentication failure + // 3. connection timed out + try { + await Promise.race([ + abortRaceProm.p, + await Promise.all([authenticateProm.p, connectProm.p]), + ]); + } catch (e) { + // Clean up + // unregister handlers + ws.removeAllListeners('error'); + ws.removeAllListeners('upgrade'); + ws.removeAllListeners('open'); + // Close the ws if it's open at this stage + ws.terminate(); + // Ensure the connection is removed from the active connection set before + // returning. + await earlyCloseProm.p; + throw e; + } finally { + cleanUp(); + // Cleaning up connection error + ws.removeEventListener('error', openErrorHandler); + } + // Constructing the `ReadableWritablePair`, the lifecycle is handed off to + // the webSocketStream at this point. + const webSocketStreamClient = new WebSocketStream( + ws, + this.pingIntervalTime, + this.pingTimeoutTimeTime, + { + ...(await authenticateProm.p), + }, + this.logger.getChild(WebSocketStream.name), + ); + const abortStream = () => { + webSocketStreamClient.cancel( + new webSocketErrors.ErrorClientStreamAborted(undefined, { + cause: signal?.reason, + }), + ); + }; + // Setting up activeStream map lifecycle + this.activeConnections.add(webSocketStreamClient); + void webSocketStreamClient.endedProm + // Ignore errors, we only care that it finished + .catch(() => {}) + .finally(() => { + this.activeConnections.delete(webSocketStreamClient); + signal?.removeEventListener('abort', abortStream); + }); + // Abort connection on signal + if (signal?.aborted === true) abortStream(); + else signal?.addEventListener('abort', abortStream); + return webSocketStreamClient; + } } +// This is the internal implementation of the client's stream pair. export default WebSocketClient; diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index a43d93e5..100955a5 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -1,5 +1,293 @@ -class WebSocketServer { +import type { TLSConfig } from '../network/types'; +import type { IncomingMessage, ServerResponse } from 'http'; +import type tls from 'tls'; +import https from 'https'; +import { startStop, status } from '@matrixai/async-init'; +import Logger from '@matrixai/logger'; +import * as ws from 'ws'; +import WebSocketStream from './WebSocketStream'; +import * as webSocketErrors from './errors'; +import * as webSocketEvents from './events'; +import { never, promise } from './utils'; +type ConnectionCallback = (streamPair: WebSocketStream) => void; + +/** + * Events: + * - start + * - stop + * - connection + */ +interface WebSocketServer extends startStop.StartStop {} +@startStop.StartStop() +class WebSocketServer extends EventTarget { + /** + * @param obj + * @param obj.connectionCallback - + * @param obj.tlsConfig - TLSConfig containing the private key and cert chain used for TLS. + * @param obj.host - Listen address to bind to. + * @param obj.port - Listen port to bind to. + * @param obj.maxIdleTimeout - Timeout time for when the connection is cleaned up after no activity. + * Default is 120 seconds. + * @param obj.pingIntervalTime - Time between pings for checking connection health and keep alive. + * Default is 1,000 milliseconds. + * @param obj.pingTimeoutTimeTime - Time before connection is cleaned up after no ping responses. + * Default is 10,000 milliseconds. + * @param obj.logger + */ + static async createWebSocketServer({ + connectionCallback, + tlsConfig, + host, + port, + maxIdleTimeout = 120, + pingIntervalTime = 1_000, + pingTimeoutTimeTime = 10_000, + logger = new Logger(this.name), + }: { + connectionCallback: ConnectionCallback; + tlsConfig: TLSConfig; + host?: string; + port?: number; + maxIdleTimeout?: number; + pingIntervalTime?: number; + pingTimeoutTimeTime?: number; + logger?: Logger; + }) { + logger.info(`Creating ${this.name}`); + const wsServer = new this( + logger, + maxIdleTimeout, + pingIntervalTime, + pingTimeoutTimeTime, + ); + await wsServer.start({ + connectionCallback, + tlsConfig, + host, + port, + }); + logger.info(`Created ${this.name}`); + return wsServer; + } + + protected server: https.Server; + protected webSocketServer: ws.WebSocketServer; + protected _port: number; + protected _host: string; + protected connectionEventHandler: ( + event: webSocketEvents.ConnectionEvent, + ) => void; + protected activeSockets: Set = new Set(); + + /** + * + * @param logger + * @param maxIdleTimeout + * @param pingIntervalTime + * @param pingTimeoutTimeTime + */ + constructor( + protected logger: Logger, + protected maxIdleTimeout: number | undefined, + protected pingIntervalTime: number, + protected pingTimeoutTimeTime: number, + ) { + super(); + } + + public async start({ + tlsConfig, + host, + port = 0, + connectionCallback, + }: { + tlsConfig: TLSConfig; + host?: string; + port?: number; + connectionCallback?: ConnectionCallback; + }): Promise { + this.logger.info(`Starting ${this.constructor.name}`); + if (connectionCallback != null) { + this.connectionEventHandler = ( + event: webSocketEvents.ConnectionEvent, + ) => { + connectionCallback(event.detail.webSocketStream); + }; + this.addEventListener('connection', this.connectionEventHandler); + } + this.server = https.createServer({ + key: tlsConfig.keyPrivatePem, + cert: tlsConfig.certChainPem, + }); + this.webSocketServer = new ws.WebSocketServer({ + server: this.server, + }); + + this.webSocketServer.on('connection', this.connectionHandler); + this.webSocketServer.on('close', this.closeHandler); + this.server.on('close', this.closeHandler); + this.webSocketServer.on('error', this.errorHandler); + this.server.on('error', this.errorHandler); + this.server.on('request', this.requestHandler); + + const listenProm = promise(); + this.server.listen(port ?? 0, host, listenProm.resolveP); + await listenProm.p; + const address = this.server.address(); + if (address == null || typeof address === 'string') never(); + this._port = address.port; + this.logger.debug(`Listening on port ${this._port}`); + this._host = address.address ?? '127.0.0.1'; + this.dispatchEvent( + new webSocketEvents.StartEvent({ + detail: { + host: this._host, + port: this._port, + }, + }), + ); + this.logger.info(`Started ${this.constructor.name}`); + } + + public async stop(force: boolean = false): Promise { + this.logger.info(`Stopping ${this.constructor.name}`); + // Shutting down active websockets + if (force) { + for (const webSocketStream of this.activeSockets) { + webSocketStream.cancel(); + } + } + // Wait for all active websockets to close + for (const webSocketStream of this.activeSockets) { + // Ignore errors, we only care that it finished + webSocketStream.endedProm.catch(() => {}); + } + // Close the server by closing the underlying socket + const wssCloseProm = promise(); + this.webSocketServer.close((e) => { + if (e == null || e.message === 'The server is not running') { + wssCloseProm.resolveP(); + } else { + wssCloseProm.rejectP(e); + } + }); + await wssCloseProm.p; + const serverCloseProm = promise(); + this.server.close((e) => { + if (e == null || e.message === 'Server is not running.') { + serverCloseProm.resolveP(); + } else { + serverCloseProm.rejectP(e); + } + }); + await serverCloseProm.p; + // Removing handlers + if (this.connectionEventHandler != null) { + this.removeEventListener('connection', this.connectionEventHandler); + } + + this.webSocketServer.off('connection', this.connectionHandler); + this.webSocketServer.off('close', this.closeHandler); + this.server.off('close', this.closeHandler); + this.webSocketServer.off('error', this.errorHandler); + this.server.off('error', this.errorHandler); + this.server.on('request', this.requestHandler); + + this.dispatchEvent(new webSocketEvents.StopEvent()); + this.logger.info(`Stopped ${this.constructor.name}`); + } + + @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) + public getPort(): number { + return this._port; + } + + @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) + public getHost(): string { + return this._host; + } + + @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) + public setTlsConfig(tlsConfig: TLSConfig): void { + const tlsServer = this.server as tls.Server; + tlsServer.setSecureContext({ + key: tlsConfig.keyPrivatePem, + cert: tlsConfig.certChainPem, + }); + } + + /** + * Handles the creation of the `ReadableWritablePair` and provides it to the + * StreamPair handler. + */ + protected connectionHandler = ( + webSocket: ws.WebSocket, + request: IncomingMessage, + ) => { + const connection = request.connection; + const webSocketStream = new WebSocketStream( + webSocket, + this.pingIntervalTime, + this.pingTimeoutTimeTime, + { + localHost: connection.localAddress ?? '', + localPort: connection.localPort ?? 0, + remoteHost: connection.remoteAddress ?? '', + remotePort: connection.remotePort ?? 0, + }, + this.logger.getChild(WebSocketStream.name), + ); + // Adding socket to the active sockets map + this.activeSockets.add(webSocketStream); + void webSocketStream.endedProm + // Ignore errors, we only care that it finished + .catch(() => {}) + .finally(() => { + this.activeSockets.delete(webSocketStream); + }); + + // There is not nodeId or certs for the client, and we can't get the remote + // port from the `uWebsocket` library. + this.dispatchEvent( + new webSocketEvents.ConnectionEvent({ + detail: { + webSocketStream, + }, + }), + ); + }; + + /** + * Used to trigger stopping if the underlying server fails + */ + protected closeHandler = async () => { + if (this[status] == null || this[status] === 'stopping') { + this.logger.debug('close event but already stopping'); + return; + } + this.logger.debug('close event, forcing stop'); + await this.stop(true); + }; + + /** + * Used to propagate error conditions + */ + protected errorHandler = (e: Error) => { + this.logger.error(e); + }; + + /** + * Will tell any normal HTTP request to upgrade + */ + protected requestHandler = (_req, res: ServerResponse) => { + res + .writeHead(426, '426 Upgrade Required', { + connection: 'Upgrade', + upgrade: 'websocket', + }) + .end('426 Upgrade Required'); + }; } export default WebSocketServer; diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 31a4066f..cbe9c1f4 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,5 +1,362 @@ -class WebSocketStream { +import type { ReadableWritablePair } from 'stream/web'; +import type { + ReadableStreamController, + WritableStreamDefaultController, +} from 'stream/web'; +import type * as ws from 'ws'; +import type Logger from '@matrixai/logger'; +import type { NodeIdEncoded } from './ids/types'; +import { WritableStream, ReadableStream } from 'stream/web'; +import * as webSocketErrors from './errors'; +import * as utilsErrors from './utils/errors'; +import { promise } from './utils'; +class WebSocketStream implements ReadableWritablePair { + public readable: ReadableStream; + public writable: WritableStream; + + protected _readableEnded = false; + protected _readableEndedProm = promise(); + protected _writableEnded = false; + protected _writableEndedProm = promise(); + protected _webSocketEnded = false; + protected _webSocketEndedProm = promise(); + protected _endedProm: Promise; + + protected readableController: + | ReadableStreamController + | undefined; + protected writableController: WritableStreamDefaultController | undefined; + + constructor( + protected ws: ws.WebSocket, + pingInterval: number, + pingTimeoutTime: number, + protected metadata: { + nodeId?: NodeIdEncoded; + localHost: string; + localPort: number; + remoteHost: string; + remotePort: number; + }, + logger: Logger, + ) { + // Sanitise promises so they don't result in unhandled rejections + this._readableEndedProm.p.catch(() => {}); + this._writableEndedProm.p.catch(() => {}); + this._webSocketEndedProm.p.catch(() => {}); + // Creating the endedPromise + this._endedProm = Promise.allSettled([ + this._readableEndedProm.p, + this._writableEndedProm.p, + this._webSocketEndedProm.p, + ]).then((result) => { + if ( + result[0].status === 'rejected' || + result[1].status === 'rejected' || + result[2].status === 'rejected' + ) { + // Throw a compound error + throw AggregateError(result, 'stream failed'); + } + // Otherwise return nothing + }); + // Ignore errors if it's never used + this._endedProm.catch(() => {}); + + logger.info('WS opened'); + const readableLogger = logger.getChild('readable'); + const writableLogger = logger.getChild('writable'); + // Setting up the readable stream + this.readable = new ReadableStream( + { + start: (controller) => { + readableLogger.debug('Starting'); + this.readableController = controller; + const messageHandler = (data: ws.RawData, isBinary: boolean) => { + if (!isBinary || data instanceof Array) { + controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); + return; + } + const message = data as Buffer; + readableLogger.debug(`Received ${message.toString()}`); + if (message.length === 0) { + readableLogger.debug('Null message received'); + ws.removeListener('message', messageHandler); + if (!this._readableEnded) { + readableLogger.debug('Closing'); + this.signalReadableEnd(); + controller.close(); + } + if (this._writableEnded) { + logger.debug('Closing socket'); + ws.close(); + } + return; + } + if (this._readableEnded) { + return; + } + controller.enqueue(message); + if (controller.desiredSize == null) { + controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); + return; + } + if (controller.desiredSize < 0) { + readableLogger.debug('Applying readable backpressure'); + ws.pause(); + } + }; + readableLogger.debug('Registering socket message handler'); + ws.on('message', messageHandler); + ws.once('close', (code, reason) => { + logger.info('Socket closed'); + ws.removeListener('message', messageHandler); + if (!this._readableEnded) { + readableLogger.debug( + `Closed early, ${code}, ${reason.toString()}`, + ); + const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); + this.signalReadableEnd(e); + controller.error(e); + } + }); + ws.once('error', (e) => { + if (!this._readableEnded) { + readableLogger.error(e); + this.signalReadableEnd(e); + controller.error(e); + } + }); + }, + cancel: (reason) => { + readableLogger.debug('Cancelled'); + this.signalReadableEnd(reason); + if (this._writableEnded) { + readableLogger.debug('Closing socket'); + this.signalWritableEnd(reason); + ws.close(); + } + }, + pull: () => { + readableLogger.debug('Releasing backpressure'); + ws.resume(); + }, + }, + { highWaterMark: 1 }, + ); + this.writable = new WritableStream( + { + start: (controller) => { + this.writableController = controller; + writableLogger.info('Starting'); + ws.once('error', (e) => { + if (!this._writableEnded) { + writableLogger.error(e); + this.signalWritableEnd(e); + controller.error(e); + } + }); + ws.once('close', (code, reason) => { + if (!this._writableEnded) { + writableLogger.debug( + `Closed early, ${code}, ${reason.toString()}`, + ); + const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); + this.signalWritableEnd(e); + controller.error(e); + } + }); + }, + close: async () => { + writableLogger.debug('Closing, sending null message'); + const sendProm = promise(); + ws.send(Buffer.from([]), (err) => { + if (err == null) sendProm.resolveP(); + else sendProm.rejectP(err); + }); + await sendProm.p; + this.signalWritableEnd(); + if (this._readableEnded) { + writableLogger.debug('Closing socket'); + ws.close(); + } + }, + abort: (reason) => { + writableLogger.debug('Aborted'); + this.signalWritableEnd(reason); + if (this._readableEnded) { + writableLogger.debug('Closing socket'); + ws.close(4000, `Aborting connection with ${reason.message}`); + } + }, + write: async (chunk, controller) => { + if (this._writableEnded) return; + writableLogger.debug(`Sending ${chunk?.toString()}`); + const wait = promise(); + ws.send(chunk, (e) => { + if (e != null && !this._writableEnded) { + // Opting to debug message here and not log an error, sending + // failure is common if we send before the close event. + writableLogger.debug('failed to send'); + const err = new webSocketErrors.ErrorClientConnectionEndedEarly( + undefined, + { + cause: e, + }, + ); + this.signalWritableEnd(err); + controller.error(err); + } + wait.resolveP(); + }); + await wait.p; + }, + }, + { highWaterMark: 1 }, + ); + + // Setting up heartbeat + const pingTimer = setInterval(() => { + ws.ping(); + }, pingInterval); + const pingTimeoutTimeTimer = setTimeout(() => { + logger.debug('Ping timed out'); + ws.close(4002, 'Timed out'); + }, pingTimeoutTime); + const pingHandler = () => { + logger.debug('Received ping'); + ws.pong(); + }; + const pongHandler = () => { + logger.debug('Received pong'); + pingTimeoutTimeTimer.refresh(); + }; + ws.on('ping', pingHandler); + ws.on('pong', pongHandler); + ws.once('close', (code, reason) => { + ws.off('ping', pingHandler); + ws.off('pong', pongHandler); + logger.debug('WebSocket closed'); + const err = + code !== 1000 + ? new webSocketErrors.ErrorClientConnectionEndedEarly( + `ended with code ${code}, ${reason.toString()}`, + ) + : undefined; + this.signalWebSocketEnd(err); + logger.debug('Cleaning up timers'); + // Clean up timers + clearTimeout(pingTimer); + clearTimeout(pingTimeoutTimeTimer); + }); + } + + get readableEnded() { + return this._readableEnded; + } + + /** + * Resolves when the readable has ended and rejects with any errors. + */ + get readableEndedProm() { + return this._readableEndedProm.p; + } + + get writableEnded() { + return this._writableEnded; + } + + /** + * Resolves when the writable has ended and rejects with any errors. + */ + get writableEndedProm() { + return this._writableEndedProm.p; + } + + get webSocketEnded() { + return this._webSocketEnded; + } + + /** + * Resolves when the webSocket has ended and rejects with any errors. + */ + get webSocketEndedProm() { + return this._webSocketEndedProm.p; + } + + get ended() { + return this._readableEnded && this._writableEnded; + } + + /** + * Resolves when the stream has fully closed + */ + get endedProm(): Promise { + return this._endedProm; + } + + get meta() { + // Spreading to avoid modifying the data + return { + ...this.metadata, + }; + } + + /** + * Forces the active stream to end early + */ + public cancel(reason?: any): void { + // Default error + const err = reason ?? new webSocketErrors.ErrorClientConnectionEndedEarly(); + // Close the streams with the given error, + if (!this._readableEnded) { + this.readableController?.error(err); + this.signalReadableEnd(err); + } + if (!this._writableEnded) { + this.writableController?.error(err); + this.signalWritableEnd(err); + } + // Then close the websocket + if (!this._webSocketEnded) { + this.ws.close(4000, 'Ending connection'); + this.signalWebSocketEnd(err); + } + } + + /** + * Signals the end of the ReadableStream. to be used with the extended class + * to track the streams state. + */ + protected signalReadableEnd(reason?: any) { + if (this._readableEnded) return; + this._readableEnded = true; + if (reason == null) this._readableEndedProm.resolveP(); + else this._readableEndedProm.rejectP(reason); + } + + /** + * Signals the end of the WritableStream. to be used with the extended class + * to track the streams state. + */ + protected signalWritableEnd(reason?: any) { + if (this._writableEnded) return; + this._writableEnded = true; + if (reason == null) this._writableEndedProm.resolveP(); + else this._writableEndedProm.rejectP(reason); + } + + /** + * Signals the end of the WebSocket. to be used with the extended class + * to track the streams state. + */ + protected signalWebSocketEnd(reason?: any) { + if (this._webSocketEnded) return; + this._webSocketEnded = true; + if (reason == null) this._webSocketEndedProm.resolveP(); + else this._webSocketEndedProm.rejectP(reason); + } } export default WebSocketStream; diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 00000000..26ed1cf9 --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,138 @@ +import ErrorWebSocket from './ErrorWebSocket'; +import { sysexits } from './utils'; + +class ErrorWebSocketClient extends ErrorWebSocket {} + +class ErrorClientDestroyed extends ErrorWebSocketClient { + static description = 'ClientClient has been destroyed'; + exitCode = sysexits.USAGE; +} + +class ErrorClientInvalidHost extends ErrorWebSocketClient { + static description = 'Host must be a valid IPv4 or IPv6 address string'; + exitCode = sysexits.USAGE; +} + +class ErrorClientConnectionFailed extends ErrorWebSocketClient { + static description = 'Failed to establish connection to server'; + exitCode = sysexits.UNAVAILABLE; +} + +class ErrorClientConnectionTimedOut extends ErrorWebSocketClient { + static description = 'Connection timed out'; + exitCode = sysexits.UNAVAILABLE; +} + +class ErrorClientConnectionEndedEarly extends ErrorWebSocketClient { + static description = 'Connection ended before stream ended'; + exitCode = sysexits.UNAVAILABLE; +} + +class ErrorClientStreamAborted extends ErrorWebSocketClient { + static description = 'Stream was ended early with an abort signal'; + exitCode = sysexits.USAGE; +} + +class ErrorClientEndingConnections extends ErrorWebSocketClient { + static description = 'WebSocketClient is ending active connections'; + exitCode = sysexits.USAGE; +} + +class ErrorWebSocketServer extends ErrorWebSocket {} + +class ErrorWebSocketServerNotRunning extends ErrorWebSocketServer { + static description = 'WebSocketServer is not running'; + exitCode = sysexits.USAGE; +} + +class ErrorServerPortUnavailable extends ErrorWebSocketServer { + static description = 'Failed to bind a free port'; + exitCode = sysexits.UNAVAILABLE; +} + +class ErrorServerSendFailed extends ErrorWebSocketServer { + static description = 'Failed to send message'; + exitCode = sysexits.UNAVAILABLE; +} + +class ErrorServerReadableBufferLimit extends ErrorWebSocketServer { + static description = 'Readable buffer is full, messages received too quickly'; + exitCode = sysexits.USAGE; +} + +class ErrorServerConnectionEndedEarly extends ErrorWebSocketServer { + static description = 'Connection ended before stream ended'; + exitCode = sysexits.UNAVAILABLE; +} + +/** + * Used for certificate verification + */ +class ErrorCertChain extends ErrorWebSocket {} + +class ErrorCertChainEmpty extends ErrorCertChain { + static description = 'Certificate chain is empty'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainUnclaimed extends ErrorCertChain { + static description = 'The target node id is not claimed by any certificate'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainBroken extends ErrorCertChain { + static description = 'The signature chain is broken'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainDateInvalid extends ErrorCertChain { + static description = 'Certificate in the chain is expired'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainNameInvalid extends ErrorCertChain { + static description = 'Certificate is missing the common name'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainKeyInvalid extends ErrorCertChain { + static description = 'Certificate public key does not generate the Node ID'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorCertChainSignatureInvalid extends ErrorCertChain { + static description = 'Certificate self-signed signature is invalid'; + exitCode = sysexits.PROTOCOL; +} + +class ErrorConnectionNodesEmpty extends ErrorWebSocket { + static description = 'Nodes list to verify against was empty'; + exitCode = sysexits.USAGE; +} + +export { + ErrorWebSocketClient, + ErrorClientDestroyed, + ErrorClientInvalidHost, + ErrorClientConnectionFailed, + ErrorClientConnectionTimedOut, + ErrorClientConnectionEndedEarly, + ErrorClientStreamAborted, + ErrorClientEndingConnections, + ErrorWebSocketServer, + ErrorWebSocketServerNotRunning, + ErrorServerPortUnavailable, + ErrorServerSendFailed, + ErrorServerReadableBufferLimit, + ErrorServerConnectionEndedEarly, + ErrorCertChainEmpty, + ErrorCertChainUnclaimed, + ErrorCertChainBroken, + ErrorCertChainDateInvalid, + ErrorCertChainNameInvalid, + ErrorCertChainKeyInvalid, + ErrorCertChainSignatureInvalid, + ErrorConnectionNodesEmpty, +}; + +export * from './utils/errors'; diff --git a/src/events.ts b/src/events.ts new file mode 100644 index 00000000..fc70e825 --- /dev/null +++ b/src/events.ts @@ -0,0 +1,43 @@ +import type WebSocketStream from './WebSocketStream'; + +class StartEvent extends Event { + public detail: { + host: string; + port: number; + }; + constructor( + options: EventInit & { + detail: { + host: string; + port: number; + }; + }, + ) { + super('start', options); + this.detail = options.detail; + } +} + +class StopEvent extends Event { + constructor(options?: EventInit) { + super('stop', options); + } +} + +class ConnectionEvent extends Event { + public detail: { + webSocketStream: WebSocketStream; + }; + constructor( + options: EventInit & { + detail: { + webSocketStream: WebSocketStream; + }; + }, + ) { + super('connection', options); + this.detail = options.detail; + } +} + +export { StartEvent, StopEvent, ConnectionEvent }; diff --git a/src/ids/index.ts b/src/ids/index.ts new file mode 100644 index 00000000..75375c02 --- /dev/null +++ b/src/ids/index.ts @@ -0,0 +1,383 @@ +import type { + PermissionId, + CertId, + CertIdEncoded, + NodeId, + NodeIdString, + NodeIdEncoded, + VaultId, + VaultIdEncoded, + TaskId, + TaskIdEncoded, + ClaimId, + ClaimIdEncoded, + ProviderIdentityId, + ProviderIdentityIdEncoded, + NotificationId, + NotificationIdEncoded, + GestaltId, + GestaltIdEncoded, + GestaltLinkId, +} from './types'; +import { IdInternal, IdSortable, IdRandom } from '@matrixai/id'; +import * as keysUtilsRandom from '../keys/utils/random'; + +function createPermIdGenerator(): () => PermissionId { + const generator = new IdRandom({ + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +/** + * Creates a NodeId generator. + * This does not use `IdRandom` because it is not a UUID4. + * Instead this just generates random 32 bytes. + */ +function createNodeIdGenerator(): () => NodeId { + return () => { + return IdInternal.fromBuffer(keysUtilsRandom.getRandomBytes(32)); + }; +} + +/** + * Encodes the NodeId as a `base32hex` string + */ +function encodeNodeId(nodeId: NodeId): NodeIdEncoded { + return nodeId.toMultibase('base32hex') as NodeIdEncoded; +} + +/** + * Decodes an encoded NodeId string into a NodeId + */ +function decodeNodeId(nodeIdEncoded: unknown): NodeId | undefined { + if (typeof nodeIdEncoded !== 'string') { + return; + } + const nodeId = IdInternal.fromMultibase(nodeIdEncoded); + if (nodeId == null) { + return; + } + // All NodeIds are 32 bytes long + // The NodeGraph requires a fixed size for Node Ids + if (nodeId.length !== 32) { + return; + } + return nodeId; +} + +function encodeNodeIdString(nodeId: NodeId): NodeIdString { + return nodeId.toString() as NodeIdString; +} + +function decodeNodeIdString(nodeIdString: unknown): NodeId | undefined { + if (typeof nodeIdString !== 'string') { + return; + } + const nodeId = IdInternal.fromString(nodeIdString); + if (nodeId == null) { + return; + } + // All NodeIds are 32 bytes long + // The NodeGraph requires a fixed size for Node Ids + if (nodeId.length !== 32) { + return; + } + return nodeId; +} + +/** + * Generates CertId + */ +function createCertIdGenerator(lastCertId?: CertId): () => CertId { + const generator = new IdSortable({ + lastId: lastCertId, + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +/** + * Encodes `CertId` to `CertIdEncoded` + */ +function encodeCertId(certId: CertId): CertIdEncoded { + return certId.toBuffer().toString('hex') as CertIdEncoded; +} + +/** + * Decodes `CertIdEncoded` to `CertId` + */ +function decodeCertId(certIdEncoded: unknown): CertId | undefined { + if (typeof certIdEncoded !== 'string') { + return; + } + const certIdBuffer = Buffer.from(certIdEncoded, 'hex'); + const certId = IdInternal.fromBuffer(certIdBuffer); + if (certId == null) { + return; + } + // All `CertId` are 16 bytes long + if (certId.length !== 16) { + return; + } + return certId; +} + +function createVaultIdGenerator(): () => VaultId { + const generator = new IdRandom({ + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +function encodeVaultId(vaultId: VaultId): VaultIdEncoded { + return vaultId.toMultibase('base58btc') as VaultIdEncoded; +} + +function decodeVaultId(vaultIdEncoded: unknown): VaultId | undefined { + if (typeof vaultIdEncoded !== 'string') return; + const vaultId = IdInternal.fromMultibase(vaultIdEncoded); + if (vaultId == null) return; + // All VaultIds are 16 bytes long + if (vaultId.length !== 16) return; + return vaultId; +} + +/** + * Generates TaskId + * TaskIds are lexicographically sortable 128 bit IDs + * They are strictly monotonic and unique with respect to the `nodeId` + * When the `NodeId` changes, make sure to regenerate this generator + */ +function createTaskIdGenerator(lastTaskId?: TaskId) { + const generator = new IdSortable({ + lastId: lastTaskId, + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +/** + * Encodes the TaskId as a `base32hex` string + */ +function encodeTaskId(taskId: TaskId): TaskIdEncoded { + return taskId.toMultibase('base32hex') as TaskIdEncoded; +} + +/** + * Decodes an encoded TaskId string into a TaskId + */ +function decodeTaskId(taskIdEncoded: unknown): TaskId | undefined { + if (typeof taskIdEncoded !== 'string') { + return; + } + const taskId = IdInternal.fromMultibase(taskIdEncoded); + if (taskId == null) { + return; + } + // All TaskIds are 16 bytes long + if (taskId.length !== 16) { + return; + } + return taskId; +} + +/** + * Generator for `ClaimId` + * Make sure the `nodeId` is set to this node's own `NodeId` + */ +function createClaimIdGenerator(nodeId: NodeId, lastClaimId?: ClaimId) { + const generator = new IdSortable({ + nodeId, + lastId: lastClaimId, + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +function encodeClaimId(claimId: ClaimId): ClaimIdEncoded { + return claimId.toMultibase('base32hex') as ClaimIdEncoded; +} + +function decodeClaimId(claimIdEncoded: unknown): ClaimId | undefined { + if (typeof claimIdEncoded !== 'string') { + return; + } + const claimId = IdInternal.fromMultibase(claimIdEncoded); + if (claimId == null) { + return; + } + return claimId; +} + +function encodeProviderIdentityId( + providerIdentityId: ProviderIdentityId, +): ProviderIdentityIdEncoded { + return JSON.stringify(providerIdentityId) as ProviderIdentityIdEncoded; +} + +function decodeProviderIdentityId( + providerIdentityIdEncoded: unknown, +): ProviderIdentityId | undefined { + if (typeof providerIdentityIdEncoded !== 'string') { + return; + } + let providerIdentityId: unknown; + try { + providerIdentityId = JSON.parse(providerIdentityIdEncoded); + } catch { + return; + } + if ( + !Array.isArray(providerIdentityId) || + providerIdentityId.length !== 2 || + typeof providerIdentityId[0] !== 'string' || + typeof providerIdentityId[1] !== 'string' + ) { + return; + } + return providerIdentityId as ProviderIdentityId; +} + +function encodeGestaltId(gestaltId: GestaltId): GestaltIdEncoded { + switch (gestaltId[0]) { + case 'node': + return encodeGestaltNodeId(gestaltId); + case 'identity': + return encodeGestaltIdentityId(gestaltId); + } +} + +function encodeGestaltNodeId( + gestaltNodeId: ['node', NodeId], +): GestaltIdEncoded { + return (gestaltNodeId[0] + + '-' + + encodeNodeId(gestaltNodeId[1])) as GestaltIdEncoded; +} + +function encodeGestaltIdentityId( + gestaltIdentityId: ['identity', ProviderIdentityId], +): GestaltIdEncoded { + return (gestaltIdentityId[0] + + '-' + + encodeProviderIdentityId(gestaltIdentityId[1])) as GestaltIdEncoded; +} + +function decodeGestaltId(gestaltIdEncoded: unknown): GestaltId | undefined { + if (typeof gestaltIdEncoded !== 'string') { + return; + } + switch (gestaltIdEncoded[0]) { + case 'n': + return decodeGestaltNodeId(gestaltIdEncoded); + case 'i': + return decodeGestaltIdentityId(gestaltIdEncoded); + } +} + +function decodeGestaltNodeId( + gestaltNodeIdEncoded: unknown, +): ['node', NodeId] | undefined { + if (typeof gestaltNodeIdEncoded !== 'string') { + return; + } + if (!gestaltNodeIdEncoded.startsWith('node-')) { + return; + } + const nodeIdEncoded = gestaltNodeIdEncoded.slice(5); + const nodeId = decodeNodeId(nodeIdEncoded); + if (nodeId == null) { + return; + } + return ['node', nodeId]; +} + +function decodeGestaltIdentityId( + gestaltIdentityId: unknown, +): ['identity', ProviderIdentityId] | undefined { + if (typeof gestaltIdentityId !== 'string') { + return; + } + if (!gestaltIdentityId.startsWith('identity-')) { + return; + } + const providerIdentityIdEncoded = gestaltIdentityId.slice(9); + const providerIdentityId = decodeProviderIdentityId( + providerIdentityIdEncoded, + ); + if (providerIdentityId == null) { + return; + } + return ['identity', providerIdentityId]; +} + +function createGestaltLinkIdGenerator() { + const generator = new IdRandom({ + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +function createNotificationIdGenerator( + lastId?: NotificationId, +): () => NotificationId { + const generator = new IdSortable({ + lastId, + randomSource: keysUtilsRandom.getRandomBytes, + }); + return () => generator.get(); +} + +function encodeNotificationId( + notificationId: NotificationId, +): NotificationIdEncoded { + return notificationId.toMultibase('base32hex') as NotificationIdEncoded; +} + +function decodeNotificationId( + notificationIdEncoded: string, +): NotificationId | undefined { + const notificationId = IdInternal.fromMultibase( + notificationIdEncoded, + ); + if (notificationId == null) { + return; + } + return notificationId; +} + +export { + createPermIdGenerator, + createNodeIdGenerator, + encodeNodeId, + decodeNodeId, + encodeNodeIdString, + decodeNodeIdString, + createCertIdGenerator, + encodeCertId, + decodeCertId, + createVaultIdGenerator, + encodeVaultId, + decodeVaultId, + createTaskIdGenerator, + encodeTaskId, + decodeTaskId, + createClaimIdGenerator, + encodeClaimId, + decodeClaimId, + encodeProviderIdentityId, + decodeProviderIdentityId, + encodeGestaltId, + encodeGestaltNodeId, + encodeGestaltIdentityId, + decodeGestaltId, + decodeGestaltNodeId, + decodeGestaltIdentityId, + createGestaltLinkIdGenerator, + createNotificationIdGenerator, + encodeNotificationId, + decodeNotificationId, +}; + +export * from './types'; diff --git a/src/ids/types.ts b/src/ids/types.ts new file mode 100644 index 00000000..b110b361 --- /dev/null +++ b/src/ids/types.ts @@ -0,0 +1,131 @@ +import type { Id } from '@matrixai/id'; +import type { Opaque } from '../types'; + +// ACL + +type PermissionId = Opaque<'PermissionId', Id>; +type PermissionIdString = Opaque<'PermissionIdString', string>; + +// Keys + +type CertId = Opaque<'CertId', Id>; +type CertIdString = Opaque<'CertIdString', string>; +/** + * This must be a raw lowercase base16 string and not a multibase string. + * The x509 certificate will strip any non-hex characters and add padding + * to the nearest byte. + */ +type CertIdEncoded = Opaque<'CertIdEncoded', string>; + +// Nodes + +type NodeId = Opaque<'NodeId', Id>; +type NodeIdString = Opaque<'NodeIdString', string>; +type NodeIdEncoded = Opaque<'NodeIdEncoded', string>; + +// Vaults + +type VaultId = Opaque<'VaultId', Id>; +type VaultIdString = Opaque<'VaultIdString', string>; +type VaultIdEncoded = Opaque<'VaultIdEncoded', string>; + +// Tasks + +type TaskId = Opaque<'TaskId', Id>; +type TaskIdString = Opaque<'TaskIdEncoded', string>; +type TaskIdEncoded = Opaque<'TaskIdEncoded', string>; +type TaskHandlerId = Opaque<'TaskHandlerId', string>; + +// Claims + +type ClaimId = Opaque<'ClaimId', Id>; +type ClaimIdString = Opaque<'ClaimIdString', string>; +type ClaimIdEncoded = Opaque<'ClaimIdEncoded', string>; + +// Identities + +/** + * Provider Id identifies an identity provider. + * e.g. `github.com` + */ +type ProviderId = Opaque<'ProviderId', string>; + +/** + * Identity Id must uniquely identify the identity on the identity provider. + * It must be the key that is used to look up the identity. + * If the provider uses a non-string type, make the necessary conversions. + * e.g. `cmcdragonkai` + */ +type IdentityId = Opaque<'IdentityId', string>; + +/** + * Tuple of `[ProviderId, IdentityId]` + */ +type ProviderIdentityId = [ProviderId, IdentityId]; + +/** + * This is a JSON encoding of `[ProviderId, IdentityId]` + */ +type ProviderIdentityIdEncoded = Opaque<'ProviderIdentityIdEncoded', string>; + +/** + * A unique identifier for the published claim, found on the identity provider. + * e.g. the gist ID on GitHub + */ +type ProviderIdentityClaimId = Opaque<'ProviderIdentityClaimId', string>; + +// Gestalts + +/** + * Prefixed NodeId and ProviderIdentityId. + * This is done to ensure there is no chance of conflict between + * `NodeId` and `ProviderIdentityId`. + */ +type GestaltId = ['node', NodeId] | ['identity', ProviderIdentityId]; + +/** + * GestaltId encoded. + */ +type GestaltIdEncoded = Opaque<'GestaltIdEncoded', string>; + +type GestaltLinkId = Opaque<'GestaltLinkId', Id>; +type GestaltLinkIdString = Opaque<'GestaltLinkIdString', string>; + +// Notifications + +type NotificationId = Opaque<'NotificationId', Id>; +type NotificationIdString = Opaque<'NotificationIdString', string>; +type NotificationIdEncoded = Opaque<'NotificationIdEncoded', string>; + +export type { + PermissionId, + PermissionIdString, + CertId, + CertIdString, + CertIdEncoded, + NodeId, + NodeIdString, + NodeIdEncoded, + VaultId, + VaultIdString, + VaultIdEncoded, + TaskId, + TaskIdString, + TaskIdEncoded, + TaskHandlerId, + ClaimId, + ClaimIdString, + ClaimIdEncoded, + ProviderId, + IdentityId, + ProviderIdentityId, + ProviderIdentityIdEncoded, + ProviderIdentityClaimId, + GestaltId, + GestaltIdEncoded, + GestaltLinkId, + GestaltLinkIdString, + NotificationId, + NotificationIdString, + NotificationIdEncoded, +}; diff --git a/src/types.ts b/src/types.ts index f668d518..1bdbea56 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1 +1,17 @@ -export type {} +/** + * Deconstructed promise + */ + type PromiseDeconstructed = { + p: Promise; + resolveP: (value: T | PromiseLike) => void; + rejectP: (reason?: any) => void; +}; + +type TLSConfig = { + keyPrivatePem: PrivateKeyPEM; + certChainPem: CertificatePEMChain; +}; + +export type { + PromiseDeconstructed +} diff --git a/src/utils.ts b/src/utils.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 00000000..379e77cd --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,31 @@ +import sysexits from './sysexits'; +import ErrorWebSocket from '../ErrorWebSocket'; + +class ErrorUtils extends ErrorWebSocket {} + +/** + * This is a special error that is only used for absurd situations + * Intended to placate typescript so that unreachable code type checks + * If this is thrown, this means there is a bug in the code + */ +class ErrorUtilsUndefinedBehaviour extends ErrorUtils { + static description = 'You should never see this error'; + exitCode = sysexits.SOFTWARE; +} + +class ErrorUtilsPollTimeout extends ErrorUtils { + static description = 'Poll timed out'; + exitCode = sysexits.TEMPFAIL; +} + +class ErrorUtilsNodePath extends ErrorUtils { + static description = 'Cannot derive default node path from unknown platform'; + exitCode = sysexits.USAGE; +} + +export { + ErrorUtils, + ErrorUtilsUndefinedBehaviour, + ErrorUtilsPollTimeout, + ErrorUtilsNodePath, +}; diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 00000000..1ce90c3f --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1,3 @@ +export { default as sysexits } from './sysexits'; +export * from './utils'; +export * as errors from './errors'; diff --git a/src/utils/sysexits.ts b/src/utils/sysexits.ts new file mode 100644 index 00000000..935c1810 --- /dev/null +++ b/src/utils/sysexits.ts @@ -0,0 +1,91 @@ +const sysexits = Object.freeze({ + OK: 0, + GENERAL: 1, + // Sysexit standard starts at 64 to avoid conflicts + /** + * The command was used incorrectly, e.g., with the wrong number of arguments, + * a bad flag, a bad syntax in a parameter, or whatever. + */ + USAGE: 64, + /** + * The input data was incorrect in some way. This should only be used for + * user's data and not system files. + */ + DATAERR: 65, + /** + * An input file (not a system file) did not exist or was not readable. + * This could also include errors like "No message" to a mailer + * (if it cared to catch it). + */ + NOINPUT: 66, + /** + * The user specified did not exist. This might be used for mail addresses + * or remote logins. + */ + NOUSER: 67, + /** + * The host specified did not exist. This is used in mail addresses or + * network requests. + */ + NOHOST: 68, + /** + * A service is unavailable. This can occur if a support program or file + * does not exist. This can also be used as a catchall message when + * something you wanted to do does not work, but you do not know why. + */ + UNAVAILABLE: 69, + /** + * An internal software error has been detected. This should be limited to + * non-operating system related errors as possible. + */ + SOFTWARE: 70, + /** + * An operating system error has been detected. This is intended to be used + * for such things as "cannot fork", "cannot create pipe", or the like. + * It in-cludes things like getuid returning a user that does not exist in + * the passwd file. + */ + OSERR: 71, + /** + * Some system file (e.g., /etc/passwd, /var/run/utx.active, etc.) + * does not exist, cannot be opened, or has some sort of error + * (e.g., syntax error). + */ + OSFILE: 72, + /** + * A (user specified) output file cannot be created. + */ + CANTCREAT: 73, + /** + * An error occurred while doing I/O on some file. + */ + IOERR: 74, + /** + * Temporary failure, indicating something that is not really an error. + * In sendmail, this means that a mailer (e.g.) could not create a connection, + * and the request should be reattempted later. + */ + TEMPFAIL: 75, + /** + * The remote system returned something that was "not possible" during a + * protocol exchange. + */ + PROTOCOL: 76, + /** + * You did not have sufficient permission to perform the operation. This is + * not intended for file system problems, which should use EX_NOINPUT or + * EX_CANTCREAT, but rather for higher level permissions. + */ + NOPERM: 77, + /** + * Something was found in an un-configured or mis-configured state. + */ + CONFIG: 78, + CANNOT_EXEC: 126, + COMMAND_NOT_FOUND: 127, + INVALID_EXIT_ARG: 128, + // 128+ are reserved for signal exits + UNKNOWN: 255, +}); + +export default sysexits; diff --git a/src/utils/utils.ts b/src/utils/utils.ts new file mode 100644 index 00000000..4c4b7722 --- /dev/null +++ b/src/utils/utils.ts @@ -0,0 +1,30 @@ +import type { + PromiseDeconstructed, +} from '../types'; +import * as utilsErrors from './errors'; + + +function never(): never { + throw new utilsErrors.ErrorUtilsUndefinedBehaviour(); +} + +/** + * Deconstructed promise + */ +function promise(): PromiseDeconstructed { + let resolveP, rejectP; + const p = new Promise((resolve, reject) => { + resolveP = resolve; + rejectP = reject; + }); + return { + p, + resolveP, + rejectP, + }; +} + +export { + never, + promise +}; diff --git a/tests/WebSocket.test.ts b/tests/WebSocket.test.ts new file mode 100644 index 00000000..56564ccc --- /dev/null +++ b/tests/WebSocket.test.ts @@ -0,0 +1,850 @@ +import type { ReadableWritablePair } from 'stream/web'; +import type { TLSConfig } from '@/network/types'; +import type { KeyPair } from '@/keys/types'; +import type { NodeId } from '@/ids/types'; +import type http from 'http'; +import fs from 'fs'; +import path from 'path'; +import os from 'os'; +import https from 'https'; +import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; +import { testProp, fc } from '@fast-check/jest'; +import { Timer } from '@matrixai/timer'; +import { status } from '@matrixai/async-init'; +import WebSocketServer from '@/WebSocketServer'; +import WebSocketClient from '@/WebSocketClient'; +import { promise } from '@/utils'; +import * as keysUtils from '@/keys/utils'; +import * as webSocketErrors from '@/errors'; +import * as nodesUtils from '@/nodes/utils'; +import * as testNodeUtils from '../nodes/utils'; +import * as testsUtils from './utils'; + +// This file tests both the client and server together. They're too interlinked +// to be separate. +describe('WebSocket', () => { + const logger = new Logger('websocket test', LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), + ]); + let keyPair: KeyPair; + let nodeId: NodeId; + let tlsConfig: TLSConfig; + const host = '127.0.0.2'; + let webSocketServer: WebSocketServer; + let webSocketClient: WebSocketClient; + + const messagesArb = fc.array( + fc.uint8Array({ minLength: 1 }).map((d) => Buffer.from(d)), + ); + const streamsArb = fc.array(messagesArb, { minLength: 1 }).noShrink(); + const asyncReadWrite = async ( + messages: Array, + streamPair: ReadableWritablePair, + ) => { + await Promise.allSettled([ + (async () => { + const writer = streamPair.writable.getWriter(); + for (const message of messages) { + await writer.write(message); + } + await writer.close(); + })(), + (async () => { + for await (const _ of streamPair.readable) { + // No touch, only consume + } + })(), + ]); + }; + + beforeEach(async () => { + dataDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'polykey-test-'), + ); + keyPair = keysUtils.generateKeyPair(); + nodeId = keysUtils.publicKeyToNodeId(keyPair.publicKey); + tlsConfig = await testsUtils.createTLSConfig(keyPair); + }); + afterEach(async () => { + logger.info('AFTEREACH'); + await webSocketServer?.stop(true); + await webSocketClient?.destroy(true); + await fs.promises.rm(dataDir, { force: true, recursive: true }); + }); + + // These tests are share between client and server + test('makes a connection', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + + const writer = websocket.writable.getWriter(); + const reader = websocket.readable.getReader(); + const message1 = Buffer.from('1request1'); + await writer.write(message1); + expect((await reader.read()).value).toStrictEqual(message1); + const message2 = Buffer.from('1request2'); + await writer.write(message2); + expect((await reader.read()).value).toStrictEqual(message2); + await writer.close(); + expect((await reader.read()).done).toBeTrue(); + logger.info('ending'); + }); + test('can change TLS config', async () => { + const keyPairNew = keysUtils.generateKeyPair(); + const nodeIdNew = keysUtils.publicKeyToNodeId(keyPairNew.publicKey); + const tlsConfigNew = await testsUtils.createTLSConfig(keyPairNew); + + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId, nodeIdNew], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + expect(websocket.meta.nodeId).toBe(nodesUtils.encodeNodeId(nodeId)); + websocket.cancel(); + + // Changing certs + webSocketServer.setTlsConfig(tlsConfigNew); + const websocket2 = await webSocketClient.startConnection(); + expect(websocket2.meta.nodeId).toBe(nodesUtils.encodeNodeId(nodeIdNew)); + websocket2.cancel(); + + logger.info('ending'); + }); + test('makes a connection over IPv6', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host: '::1', + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host: '::1', + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + + const writer = websocket.writable.getWriter(); + const reader = websocket.readable.getReader(); + const message1 = Buffer.from('1request1'); + await writer.write(message1); + expect((await reader.read()).value).toStrictEqual(message1); + const message2 = Buffer.from('1request2'); + await writer.write(message2); + expect((await reader.read()).value).toStrictEqual(message2); + await writer.close(); + expect((await reader.read()).done).toBeTrue(); + logger.info('ending'); + }); + test('handles a connection and closes before message', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + await websocket.writable.close(); + const reader = websocket.readable.getReader(); + expect((await reader.read()).done).toBeTrue(); + logger.info('ending'); + }); + testProp( + 'handles multiple connections', + [streamsArb], + async (streamsData) => { + try { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + + const testStream = async (messages: Array) => { + const websocket = await webSocketClient.startConnection(); + const writer = websocket.writable.getWriter(); + const reader = websocket.readable.getReader(); + for (const message of messages) { + await writer.write(message); + const response = await reader.read(); + expect(response.done).toBeFalse(); + expect(response.value?.toString()).toStrictEqual( + message.toString(), + ); + } + await writer.close(); + expect((await reader.read()).done).toBeTrue(); + }; + const streams = streamsData.map((messages) => testStream(messages)); + await Promise.all(streams); + + logger.info('ending'); + } finally { + await webSocketServer.stop(true); + } + }, + ); + test('handles https server failure', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + + const closeP = promise(); + // @ts-ignore: protected property + webSocketServer.server.close(() => { + closeP.resolveP(); + }); + await closeP.p; + // The webSocketServer should stop itself + expect(webSocketServer[status]).toBe(null); + + logger.info('ending'); + }); + test('handles webSocketServer server failure', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + + const closeP = promise(); + // @ts-ignore: protected property + webSocketServer.webSocketServer.close(() => { + closeP.resolveP(); + }); + await closeP.p; + // The webSocketServer should stop itself + expect(webSocketServer[status]).toBe(null); + + logger.info('ending'); + }); + test('client ends connection abruptly', async () => { + const streamPairProm = + promise>(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + streamPairProm.resolveP(streamPair); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + + const testProcess = await testsUtils.spawn( + 'ts-node', + [ + '--project', + testsUtils.tsConfigPath, + `${globalThis.testDir}/websockets/testClient.ts`, + ], + { + env: { + PK_TEST_HOST: host, + PK_TEST_PORT: `${webSocketServer.getPort()}`, + PK_TEST_NODE_ID: nodesUtils.encodeNodeId(nodeId), + }, + }, + logger, + ); + const startedProm = promise(); + testProcess.stdout!.on('data', (data) => { + startedProm.resolveP(data.toString()); + }); + testProcess.stderr!.on('data', (data) => + startedProm.rejectP(data.toString()), + ); + const exitedProm = promise(); + testProcess.once('exit', () => exitedProm.resolveP()); + await startedProm.p; + + // Killing the client + testProcess.kill('SIGTERM'); + await exitedProm.p; + + const streamPair = await streamPairProm.p; + // Everything should throw after websocket ends early + await expect(async () => { + for await (const _ of streamPair.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + const serverWritable = streamPair.writable.getWriter(); + await expect(serverWritable.write(Buffer.from('test'))).rejects.toThrow(); + logger.info('ending'); + }); + test('server ends connection abruptly', async () => { + const testProcess = await testsUtils.spawn( + 'ts-node', + [ + '--project', + testsUtils.tsConfigPath, + `${globalThis.testDir}/websockets/testServer.ts`, + ], + { + env: { + PK_TEST_KEY_PRIVATE_PEM: tlsConfig.keyPrivatePem, + PK_TEST_CERT_CHAIN_PEM: tlsConfig.certChainPem, + PK_TEST_HOST: host, + }, + }, + logger, + ); + const startedProm = promise(); + testProcess.stdout!.on('data', (data) => { + startedProm.resolveP(parseInt(data.toString())); + }); + testProcess.stderr!.on('data', (data) => + startedProm.rejectP(data.toString()), + ); + const exitedProm = promise(); + testProcess.once('exit', () => exitedProm.resolveP()); + + logger.info(`Server started on port ${await startedProm.p}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: await startedProm.p, + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + + // Killing the server + testProcess.kill('SIGTERM'); + await exitedProm.p; + + // Waiting for connections to end + await webSocketClient.destroy(); + // Checking client's response to connection dropping + await expect(async () => { + for await (const _ of websocket.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + const clientWritable = websocket.writable.getWriter(); + await expect(clientWritable.write(Buffer.from('test'))).rejects.toThrow(); + logger.info('ending'); + }); + // These describe blocks contains tests specific to either the client or server + describe('WebSocketServer', () => { + testProp( + 'allows half closed writable closes first', + [messagesArb, messagesArb], + async (messages1, messages2) => { + try { + const serverStreamProm = promise(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void (async () => { + const writer = streamPair.writable.getWriter(); + for await (const val of messages2) { + await writer.write(val); + } + await writer.close(); + for await (const _ of streamPair.readable) { + // No touch, only consume + } + })().then(serverStreamProm.resolveP, serverStreamProm.rejectP); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + await asyncReadWrite(messages1, websocket); + await serverStreamProm.p; + logger.info('ending'); + } finally { + await webSocketServer.stop(true); + } + }, + ); + testProp( + 'allows half closed readable closes first', + [messagesArb, messagesArb], + async (messages1, messages2) => { + try { + const serverStreamProm = promise(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void (async () => { + for await (const _ of streamPair.readable) { + // No touch, only consume + } + const writer = streamPair.writable.getWriter(); + for await (const val of messages2) { + await writer.write(val); + } + await writer.close(); + })().then(serverStreamProm.resolveP, serverStreamProm.rejectP); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + await asyncReadWrite(messages1, websocket); + await serverStreamProm.p; + logger.info('ending'); + } finally { + await webSocketServer.stop(true); + } + }, + ); + testProp( + 'handles early close of readable', + [messagesArb, messagesArb], + async (messages1, messages2) => { + try { + const serverStreamProm = promise(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void (async () => { + await streamPair.readable.cancel(); + const writer = streamPair.writable.getWriter(); + for await (const val of messages2) { + await writer.write(val); + } + await writer.close(); + })().then(serverStreamProm.resolveP, serverStreamProm.rejectP); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + await asyncReadWrite(messages1, websocket); + await serverStreamProm.p; + logger.info('ending'); + } finally { + await webSocketServer.stop(true); + } + }, + ); + test('destroying ClientServer stops all connections', async () => { + const streamPairProm = + promise>(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + streamPairProm.resolveP(streamPair); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + await webSocketServer.stop(true); + const streamPair = await streamPairProm.p; + // Everything should throw after websocket ends early + await expect(async () => { + for await (const _ of websocket.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + await expect(async () => { + for await (const _ of streamPair.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + const clientWritable = websocket.writable.getWriter(); + const serverWritable = streamPair.writable.getWriter(); + await expect(clientWritable.write(Buffer.from('test'))).rejects.toThrow(); + await expect(serverWritable.write(Buffer.from('test'))).rejects.toThrow(); + logger.info('ending'); + }); + test('server rejects normal HTTPS requests', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + const getResProm = promise(); + https.get( + `https://${host}:${webSocketServer.getPort()}/`, + { rejectUnauthorized: false }, + getResProm.resolveP, + ); + const res = await getResProm.p; + const contentProm = promise(); + res.once('data', (d) => contentProm.resolveP(d.toString())); + const endProm = promise(); + res.on('error', endProm.rejectP); + res.on('close', endProm.resolveP); + + expect(res.statusCode).toBe(426); + await expect(contentProm.p).resolves.toBe('426 Upgrade Required'); + expect(res.headers['connection']).toBe('Upgrade'); + expect(res.headers['upgrade']).toBe('websocket'); + }); + test('ping timeout', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (_) => { + logger.info('inside callback'); + // Hang connection + }, + tlsConfig, + host, + pingTimeoutTimeTime: 100, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + await webSocketClient.startConnection(); + await webSocketClient.destroy(); + logger.info('ending'); + }); + }); + describe('WebSocketClient', () => { + test('destroying ClientClient stops all connections', async () => { + const streamPairProm = + promise>(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + streamPairProm.resolveP(streamPair); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const websocket = await webSocketClient.startConnection(); + // Destroying the client, force close connections + await webSocketClient.destroy(true); + const streamPair = await streamPairProm.p; + // Everything should throw after websocket ends early + await expect(async () => { + for await (const _ of websocket.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + await expect(async () => { + for await (const _ of streamPair.readable) { + // No touch, only consume + } + }).rejects.toThrow(); + const clientWritable = websocket.writable.getWriter(); + const serverWritable = streamPair.writable.getWriter(); + await expect(clientWritable.write(Buffer.from('test'))).rejects.toThrow(); + await expect(serverWritable.write(Buffer.from('test'))).rejects.toThrow(); + await webSocketServer.stop(); + logger.info('ending'); + }); + test('authentication rejects bad server certificate', async () => { + const invalidNodeId = testNodeUtils.generateRandomNodeId(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [invalidNodeId], + logger: logger.getChild('clientClient'), + }); + await expect(webSocketClient.startConnection()).rejects.toThrow( + webSocketErrors.ErrorCertChainUnclaimed, + ); + // @ts-ignore: kidnap protected property + const activeConnections = webSocketClient.activeConnections; + expect(activeConnections.size).toBe(0); + await webSocketServer.stop(); + logger.info('ending'); + }); + test('authenticates with multiple certs in chain', async () => { + const keyPairs: Array = [ + keyPair, + keysUtils.generateKeyPair(), + keysUtils.generateKeyPair(), + keysUtils.generateKeyPair(), + keysUtils.generateKeyPair(), + ]; + const tlsConfig = await testsUtils.createTLSConfigWithChain(keyPairs); + const nodeId = keysUtils.publicKeyToNodeId(keyPairs[1].publicKey); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const connProm = webSocketClient.startConnection(); + await connProm; + await expect(connProm).toResolve(); + // @ts-ignore: kidnap protected property + const activeConnections = webSocketClient.activeConnections; + expect(activeConnections.size).toBe(1); + logger.info('ending'); + }); + test('authenticates with multiple expected nodes', async () => { + const alternativeNodeId = testNodeUtils.generateRandomNodeId(); + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void streamPair.readable + .pipeTo(streamPair.writable) + .catch(() => {}) + .finally(() => logger.info('STREAM HANDLING ENDED')); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId, alternativeNodeId], + logger: logger.getChild('clientClient'), + }); + await expect(webSocketClient.startConnection()).toResolve(); + // @ts-ignore: kidnap protected property + const activeConnections = webSocketClient.activeConnections; + expect(activeConnections.size).toBe(1); + logger.info('ending'); + }); + test('connection times out', async () => { + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: 12345, + expectedNodeIds: [nodeId], + connectionTimeoutTime: 0, + logger: logger.getChild('clientClient'), + }); + await expect(webSocketClient.startConnection({})).rejects.toThrow(); + await expect( + webSocketClient.startConnection({ + timer: new Timer({ delay: 0 }), + }), + ).rejects.toThrow(); + logger.info('ending'); + }); + test('ping timeout', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (_) => { + logger.info('inside callback'); + // Hang connection + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + pingTimeoutTimeTime: 100, + logger: logger.getChild('clientClient'), + }); + await webSocketClient.startConnection(); + await webSocketClient.destroy(); + logger.info('ending'); + }); + test('stream is aborted', async () => { + webSocketServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (streamPair) => { + logger.info('inside callback'); + void Promise.all([ + (async () => { + for await (const _ of streamPair.readable) { + // Do nothing + } + })().catch(() => {}), + (async () => { + const message = Buffer.alloc(5, 123); + const writer = streamPair.writable.getWriter(); + await writer.write(message); + await writer.write(message); + await writer.write(message); + await writer.close(); + })().catch(() => {}), + ]); + }, + tlsConfig, + host, + logger: logger.getChild('server'), + }); + logger.info(`Server started on port ${webSocketServer.getPort()}`); + webSocketClient = await WebSocketClient.createWebSocketClient({ + host, + port: webSocketServer.getPort(), + expectedNodeIds: [nodeId], + logger: logger.getChild('clientClient'), + }); + const abortController = new AbortController(); + const websocket = await webSocketClient.startConnection({ + signal: abortController.signal, + }); + // Signal the connection to end + abortController.abort('SOME REASON'); + await expect(async () => { + for await (const _ of websocket.readable) { + // Await error + } + }).rejects.toThrow(webSocketErrors.ErrorClientStreamAborted); + logger.info('ending'); + }); + }); +}); diff --git a/tests/testClient.ts b/tests/testClient.ts new file mode 100644 index 00000000..ddc8c761 --- /dev/null +++ b/tests/testClient.ts @@ -0,0 +1,31 @@ +/** + * This is spawned as a background process for use in some NodeConnection.test.ts tests + * This process will not preserve jest testing environment, + * any usage of jest globals will result in an error + * Beware of propagated usage of jest globals through the script dependencies + * @module + */ +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import WebSocketClient from '@/WebSocketClient'; +import * as nodesUtils from '@/nodes/utils'; + +async function main() { + const logger = new Logger('websocket test', LogLevel.WARN, [ + new StreamHandler(), + ]); + const clientClient = await WebSocketClient.createWebSocketClient({ + expectedNodeIds: [nodesUtils.decodeNodeId(process.env.PK_TEST_NODE_ID!)!], + host: process.env.PK_TEST_HOST ?? '127.0.0.1', + port: parseInt(process.env.PK_TEST_PORT!), + logger, + }); + // Ignore streams, make connection hang + await clientClient.startConnection(); + process.stdout.write(`ready`); +} + +if (require.main === module) { + void main(); +} + +export default main; diff --git a/tests/testServer.ts b/tests/testServer.ts new file mode 100644 index 00000000..3cad3b28 --- /dev/null +++ b/tests/testServer.ts @@ -0,0 +1,36 @@ +/** + * This is spawned as a background process for use in some NodeConnection.test.ts tests + * This process will not preserve jest testing environment, + * any usage of jest globals will result in an error + * Beware of propagated usage of jest globals through the script dependencies + * @module + */ +import type { CertificatePEMChain, PrivateKeyPEM } from '@/keys/types'; +import type { TLSConfig } from '@/network/types'; +import Logger, { LogLevel, StreamHandler } from '@matrixai/logger'; +import WebSocketServer from '@/websockets/WebSocketServer'; + +async function main() { + const logger = new Logger('websocket test', LogLevel.WARN, [ + new StreamHandler(), + ]); + const tlsConfig: TLSConfig = { + keyPrivatePem: process.env.PK_TEST_KEY_PRIVATE_PEM as PrivateKeyPEM, + certChainPem: process.env.PK_TEST_CERT_CHAIN_PEM as CertificatePEMChain, + }; + const clientServer = await WebSocketServer.createWebSocketServer({ + connectionCallback: (_) => { + // Ignore streams and hang connections + }, + host: process.env.PK_TEST_HOST ?? '127.0.0.1', + tlsConfig, + logger, + }); + process.stdout.write(`${clientServer.getPort()}`); +} + +if (require.main === module) { + void main(); +} + +export default main; From 8e16eb1d6eedd11332c931517090d2fa191004e1 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:54:09 +1000 Subject: [PATCH 002/149] feat: generic WebSocketClient --- src/WebSocketClient.ts | 28 ++- src/WebSocketServer.ts | 2 +- src/WebSocketStream.ts | 2 - src/ids/index.ts | 383 ---------------------------------------- src/ids/types.ts | 131 -------------- src/index.ts | 9 +- src/types.ts | 7 +- tests/WebSocket.test.ts | 5 - 8 files changed, 26 insertions(+), 541 deletions(-) delete mode 100644 src/ids/index.ts delete mode 100644 src/ids/types.ts diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index c370cf38..59f08f3e 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,6 +1,5 @@ -import type { TLSSocket } from 'tls'; +import type { DetailedPeerCertificate, TLSSocket } from 'tls'; import type { ContextTimed } from '@matrixai/contexts'; -import type { NodeId, NodeIdEncoded } from '../ids'; import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; @@ -10,7 +9,6 @@ import WebSocketStream from './WebSocketStream'; import * as webSocketUtils from './utils'; import * as webSocketErrors from './errors'; import { promise } from './utils'; -import * as nodesUtils from '../nodes/utils'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() @@ -32,29 +30,29 @@ class WebSocketClient { static async createWebSocketClient({ host, port, - expectedNodeIds, connectionTimeoutTime = Infinity, pingIntervalTime = 1_000, pingTimeoutTimeTime = 10_000, logger = new Logger(this.name), + verifyCallback }: { host: string; port: number; - expectedNodeIds: Array; connectionTimeoutTime?: number; pingIntervalTime?: number; pingTimeoutTimeTime?: number; logger?: Logger; + verifyCallback?: () => Promise; }): Promise { logger.info(`Creating ${this.name}`); const clientClient = new this( logger, host, port, - expectedNodeIds, connectionTimeoutTime, pingIntervalTime, pingTimeoutTimeTime, + verifyCallback ); logger.info(`Created ${this.name}`); return clientClient; @@ -67,10 +65,10 @@ class WebSocketClient { protected logger: Logger, host: string, protected port: number, - protected expectedNodeIds: Array, protected connectionTimeoutTime: number, protected pingIntervalTime: number, protected pingTimeoutTimeTime: number, + protected verifyCallback?: (peerCert: DetailedPeerCertificate) => Promise ) { if (Validator.isValidIPv4String(host)[0]) { this.host = host; @@ -151,14 +149,15 @@ class WebSocketClient { this.logger.info(`Connecting to ${address}`); const connectProm = promise(); const authenticateProm = promise<{ - nodeId: NodeIdEncoded; localHost: string; localPort: number; remoteHost: string; remotePort: number; + peerCert: DetailedPeerCertificate; }>(); + // Let ws handle authentication if no custom verify callback is provided. const ws = new WebSocket(address, { - rejectUnauthorized: false, + rejectUnauthorized: this.verifyCallback != null, }); // Handle connection failure const openErrorHandler = (e) => { @@ -169,21 +168,20 @@ class WebSocketClient { ); }; ws.once('error', openErrorHandler); - // Authenticate server's certificates + // Authenticate server's certificate (this will be automatically done) ws.once('upgrade', async (request) => { const tlsSocket = request.socket as TLSSocket; const peerCert = tlsSocket.getPeerCertificate(true); try { - const nodeId = await webSocketUtils.verifyServerCertificateChain( - this.expectedNodeIds, - webSocketUtils.detailedToCertChain(peerCert), - ); + if (this.verifyCallback != null) { + await this.verifyCallback(peerCert); + } authenticateProm.resolveP({ - nodeId: nodesUtils.encodeNodeId(nodeId), localHost: request.connection.localAddress ?? '', localPort: request.connection.localPort ?? 0, remoteHost: request.connection.remoteAddress ?? '', remotePort: request.connection.remotePort ?? 0, + peerCert }); } catch (e) { authenticateProm.rejectP(e); diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 100955a5..4b53b054 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -1,4 +1,3 @@ -import type { TLSConfig } from '../network/types'; import type { IncomingMessage, ServerResponse } from 'http'; import type tls from 'tls'; import https from 'https'; @@ -9,6 +8,7 @@ import WebSocketStream from './WebSocketStream'; import * as webSocketErrors from './errors'; import * as webSocketEvents from './events'; import { never, promise } from './utils'; +import { TLSConfig } from './types'; type ConnectionCallback = (streamPair: WebSocketStream) => void; diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index cbe9c1f4..721bf5a4 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -5,7 +5,6 @@ import type { } from 'stream/web'; import type * as ws from 'ws'; import type Logger from '@matrixai/logger'; -import type { NodeIdEncoded } from './ids/types'; import { WritableStream, ReadableStream } from 'stream/web'; import * as webSocketErrors from './errors'; import * as utilsErrors from './utils/errors'; @@ -33,7 +32,6 @@ class WebSocketStream implements ReadableWritablePair { pingInterval: number, pingTimeoutTime: number, protected metadata: { - nodeId?: NodeIdEncoded; localHost: string; localPort: number; remoteHost: string; diff --git a/src/ids/index.ts b/src/ids/index.ts deleted file mode 100644 index 75375c02..00000000 --- a/src/ids/index.ts +++ /dev/null @@ -1,383 +0,0 @@ -import type { - PermissionId, - CertId, - CertIdEncoded, - NodeId, - NodeIdString, - NodeIdEncoded, - VaultId, - VaultIdEncoded, - TaskId, - TaskIdEncoded, - ClaimId, - ClaimIdEncoded, - ProviderIdentityId, - ProviderIdentityIdEncoded, - NotificationId, - NotificationIdEncoded, - GestaltId, - GestaltIdEncoded, - GestaltLinkId, -} from './types'; -import { IdInternal, IdSortable, IdRandom } from '@matrixai/id'; -import * as keysUtilsRandom from '../keys/utils/random'; - -function createPermIdGenerator(): () => PermissionId { - const generator = new IdRandom({ - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -/** - * Creates a NodeId generator. - * This does not use `IdRandom` because it is not a UUID4. - * Instead this just generates random 32 bytes. - */ -function createNodeIdGenerator(): () => NodeId { - return () => { - return IdInternal.fromBuffer(keysUtilsRandom.getRandomBytes(32)); - }; -} - -/** - * Encodes the NodeId as a `base32hex` string - */ -function encodeNodeId(nodeId: NodeId): NodeIdEncoded { - return nodeId.toMultibase('base32hex') as NodeIdEncoded; -} - -/** - * Decodes an encoded NodeId string into a NodeId - */ -function decodeNodeId(nodeIdEncoded: unknown): NodeId | undefined { - if (typeof nodeIdEncoded !== 'string') { - return; - } - const nodeId = IdInternal.fromMultibase(nodeIdEncoded); - if (nodeId == null) { - return; - } - // All NodeIds are 32 bytes long - // The NodeGraph requires a fixed size for Node Ids - if (nodeId.length !== 32) { - return; - } - return nodeId; -} - -function encodeNodeIdString(nodeId: NodeId): NodeIdString { - return nodeId.toString() as NodeIdString; -} - -function decodeNodeIdString(nodeIdString: unknown): NodeId | undefined { - if (typeof nodeIdString !== 'string') { - return; - } - const nodeId = IdInternal.fromString(nodeIdString); - if (nodeId == null) { - return; - } - // All NodeIds are 32 bytes long - // The NodeGraph requires a fixed size for Node Ids - if (nodeId.length !== 32) { - return; - } - return nodeId; -} - -/** - * Generates CertId - */ -function createCertIdGenerator(lastCertId?: CertId): () => CertId { - const generator = new IdSortable({ - lastId: lastCertId, - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -/** - * Encodes `CertId` to `CertIdEncoded` - */ -function encodeCertId(certId: CertId): CertIdEncoded { - return certId.toBuffer().toString('hex') as CertIdEncoded; -} - -/** - * Decodes `CertIdEncoded` to `CertId` - */ -function decodeCertId(certIdEncoded: unknown): CertId | undefined { - if (typeof certIdEncoded !== 'string') { - return; - } - const certIdBuffer = Buffer.from(certIdEncoded, 'hex'); - const certId = IdInternal.fromBuffer(certIdBuffer); - if (certId == null) { - return; - } - // All `CertId` are 16 bytes long - if (certId.length !== 16) { - return; - } - return certId; -} - -function createVaultIdGenerator(): () => VaultId { - const generator = new IdRandom({ - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -function encodeVaultId(vaultId: VaultId): VaultIdEncoded { - return vaultId.toMultibase('base58btc') as VaultIdEncoded; -} - -function decodeVaultId(vaultIdEncoded: unknown): VaultId | undefined { - if (typeof vaultIdEncoded !== 'string') return; - const vaultId = IdInternal.fromMultibase(vaultIdEncoded); - if (vaultId == null) return; - // All VaultIds are 16 bytes long - if (vaultId.length !== 16) return; - return vaultId; -} - -/** - * Generates TaskId - * TaskIds are lexicographically sortable 128 bit IDs - * They are strictly monotonic and unique with respect to the `nodeId` - * When the `NodeId` changes, make sure to regenerate this generator - */ -function createTaskIdGenerator(lastTaskId?: TaskId) { - const generator = new IdSortable({ - lastId: lastTaskId, - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -/** - * Encodes the TaskId as a `base32hex` string - */ -function encodeTaskId(taskId: TaskId): TaskIdEncoded { - return taskId.toMultibase('base32hex') as TaskIdEncoded; -} - -/** - * Decodes an encoded TaskId string into a TaskId - */ -function decodeTaskId(taskIdEncoded: unknown): TaskId | undefined { - if (typeof taskIdEncoded !== 'string') { - return; - } - const taskId = IdInternal.fromMultibase(taskIdEncoded); - if (taskId == null) { - return; - } - // All TaskIds are 16 bytes long - if (taskId.length !== 16) { - return; - } - return taskId; -} - -/** - * Generator for `ClaimId` - * Make sure the `nodeId` is set to this node's own `NodeId` - */ -function createClaimIdGenerator(nodeId: NodeId, lastClaimId?: ClaimId) { - const generator = new IdSortable({ - nodeId, - lastId: lastClaimId, - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -function encodeClaimId(claimId: ClaimId): ClaimIdEncoded { - return claimId.toMultibase('base32hex') as ClaimIdEncoded; -} - -function decodeClaimId(claimIdEncoded: unknown): ClaimId | undefined { - if (typeof claimIdEncoded !== 'string') { - return; - } - const claimId = IdInternal.fromMultibase(claimIdEncoded); - if (claimId == null) { - return; - } - return claimId; -} - -function encodeProviderIdentityId( - providerIdentityId: ProviderIdentityId, -): ProviderIdentityIdEncoded { - return JSON.stringify(providerIdentityId) as ProviderIdentityIdEncoded; -} - -function decodeProviderIdentityId( - providerIdentityIdEncoded: unknown, -): ProviderIdentityId | undefined { - if (typeof providerIdentityIdEncoded !== 'string') { - return; - } - let providerIdentityId: unknown; - try { - providerIdentityId = JSON.parse(providerIdentityIdEncoded); - } catch { - return; - } - if ( - !Array.isArray(providerIdentityId) || - providerIdentityId.length !== 2 || - typeof providerIdentityId[0] !== 'string' || - typeof providerIdentityId[1] !== 'string' - ) { - return; - } - return providerIdentityId as ProviderIdentityId; -} - -function encodeGestaltId(gestaltId: GestaltId): GestaltIdEncoded { - switch (gestaltId[0]) { - case 'node': - return encodeGestaltNodeId(gestaltId); - case 'identity': - return encodeGestaltIdentityId(gestaltId); - } -} - -function encodeGestaltNodeId( - gestaltNodeId: ['node', NodeId], -): GestaltIdEncoded { - return (gestaltNodeId[0] + - '-' + - encodeNodeId(gestaltNodeId[1])) as GestaltIdEncoded; -} - -function encodeGestaltIdentityId( - gestaltIdentityId: ['identity', ProviderIdentityId], -): GestaltIdEncoded { - return (gestaltIdentityId[0] + - '-' + - encodeProviderIdentityId(gestaltIdentityId[1])) as GestaltIdEncoded; -} - -function decodeGestaltId(gestaltIdEncoded: unknown): GestaltId | undefined { - if (typeof gestaltIdEncoded !== 'string') { - return; - } - switch (gestaltIdEncoded[0]) { - case 'n': - return decodeGestaltNodeId(gestaltIdEncoded); - case 'i': - return decodeGestaltIdentityId(gestaltIdEncoded); - } -} - -function decodeGestaltNodeId( - gestaltNodeIdEncoded: unknown, -): ['node', NodeId] | undefined { - if (typeof gestaltNodeIdEncoded !== 'string') { - return; - } - if (!gestaltNodeIdEncoded.startsWith('node-')) { - return; - } - const nodeIdEncoded = gestaltNodeIdEncoded.slice(5); - const nodeId = decodeNodeId(nodeIdEncoded); - if (nodeId == null) { - return; - } - return ['node', nodeId]; -} - -function decodeGestaltIdentityId( - gestaltIdentityId: unknown, -): ['identity', ProviderIdentityId] | undefined { - if (typeof gestaltIdentityId !== 'string') { - return; - } - if (!gestaltIdentityId.startsWith('identity-')) { - return; - } - const providerIdentityIdEncoded = gestaltIdentityId.slice(9); - const providerIdentityId = decodeProviderIdentityId( - providerIdentityIdEncoded, - ); - if (providerIdentityId == null) { - return; - } - return ['identity', providerIdentityId]; -} - -function createGestaltLinkIdGenerator() { - const generator = new IdRandom({ - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -function createNotificationIdGenerator( - lastId?: NotificationId, -): () => NotificationId { - const generator = new IdSortable({ - lastId, - randomSource: keysUtilsRandom.getRandomBytes, - }); - return () => generator.get(); -} - -function encodeNotificationId( - notificationId: NotificationId, -): NotificationIdEncoded { - return notificationId.toMultibase('base32hex') as NotificationIdEncoded; -} - -function decodeNotificationId( - notificationIdEncoded: string, -): NotificationId | undefined { - const notificationId = IdInternal.fromMultibase( - notificationIdEncoded, - ); - if (notificationId == null) { - return; - } - return notificationId; -} - -export { - createPermIdGenerator, - createNodeIdGenerator, - encodeNodeId, - decodeNodeId, - encodeNodeIdString, - decodeNodeIdString, - createCertIdGenerator, - encodeCertId, - decodeCertId, - createVaultIdGenerator, - encodeVaultId, - decodeVaultId, - createTaskIdGenerator, - encodeTaskId, - decodeTaskId, - createClaimIdGenerator, - encodeClaimId, - decodeClaimId, - encodeProviderIdentityId, - decodeProviderIdentityId, - encodeGestaltId, - encodeGestaltNodeId, - encodeGestaltIdentityId, - decodeGestaltId, - decodeGestaltNodeId, - decodeGestaltIdentityId, - createGestaltLinkIdGenerator, - createNotificationIdGenerator, - encodeNotificationId, - decodeNotificationId, -}; - -export * from './types'; diff --git a/src/ids/types.ts b/src/ids/types.ts deleted file mode 100644 index b110b361..00000000 --- a/src/ids/types.ts +++ /dev/null @@ -1,131 +0,0 @@ -import type { Id } from '@matrixai/id'; -import type { Opaque } from '../types'; - -// ACL - -type PermissionId = Opaque<'PermissionId', Id>; -type PermissionIdString = Opaque<'PermissionIdString', string>; - -// Keys - -type CertId = Opaque<'CertId', Id>; -type CertIdString = Opaque<'CertIdString', string>; -/** - * This must be a raw lowercase base16 string and not a multibase string. - * The x509 certificate will strip any non-hex characters and add padding - * to the nearest byte. - */ -type CertIdEncoded = Opaque<'CertIdEncoded', string>; - -// Nodes - -type NodeId = Opaque<'NodeId', Id>; -type NodeIdString = Opaque<'NodeIdString', string>; -type NodeIdEncoded = Opaque<'NodeIdEncoded', string>; - -// Vaults - -type VaultId = Opaque<'VaultId', Id>; -type VaultIdString = Opaque<'VaultIdString', string>; -type VaultIdEncoded = Opaque<'VaultIdEncoded', string>; - -// Tasks - -type TaskId = Opaque<'TaskId', Id>; -type TaskIdString = Opaque<'TaskIdEncoded', string>; -type TaskIdEncoded = Opaque<'TaskIdEncoded', string>; -type TaskHandlerId = Opaque<'TaskHandlerId', string>; - -// Claims - -type ClaimId = Opaque<'ClaimId', Id>; -type ClaimIdString = Opaque<'ClaimIdString', string>; -type ClaimIdEncoded = Opaque<'ClaimIdEncoded', string>; - -// Identities - -/** - * Provider Id identifies an identity provider. - * e.g. `github.com` - */ -type ProviderId = Opaque<'ProviderId', string>; - -/** - * Identity Id must uniquely identify the identity on the identity provider. - * It must be the key that is used to look up the identity. - * If the provider uses a non-string type, make the necessary conversions. - * e.g. `cmcdragonkai` - */ -type IdentityId = Opaque<'IdentityId', string>; - -/** - * Tuple of `[ProviderId, IdentityId]` - */ -type ProviderIdentityId = [ProviderId, IdentityId]; - -/** - * This is a JSON encoding of `[ProviderId, IdentityId]` - */ -type ProviderIdentityIdEncoded = Opaque<'ProviderIdentityIdEncoded', string>; - -/** - * A unique identifier for the published claim, found on the identity provider. - * e.g. the gist ID on GitHub - */ -type ProviderIdentityClaimId = Opaque<'ProviderIdentityClaimId', string>; - -// Gestalts - -/** - * Prefixed NodeId and ProviderIdentityId. - * This is done to ensure there is no chance of conflict between - * `NodeId` and `ProviderIdentityId`. - */ -type GestaltId = ['node', NodeId] | ['identity', ProviderIdentityId]; - -/** - * GestaltId encoded. - */ -type GestaltIdEncoded = Opaque<'GestaltIdEncoded', string>; - -type GestaltLinkId = Opaque<'GestaltLinkId', Id>; -type GestaltLinkIdString = Opaque<'GestaltLinkIdString', string>; - -// Notifications - -type NotificationId = Opaque<'NotificationId', Id>; -type NotificationIdString = Opaque<'NotificationIdString', string>; -type NotificationIdEncoded = Opaque<'NotificationIdEncoded', string>; - -export type { - PermissionId, - PermissionIdString, - CertId, - CertIdString, - CertIdEncoded, - NodeId, - NodeIdString, - NodeIdEncoded, - VaultId, - VaultIdString, - VaultIdEncoded, - TaskId, - TaskIdString, - TaskIdEncoded, - TaskHandlerId, - ClaimId, - ClaimIdString, - ClaimIdEncoded, - ProviderId, - IdentityId, - ProviderIdentityId, - ProviderIdentityIdEncoded, - ProviderIdentityClaimId, - GestaltId, - GestaltIdEncoded, - GestaltLinkId, - GestaltLinkIdString, - NotificationId, - NotificationIdString, - NotificationIdEncoded, -}; diff --git a/src/index.ts b/src/index.ts index eea524d6..9a61d906 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1,8 @@ -export * from "./types"; +export { default as WebSocketServer } from './WebSocketServer'; +export { default as WebSocketClient } from './WebSocketClient'; +export { default as WebSocketConnection } from './WebSocketConnection'; +export { default as WebSocketStream } from './WebSocketStream'; + +export * as utils from './utils'; +export * as events from './events'; +export * as errors from './errors'; diff --git a/src/types.ts b/src/types.ts index 1bdbea56..5b95e376 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,10 +8,11 @@ }; type TLSConfig = { - keyPrivatePem: PrivateKeyPEM; - certChainPem: CertificatePEMChain; + keyPrivatePem: string; + certChainPem: string; }; export type { - PromiseDeconstructed + PromiseDeconstructed, + TLSConfig } diff --git a/tests/WebSocket.test.ts b/tests/WebSocket.test.ts index 56564ccc..26beb96b 100644 --- a/tests/WebSocket.test.ts +++ b/tests/WebSocket.test.ts @@ -1,6 +1,4 @@ import type { ReadableWritablePair } from 'stream/web'; -import type { TLSConfig } from '@/network/types'; -import type { KeyPair } from '@/keys/types'; import type { NodeId } from '@/ids/types'; import type http from 'http'; import fs from 'fs'; @@ -14,10 +12,7 @@ import { status } from '@matrixai/async-init'; import WebSocketServer from '@/WebSocketServer'; import WebSocketClient from '@/WebSocketClient'; import { promise } from '@/utils'; -import * as keysUtils from '@/keys/utils'; import * as webSocketErrors from '@/errors'; -import * as nodesUtils from '@/nodes/utils'; -import * as testNodeUtils from '../nodes/utils'; import * as testsUtils from './utils'; // This file tests both the client and server together. They're too interlinked From 21ae525fe661cd3b59152779e0d30bcebc6f8bb1 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:23:23 +1000 Subject: [PATCH 003/149] feat: WebSocketServer and WebSocketConnection --- package.json | 1 + src/ErrorWebSocket.ts | 44 ---- src/WebSocketClient.ts | 9 +- src/WebSocketConnection.ts | 397 +++++++++++++++++++++++++++++++- src/WebSocketServer.ts | 272 +++++++++++----------- src/WebSocketStream.bak.ts | 361 +++++++++++++++++++++++++++++ src/WebSocketStream.ts | 416 +++++++--------------------------- src/config.ts | 11 + src/errors.ts | 138 ++--------- src/events.ts | 78 +++++-- src/types.ts | 99 +++++++- src/utils/errors.ts | 31 --- src/utils/index.ts | 3 +- src/utils/sysexits.ts | 91 -------- src/utils/types.ts | 12 + src/utils/utils.ts | 6 +- tests/WebSocketServer.test.ts | 16 ++ tests/utils.ts | 2 +- 18 files changed, 1192 insertions(+), 795 deletions(-) delete mode 100644 src/ErrorWebSocket.ts create mode 100644 src/WebSocketStream.bak.ts create mode 100644 src/config.ts delete mode 100644 src/utils/errors.ts delete mode 100644 src/utils/sysexits.ts create mode 100644 src/utils/types.ts create mode 100644 tests/WebSocketServer.test.ts diff --git a/package.json b/package.json index 37e47936..3d76dc37 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", "ip-num": "^1.5.0", + "resource-counter": "^1.2.4", "ws": "^8.13.0" }, "devDependencies": { diff --git a/src/ErrorWebSocket.ts b/src/ErrorWebSocket.ts deleted file mode 100644 index 1a53e540..00000000 --- a/src/ErrorWebSocket.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { Class } from '@matrixai/errors'; -import { AbstractError } from '@matrixai/errors'; -import sysexits from './utils/sysexits'; - -class ErrorWebSocket extends AbstractError { - static description: string = 'Polykey error'; - exitCode: number = sysexits.GENERAL; - - public static fromJSON>( - this: T, - json: any, - ): InstanceType { - if ( - typeof json !== 'object' || - json.type !== this.name || - typeof json.data !== 'object' || - typeof json.data.message !== 'string' || - isNaN(Date.parse(json.data.timestamp)) || - typeof json.data.description !== 'string' || - typeof json.data.data !== 'object' || - typeof json.data.exitCode !== 'number' || - ('stack' in json.data && typeof json.data.stack !== 'string') - ) { - throw new TypeError(`Cannot decode JSON to ${this.name}`); - } - const e = new this(json.data.message, { - timestamp: new Date(json.data.timestamp), - data: json.data.data, - cause: json.data.cause, - }); - e.exitCode = json.data.exitCode; - e.stack = json.data.stack; - return e; - } - - public toJSON(): any { - const json = super.toJSON(); - json.data.description = this.description; - json.data.exitCode = this.exitCode; - return json; - } -} - -export default ErrorWebSocket; diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 59f08f3e..3fd0a6b1 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -6,9 +6,10 @@ import WebSocket from 'ws'; import { Validator } from 'ip-num'; import { Timer } from '@matrixai/timer'; import WebSocketStream from './WebSocketStream'; -import * as webSocketUtils from './utils'; import * as webSocketErrors from './errors'; import { promise } from './utils'; +import { VerifyCallback } from './types'; +import WebSocketConnection from './WebSocketConnection'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() @@ -42,7 +43,7 @@ class WebSocketClient { pingIntervalTime?: number; pingTimeoutTimeTime?: number; logger?: Logger; - verifyCallback?: () => Promise; + verifyCallback?: VerifyCallback; }): Promise { logger.info(`Creating ${this.name}`); const clientClient = new this( @@ -68,7 +69,7 @@ class WebSocketClient { protected connectionTimeoutTime: number, protected pingIntervalTime: number, protected pingTimeoutTimeTime: number, - protected verifyCallback?: (peerCert: DetailedPeerCertificate) => Promise + protected verifyCallback?: VerifyCallback ) { if (Validator.isValidIPv4String(host)[0]) { this.host = host; @@ -224,7 +225,7 @@ class WebSocketClient { // Constructing the `ReadableWritablePair`, the lifecycle is handed off to // the webSocketStream at this point. - const webSocketStreamClient = new WebSocketStream( + const webSocketStreamClient = WebSocketConnection.createWebSocketConnection( ws, this.pingIntervalTime, this.pingTimeoutTimeTime, diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 643ee7fd..493e05fc 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -1,5 +1,400 @@ -class WebSocketConnection { +import { PromiseCancellable } from '@matrixai/async-cancellable'; +import { startStop } from '@matrixai/async-init'; +import { ContextTimed, ContextTimedInput } from '@matrixai/contexts'; +import { Lock } from '@matrixai/async-locks'; +import { context, timedCancellable } from '@matrixai/contexts/dist/decorators'; +import Logger from '@matrixai/logger'; +import * as ws from 'ws'; +import { Host, RemoteInfo, StreamId, VerifyCallback, WebSocketConfig } from './types'; +import WebSocketClient from './WebSocketClient'; +import WebSocketServer from './WebSocketServer'; +import WebSocketStream from './WebSocketStream'; +import Counter from 'resource-counter'; +import * as errors from './errors'; +import { promise } from './utils'; +import { Timer } from '@matrixai/timer'; +import * as events from './events'; +import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; +const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); + +/** + * Think of this as equivalent to `net.Socket`. + * This is one-to-one with the ws.WebSocket. + * Errors here are emitted to the connection only. + * Not to the server. + * + * Events (events are executed post-facto): + * - connectionStream + * - connectionStop + * - connectionError - can occur due to a timeout too + * - streamDestroy + */ +interface WebSocketConnection extends startStop.StartStop {} +@startStop.StartStop() +class WebSocketConnection extends EventTarget { + /** + * This determines when it is a client or server connection. + */ + public readonly type: 'client' | 'server'; + + /** + * This is the source connection ID. + */ + public readonly connectionId: number; + + /** + * Internal native connection object. + * @internal + */ + protected socket: ws.WebSocket; + + protected config: WebSocketConfig; + + /** + * Internal stream map. + * This is also used by `WebSocketStream`. + * @internal + */ + public readonly streamMap: Map = new Map(); + + /** + * Stream ID increment lock. + */ + protected streamIdLock: Lock = new Lock(); + + /** + * Client initiated bidirectional stream starts at 0. + * Increment by 4 to get the next ID. + */ + protected streamIdClientBidi: StreamId = 0b00 as StreamId; + + /** + * Server initiated bidirectional stream starts at 1. + * Increment by 4 to get the next ID. + */ + protected streamIdServerBidi: StreamId = 0b01 as StreamId; + + /** + * Client initiated unidirectional stream starts at 2. + * Increment by 4 to get the next ID. + * Currently unsupported. + */ + protected _streamIdClientUni: StreamId = 0b10 as StreamId; + + /** + * Server initiated unidirectional stream starts at 3. + * Increment by 4 to get the next ID. + * Currently unsupported. + */ + protected _streamIdServerUni: StreamId = 0b11 as StreamId; + + protected keepAliveTimeOutTimer?: Timer; + protected keepAliveIntervalTimer?: Timer; + + protected client?: WebSocketClient; + protected server?: WebSocketServer; + protected logger: Logger; + protected _remoteHost: Host; + + /** + * Connection closed promise. + * This can resolve or reject. + */ + protected closedP: Promise; + + protected resolveClosedP: () => void; + protected rejectClosedP: (reason?: any) => void; + + protected messageHandler = (data: ws.RawData, isBinary: boolean) => { + if (!isBinary || data instanceof Array) { + this.dispatchEvent( + new events.WebSocketConnectionErrorEvent({ + detail: new errors.ErrorWebSocketUndefinedBehaviour() + }) + ); + return; + } + } + + public static createWebSocketConnection( + args: { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }, + ctx?: Partial, + ): PromiseCancellable; + @timedCancellable( + true, + Infinity, + errors.ErrorWebSocketConnectionStartTimeOut + ) + public static async createWebSocketConnection( + args: { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }, + @context ctx: ContextTimed, + ): Promise { + // Setting up abort/cancellation logic + const abortProm = promise(); + const abortHandler = () => { + abortProm.rejectP(ctx.signal.reason); + } + ctx.signal.addEventListener('abort', abortHandler); + const connection = new this(args); + try { + await Promise.race([ + connection.start(), + abortProm.p, + ]); + } + catch (e) { + await connection.stop({ force: true }); + throw e; + } + finally { + ctx.signal.removeEventListener('abort', abortHandler); + } + if (connection.config.keepAliveIntervalTime != null) { + connection.startKeepAliveIntervalTimer( + connection.config.keepAliveIntervalTime, + ); + } + return connection; + } + public constructor({ + type, + connectionId, + remoteInfo, + config, + socket, + server, + client, + verifyCallback, + logger, + }: { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }) { + super(); + this.logger = logger ?? new Logger(`${this.constructor.name}`); + this.connectionId = connectionId; + this.socket = socket; + this.config = config; + this.type = type; + this.server = server; + this.client = client; + this._remoteHost = remoteInfo.host; + + const { + p: closedP, + resolveP: resolveClosedP, + rejectP: rejectClosedP, + } = promise(); + this.closedP = closedP; + this.resolveClosedP = resolveClosedP; + this.rejectClosedP = rejectClosedP; + + } + public async start(): Promise { + this.logger.info(`Start ${this.constructor.name}`); + const connectProm = promise(); + if (this.socket.readyState === ws.OPEN) { + connectProm.resolveP(); + } + // Handle connection failure + const openErrorHandler = (e) => { + connectProm.rejectP( + new errors.ErrorWebSocketConnection(undefined, { + cause: e, + }), + ); + }; + this.socket.once('error', openErrorHandler); + const openHandler = () => { + connectProm.resolveP(); + }; + this.socket.once('open', openHandler); + await connectProm; + this.socket.off('open', openHandler); + + // Set the connection up + if (this.type === 'server') { + this.server!.connectionMap.set(this.connectionId, this); + } + + this.socket.once('close', () => { + this.resolveClosedP(); + if (this[startStop.running] && this[startStop.status] !== 'stopping') { + void this.stop({ force: true }); + } + }); + this.socket.on('ping', () => { + this.socket.pong(); + }); + this.socket.on('pong', () => { + this.setKeepAliveTimeoutTimer(); + }); + this.socket.on('message', this.messageHandler); + + this.logger.info(`Started ${this.constructor.name}`); + } + + @ready(new errors.ErrorWebSocketConnectionNotRunning()) + public async streamNew(streamType: 'bidi' = 'bidi'): Promise { + return await this.streamIdLock.withF(async () => { + let streamId: StreamId; + if (this.type === 'client' && streamType === 'bidi') { + streamId = this.streamIdClientBidi; + } else if (this.type === 'server' && streamType === 'bidi') { + streamId = this.streamIdServerBidi; + } + const wsStream = await WebSocketStream.createWebSocketStream({ + streamId: streamId!, + connection: this, + logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), + }); + // Ok the stream is opened and working + if (this.type === 'client' && streamType === 'bidi') { + this.streamIdClientBidi = (this.streamIdClientBidi + 4) as StreamId; + } else if (this.type === 'server' && streamType === 'bidi') { + this.streamIdServerBidi = (this.streamIdServerBidi + 4) as StreamId; + } + return wsStream; + }); + } + + public async stop({ + force = false + } : { + force: boolean + }) { + this.logger.info(`Stop ${this.constructor.name}`); + // Cleaning up existing streams + // ... + this.logger.debug('triggering stream destruction'); + this.logger.debug('waiting for streams to destroy'); + this.logger.debug('streams destroyed'); + this.stopKeepAliveIntervalTimer(); + + if (this.socket.readyState === ws.CLOSED) { + this.resolveClosedP(); + } + else { + this.socket.close(); + } + await this.closedP; + this.logger.debug('closedP'); + this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); + + if (this.type === 'server') { + this.server!.connectionMap.delete(this.connectionId); + this.server!.connectionIdCounter.deallocate(this.connectionId); + } + + this.dispatchEvent(new events.WebSocketConnectionStopEvent()); + this.logger.info(`Stopped ${this.constructor.name}`); + } + + protected setKeepAliveTimeoutTimer(): void { + const logger = this.logger.getChild('timer'); + const timeout = this.config.keepAliveTimeoutTime; + const keepAliveTimeOutHandler = () => { + this.dispatchEvent(new events.WebSocketConnectionErrorEvent({ + detail: new errors.ErrorWebSocketConnectionKeepAliveTimeOut(), + })); + if (this[startStop.running] && this[startStop.status] !== 'stopping') { + void this.stop({ force: true }); + } + }; + // If there was an existing timer, we cancel it and set a new one + if ( + this.keepAliveTimeOutTimer != null && + this.keepAliveTimeOutTimer.status === null + ) { + logger.debug(`resetting timer with ${timeout} delay`); + this.keepAliveTimeOutTimer.reset(timeout); + } else { + logger.debug(`timeout created with delay ${timeout}`); + this.keepAliveTimeOutTimer = new Timer({ + delay: timeout, + handler: keepAliveTimeOutHandler, + }); + } + } + + protected startKeepAliveIntervalTimer(ms: number): void { + const keepAliveHandler = async () => { + this.socket.ping(); + this.keepAliveIntervalTimer = new Timer({ + delay: ms, + handler: keepAliveHandler, + }); + }; + this.keepAliveIntervalTimer = new Timer({ + delay: ms, + handler: keepAliveHandler, + }); + } + + /** + * Stops the keep alive interval timer + */ + protected stopKeepAliveIntervalTimer(): void { + this.keepAliveIntervalTimer?.cancel(timerCleanupReasonSymbol); + } } export default WebSocketConnection; diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 4b53b054..f806ba66 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -4,13 +4,14 @@ import https from 'https'; import { startStop, status } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; -import WebSocketStream from './WebSocketStream'; -import * as webSocketErrors from './errors'; +import * as errors from './errors'; import * as webSocketEvents from './events'; import { never, promise } from './utils'; -import { TLSConfig } from './types'; - -type ConnectionCallback = (streamPair: WebSocketStream) => void; +import { Host, Port, WebSocketConfig } from './types'; +import WebSocketConnection from './WebSocketConnection'; +import Counter from 'resource-counter'; +import { serverDefault } from './config'; +import * as utils from './utils'; /** * Events: @@ -21,104 +22,70 @@ type ConnectionCallback = (streamPair: WebSocketStream) => void; interface WebSocketServer extends startStop.StartStop {} @startStop.StartStop() class WebSocketServer extends EventTarget { - /** - * @param obj - * @param obj.connectionCallback - - * @param obj.tlsConfig - TLSConfig containing the private key and cert chain used for TLS. - * @param obj.host - Listen address to bind to. - * @param obj.port - Listen port to bind to. - * @param obj.maxIdleTimeout - Timeout time for when the connection is cleaned up after no activity. - * Default is 120 seconds. - * @param obj.pingIntervalTime - Time between pings for checking connection health and keep alive. - * Default is 1,000 milliseconds. - * @param obj.pingTimeoutTimeTime - Time before connection is cleaned up after no ping responses. - * Default is 10,000 milliseconds. - * @param obj.logger - */ - static async createWebSocketServer({ - connectionCallback, - tlsConfig, - host, - port, - maxIdleTimeout = 120, - pingIntervalTime = 1_000, - pingTimeoutTimeTime = 10_000, - logger = new Logger(this.name), - }: { - connectionCallback: ConnectionCallback; - tlsConfig: TLSConfig; - host?: string; - port?: number; - maxIdleTimeout?: number; - pingIntervalTime?: number; - pingTimeoutTimeTime?: number; - logger?: Logger; - }) { - logger.info(`Creating ${this.name}`); - const wsServer = new this( - logger, - maxIdleTimeout, - pingIntervalTime, - pingTimeoutTimeTime, - ); - await wsServer.start({ - connectionCallback, - tlsConfig, - host, - port, - }); - logger.info(`Created ${this.name}`); - return wsServer; - } - + protected logger: Logger; + protected config: WebSocketConfig; protected server: https.Server; protected webSocketServer: ws.WebSocketServer; protected _port: number; protected _host: string; - protected connectionEventHandler: ( - event: webSocketEvents.ConnectionEvent, - ) => void; - protected activeSockets: Set = new Set(); + public readonly connectionIdCounter = new Counter(0); + public readonly connectionMap: Map = new Map(); + + protected handleWebSocketConnectionEvents = ( + event: webSocketEvents.WebSocketConnectionEvent, + ) => { + if (event instanceof webSocketEvents.WebSocketConnectionErrorEvent) { + this.dispatchEvent( + new webSocketEvents.WebSocketConnectionErrorEvent({ + detail: event.detail, + }), + ); + } else if (event instanceof webSocketEvents.WebSocketConnectionStopEvent) { + this.dispatchEvent(new webSocketEvents.WebSocketConnectionStopEvent()); + } else if (event instanceof webSocketEvents.WebSocketConnectionStreamEvent) { + this.dispatchEvent( + new webSocketEvents.WebSocketConnectionStreamEvent({ detail: event.detail }), + ); + } else { + utils.never(); + } + }; /** * * @param logger - * @param maxIdleTimeout - * @param pingIntervalTime - * @param pingTimeoutTimeTime + * @param config */ - constructor( - protected logger: Logger, - protected maxIdleTimeout: number | undefined, - protected pingIntervalTime: number, - protected pingTimeoutTimeTime: number, - ) { + constructor({ + config, + logger, + }: { + config: Partial & { + key: string, + cert: string, + }; + logger?: Logger, + }) { super(); + const wsConfig = { + ...serverDefault, + ...config, + }; + this.logger = logger ?? new Logger(this.constructor.name); + this.config = wsConfig; } public async start({ - tlsConfig, host, port = 0, - connectionCallback, }: { - tlsConfig: TLSConfig; host?: string; port?: number; - connectionCallback?: ConnectionCallback; - }): Promise { + } = {}): Promise { this.logger.info(`Starting ${this.constructor.name}`); - if (connectionCallback != null) { - this.connectionEventHandler = ( - event: webSocketEvents.ConnectionEvent, - ) => { - connectionCallback(event.detail.webSocketStream); - }; - this.addEventListener('connection', this.connectionEventHandler); - } this.server = https.createServer({ - key: tlsConfig.keyPrivatePem, - cert: tlsConfig.certChainPem, + ...this.config, + requestTimeout: this.config.connectTimeoutTime }); this.webSocketServer = new ws.WebSocketServer({ server: this.server, @@ -132,7 +99,7 @@ class WebSocketServer extends EventTarget { this.server.on('request', this.requestHandler); const listenProm = promise(); - this.server.listen(port ?? 0, host, listenProm.resolveP); + this.server.listen(port, host, listenProm.resolveP); await listenProm.p; const address = this.server.address(); if (address == null || typeof address === 'string') never(); @@ -140,29 +107,28 @@ class WebSocketServer extends EventTarget { this.logger.debug(`Listening on port ${this._port}`); this._host = address.address ?? '127.0.0.1'; this.dispatchEvent( - new webSocketEvents.StartEvent({ - detail: { - host: this._host, - port: this._port, - }, - }), + new webSocketEvents.WebSocketServerStartEvent(), ); this.logger.info(`Started ${this.constructor.name}`); } - public async stop(force: boolean = false): Promise { + public async stop({ + force = false, + }: { + force?: boolean; + }): Promise { this.logger.info(`Stopping ${this.constructor.name}`); - // Shutting down active websockets - if (force) { - for (const webSocketStream of this.activeSockets) { - webSocketStream.cancel(); - } - } - // Wait for all active websockets to close - for (const webSocketStream of this.activeSockets) { - // Ignore errors, we only care that it finished - webSocketStream.endedProm.catch(() => {}); + const destroyProms: Array> = []; + for (const webSocketConnection of this.connectionMap.values()) { + destroyProms.push( + webSocketConnection.stop({ + force + }) + ); } + this.logger.debug('Awaiting connections to destroy'); + await Promise.all(destroyProms); + this.logger.debug('All connections destroyed'); // Close the server by closing the underlying socket const wssCloseProm = promise(); this.webSocketServer.close((e) => { @@ -182,10 +148,6 @@ class WebSocketServer extends EventTarget { } }); await serverCloseProm.p; - // Removing handlers - if (this.connectionEventHandler != null) { - this.removeEventListener('connection', this.connectionEventHandler); - } this.webSocketServer.off('connection', this.connectionHandler); this.webSocketServer.off('close', this.closeHandler); @@ -194,66 +156,98 @@ class WebSocketServer extends EventTarget { this.server.off('error', this.errorHandler); this.server.on('request', this.requestHandler); - this.dispatchEvent(new webSocketEvents.StopEvent()); + this.dispatchEvent(new webSocketEvents.WebSocketServerStopEvent()); this.logger.info(`Stopped ${this.constructor.name}`); } - @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) + @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) public getPort(): number { return this._port; } - @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) + @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) public getHost(): string { return this._host; } - @startStop.ready(new webSocketErrors.ErrorWebSocketServerNotRunning()) - public setTlsConfig(tlsConfig: TLSConfig): void { + @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) + public updateConfig(config: Partial & { + key: string, + cert: string, + }): void { const tlsServer = this.server as tls.Server; tlsServer.setSecureContext({ - key: tlsConfig.keyPrivatePem, - cert: tlsConfig.certChainPem, + key: config.key, + cert: config.cert, }); + const wsConfig = { + ...this.config, + ...config, + }; + this.config = wsConfig; } /** * Handles the creation of the `ReadableWritablePair` and provides it to the * StreamPair handler. */ - protected connectionHandler = ( + protected connectionHandler = async ( webSocket: ws.WebSocket, request: IncomingMessage, ) => { - const connection = request.connection; - const webSocketStream = new WebSocketStream( - webSocket, - this.pingIntervalTime, - this.pingTimeoutTimeTime, - { - localHost: connection.localAddress ?? '', - localPort: connection.localPort ?? 0, - remoteHost: connection.remoteAddress ?? '', - remotePort: connection.remotePort ?? 0, + const httpSocket = request.connection; + const connectionId = this.connectionIdCounter.allocate(); + const connection = await WebSocketConnection.createWebSocketConnection({ + type: 'server', + connectionId: connectionId, + remoteInfo: { + host: (httpSocket.remoteAddress ?? '') as Host, + port: (httpSocket.remotePort ?? 0) as Port, + }, + config: this.config, + socket: webSocket, + logger: this.logger.getChild( + `${WebSocketConnection.name} ${connectionId}` + ), + server: this, + }); + + // Handling connection events + connection.addEventListener( + 'connectionError', + this.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'connectionStream', + this.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'streamDestroy', + this.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'connectionStop', + (event) => { + connection.removeEventListener( + 'connectionError', + this.handleWebSocketConnectionEvents, + ); + connection.removeEventListener( + 'connectionStream', + this.handleWebSocketConnectionEvents, + ); + connection.removeEventListener( + 'streamDestroy', + this.handleWebSocketConnectionEvents, + ); + this.handleWebSocketConnectionEvents(event); }, - this.logger.getChild(WebSocketStream.name), + { once: true }, ); - // Adding socket to the active sockets map - this.activeSockets.add(webSocketStream); - void webSocketStream.endedProm - // Ignore errors, we only care that it finished - .catch(() => {}) - .finally(() => { - this.activeSockets.delete(webSocketStream); - }); - // There is not nodeId or certs for the client, and we can't get the remote - // port from the `uWebsocket` library. this.dispatchEvent( - new webSocketEvents.ConnectionEvent({ - detail: { - webSocketStream, - }, + new webSocketEvents.WebSocketServerConnectionEvent({ + detail: connection }), ); }; @@ -267,7 +261,7 @@ class WebSocketServer extends EventTarget { return; } this.logger.debug('close event, forcing stop'); - await this.stop(true); + await this.stop({ force: true }); }; /** diff --git a/src/WebSocketStream.bak.ts b/src/WebSocketStream.bak.ts new file mode 100644 index 00000000..db3175fb --- /dev/null +++ b/src/WebSocketStream.bak.ts @@ -0,0 +1,361 @@ +import type { ReadableWritablePair } from 'stream/web'; +import type { + ReadableStreamController, + WritableStreamDefaultController, +} from 'stream/web'; +import type * as ws from 'ws'; +import type Logger from '@matrixai/logger'; +import { WritableStream, ReadableStream } from 'stream/web'; +import * as webSocketErrors from './errors'; +import * as utilsErrors from './utils/errors'; +import { promise } from './utils'; +import WebSocketConnection from './WebSocketConnection'; + +class WebSocketStream implements ReadableWritablePair { + public readable: ReadableStream; + public writable: WritableStream; + + protected _readableEnded = false; + protected _readableEndedProm = promise(); + protected _writableEnded = false; + protected _writableEndedProm = promise(); + protected _webSocketEnded = false; + protected _webSocketEndedProm = promise(); + protected _endedProm: Promise; + + protected readableController: + | ReadableStreamController + | undefined; + protected writableController: WritableStreamDefaultController | undefined; + + constructor( + protected connection: WebSocketConnection, + pingInterval: number, + pingTimeoutTime: number, + protected metadata: { + localHost: string; + localPort: number; + remoteHost: string; + remotePort: number; + }, + logger: Logger, + ) { + // Sanitise promises so they don't result in unhandled rejections + this._readableEndedProm.p.catch(() => {}); + this._writableEndedProm.p.catch(() => {}); + this._webSocketEndedProm.p.catch(() => {}); + // Creating the endedPromise + this._endedProm = Promise.allSettled([ + this._readableEndedProm.p, + this._writableEndedProm.p, + this._webSocketEndedProm.p, + ]).then((result) => { + if ( + result[0].status === 'rejected' || + result[1].status === 'rejected' || + result[2].status === 'rejected' + ) { + // Throw a compound error + throw AggregateError(result, 'stream failed'); + } + // Otherwise return nothing + }); + // Ignore errors if it's never used + this._endedProm.catch(() => {}); + + logger.info('WS opened'); + const readableLogger = logger.getChild('readable'); + const writableLogger = logger.getChild('writable'); + // Setting up the readable stream + this.readable = new ReadableStream( + { + start: (controller) => { + readableLogger.debug('Starting'); + this.readableController = controller; + const messageHandler = (data: ws.RawData, isBinary: boolean) => { + if (!isBinary || data instanceof Array) { + controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); + return; + } + const message = data as Buffer; + readableLogger.debug(`Received ${message.toString()}`); + if (message.length === 0) { + readableLogger.debug('Null message received'); + connection.socket.removeListener('message', messageHandler); + if (!this._readableEnded) { + readableLogger.debug('Closing'); + this.signalReadableEnd(); + controller.close(); + } + if (this._writableEnded) { + logger.debug('Closing socket'); + connection.socket.close(); + } + return; + } + if (this._readableEnded) { + return; + } + controller.enqueue(message); + if (controller.desiredSize == null) { + controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); + return; + } + if (controller.desiredSize < 0) { + readableLogger.debug('Applying readable backpressure'); + connection.socket.pause(); + } + }; + readableLogger.debug('Registering socket message handler'); + connection.socket.on('message', messageHandler); + connection.socket.once('close', (code, reason) => { + logger.info('Socket closed'); + connection.socket.removeListener('message', messageHandler); + if (!this._readableEnded) { + readableLogger.debug( + `Closed early, ${code}, ${reason.toString()}`, + ); + const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); + this.signalReadableEnd(e); + controller.error(e); + } + }); + ws.once('error', (e) => { + if (!this._readableEnded) { + readableLogger.error(e); + this.signalReadableEnd(e); + controller.error(e); + } + }); + }, + cancel: (reason) => { + readableLogger.debug('Cancelled'); + this.signalReadableEnd(reason); + if (this._writableEnded) { + readableLogger.debug('Closing socket'); + this.signalWritableEnd(reason); + ws.close(); + } + }, + pull: () => { + readableLogger.debug('Releasing backpressure'); + ws.resume(); + }, + }, + { highWaterMark: 1 }, + ); + this.writable = new WritableStream( + { + start: (controller) => { + this.writableController = controller; + writableLogger.info('Starting'); + ws.once('error', (e) => { + if (!this._writableEnded) { + writableLogger.error(e); + this.signalWritableEnd(e); + controller.error(e); + } + }); + ws.once('close', (code, reason) => { + if (!this._writableEnded) { + writableLogger.debug( + `Closed early, ${code}, ${reason.toString()}`, + ); + const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); + this.signalWritableEnd(e); + controller.error(e); + } + }); + }, + close: async () => { + writableLogger.debug('Closing, sending null message'); + const sendProm = promise(); + ws.send(Buffer.from([]), (err) => { + if (err == null) sendProm.resolveP(); + else sendProm.rejectP(err); + }); + await sendProm.p; + this.signalWritableEnd(); + if (this._readableEnded) { + writableLogger.debug('Closing socket'); + ws.close(); + } + }, + abort: (reason) => { + writableLogger.debug('Aborted'); + this.signalWritableEnd(reason); + if (this._readableEnded) { + writableLogger.debug('Closing socket'); + ws.close(4000, `Aborting connection with ${reason.message}`); + } + }, + write: async (chunk, controller) => { + if (this._writableEnded) return; + writableLogger.debug(`Sending ${chunk?.toString()}`); + const wait = promise(); + ws.send(chunk, (e) => { + if (e != null && !this._writableEnded) { + // Opting to debug message here and not log an error, sending + // failure is common if we send before the close event. + writableLogger.debug('failed to send'); + const err = new webSocketErrors.ErrorClientConnectionEndedEarly( + undefined, + { + cause: e, + }, + ); + this.signalWritableEnd(err); + controller.error(err); + } + wait.resolveP(); + }); + await wait.p; + }, + }, + { highWaterMark: 1 }, + ); + + // Setting up heartbeat + const pingTimer = setInterval(() => { + ws.ping(); + }, pingInterval); + const pingTimeoutTimeTimer = setTimeout(() => { + logger.debug('Ping timed out'); + ws.close(4002, 'Timed out'); + }, pingTimeoutTime); + const pingHandler = () => { + logger.debug('Received ping'); + ws.pong(); + }; + const pongHandler = () => { + logger.debug('Received pong'); + pingTimeoutTimeTimer.refresh(); + }; + ws.on('ping', pingHandler); + ws.on('pong', pongHandler); + ws.once('close', (code, reason) => { + ws.off('ping', pingHandler); + ws.off('pong', pongHandler); + logger.debug('WebSocket closed'); + const err = + code !== 1000 + ? new webSocketErrors.ErrorClientConnectionEndedEarly( + `ended with code ${code}, ${reason.toString()}`, + ) + : undefined; + this.signalWebSocketEnd(err); + logger.debug('Cleaning up timers'); + // Clean up timers + clearTimeout(pingTimer); + clearTimeout(pingTimeoutTimeTimer); + }); + } + + get readableEnded() { + return this._readableEnded; + } + + /** + * Resolves when the readable has ended and rejects with any errors. + */ + get readableEndedProm() { + return this._readableEndedProm.p; + } + + get writableEnded() { + return this._writableEnded; + } + + /** + * Resolves when the writable has ended and rejects with any errors. + */ + get writableEndedProm() { + return this._writableEndedProm.p; + } + + get webSocketEnded() { + return this._webSocketEnded; + } + + /** + * Resolves when the webSocket has ended and rejects with any errors. + */ + get webSocketEndedProm() { + return this._webSocketEndedProm.p; + } + + get ended() { + return this._readableEnded && this._writableEnded; + } + + /** + * Resolves when the stream has fully closed + */ + get endedProm(): Promise { + return this._endedProm; + } + + get meta() { + // Spreading to avoid modifying the data + return { + ...this.metadata, + }; + } + + /** + * Forces the active stream to end early + */ + public cancel(reason?: any): void { + // Default error + const err = reason ?? new webSocketErrors.ErrorClientConnectionEndedEarly(); + // Close the streams with the given error, + if (!this._readableEnded) { + this.readableController?.error(err); + this.signalReadableEnd(err); + } + if (!this._writableEnded) { + this.writableController?.error(err); + this.signalWritableEnd(err); + } + // Then close the websocket + if (!this._webSocketEnded) { + this.ws.close(4000, 'Ending connection'); + this.signalWebSocketEnd(err); + } + } + + /** + * Signals the end of the ReadableStream. to be used with the extended class + * to track the streams state. + */ + protected signalReadableEnd(reason?: any) { + if (this._readableEnded) return; + this._readableEnded = true; + if (reason == null) this._readableEndedProm.resolveP(); + else this._readableEndedProm.rejectP(reason); + } + + /** + * Signals the end of the WritableStream. to be used with the extended class + * to track the streams state. + */ + protected signalWritableEnd(reason?: any) { + if (this._writableEnded) return; + this._writableEnded = true; + if (reason == null) this._writableEndedProm.resolveP(); + else this._writableEndedProm.rejectP(reason); + } + + /** + * Signals the end of the WebSocket. to be used with the extended class + * to track the streams state. + */ + protected signalWebSocketEnd(reason?: any) { + if (this._webSocketEnded) return; + this._webSocketEnded = true; + if (reason == null) this._webSocketEndedProm.resolveP(); + else this._webSocketEndedProm.rejectP(reason); + } +} + +export default WebSocketStream; diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 721bf5a4..1f1e1f1f 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,359 +1,103 @@ -import type { ReadableWritablePair } from 'stream/web'; -import type { - ReadableStreamController, - WritableStreamDefaultController, -} from 'stream/web'; -import type * as ws from 'ws'; -import type Logger from '@matrixai/logger'; -import { WritableStream, ReadableStream } from 'stream/web'; -import * as webSocketErrors from './errors'; -import * as utilsErrors from './utils/errors'; -import { promise } from './utils'; - -class WebSocketStream implements ReadableWritablePair { - public readable: ReadableStream; - public writable: WritableStream; - - protected _readableEnded = false; - protected _readableEndedProm = promise(); - protected _writableEnded = false; - protected _writableEndedProm = promise(); - protected _webSocketEnded = false; - protected _webSocketEndedProm = promise(); - protected _endedProm: Promise; - +import { CreateDestroy } from "@matrixai/async-init/dist/CreateDestroy"; +import Logger from "@matrixai/logger"; +import { } from "stream/web"; +import { StreamId } from "./types"; +import WebSocketConnection from "./WebSocketConnection"; + +interface WebSocketStream extends CreateDestroy {} +@CreateDestroy() +class WebSocketStream + extends EventTarget + implements ReadableWritablePair +{ + public streamId: StreamId; + public readable: ReadableStream; + public writable: WritableStream; + + protected logger: Logger; + protected connection: WebSocketConnection; protected readableController: - | ReadableStreamController - | undefined; + | ReadableStreamController + | undefined; protected writableController: WritableStreamDefaultController | undefined; - constructor( - protected ws: ws.WebSocket, - pingInterval: number, - pingTimeoutTime: number, - protected metadata: { - localHost: string; - localPort: number; - remoteHost: string; - remotePort: number; - }, - logger: Logger, - ) { - // Sanitise promises so they don't result in unhandled rejections - this._readableEndedProm.p.catch(() => {}); - this._writableEndedProm.p.catch(() => {}); - this._webSocketEndedProm.p.catch(() => {}); - // Creating the endedPromise - this._endedProm = Promise.allSettled([ - this._readableEndedProm.p, - this._writableEndedProm.p, - this._webSocketEndedProm.p, - ]).then((result) => { - if ( - result[0].status === 'rejected' || - result[1].status === 'rejected' || - result[2].status === 'rejected' - ) { - // Throw a compound error - throw AggregateError(result, 'stream failed'); - } - // Otherwise return nothing + + public static async createWebSocketStream({ + streamId, + connection, + logger = new Logger(`${this.name} ${streamId}`) + }: { + streamId: StreamId; + connection: WebSocketConnection; + logger: Logger; + }): Promise { + logger.info(`Create ${this.name}`); + const stream = new this({ + streamId, + connection, + logger }); - // Ignore errors if it's never used - this._endedProm.catch(() => {}); + connection.streamMap.set(streamId, stream); + logger.info(`Created ${this.name}`); + return stream; + } - logger.info('WS opened'); - const readableLogger = logger.getChild('readable'); - const writableLogger = logger.getChild('writable'); - // Setting up the readable stream - this.readable = new ReadableStream( + constructor({ + streamId, + connection, + logger + }: { + streamId: StreamId; + connection: WebSocketConnection; + logger: Logger; + }) { + super(); + this.streamId = streamId; + this.connection = connection; + this.logger = logger; + + this.readable = new ReadableStream( { start: (controller) => { - readableLogger.debug('Starting'); - this.readableController = controller; - const messageHandler = (data: ws.RawData, isBinary: boolean) => { - if (!isBinary || data instanceof Array) { - controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); - return; - } - const message = data as Buffer; - readableLogger.debug(`Received ${message.toString()}`); - if (message.length === 0) { - readableLogger.debug('Null message received'); - ws.removeListener('message', messageHandler); - if (!this._readableEnded) { - readableLogger.debug('Closing'); - this.signalReadableEnd(); - controller.close(); - } - if (this._writableEnded) { - logger.debug('Closing socket'); - ws.close(); - } - return; - } - if (this._readableEnded) { - return; - } - controller.enqueue(message); - if (controller.desiredSize == null) { - controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); - return; - } - if (controller.desiredSize < 0) { - readableLogger.debug('Applying readable backpressure'); - ws.pause(); - } - }; - readableLogger.debug('Registering socket message handler'); - ws.on('message', messageHandler); - ws.once('close', (code, reason) => { - logger.info('Socket closed'); - ws.removeListener('message', messageHandler); - if (!this._readableEnded) { - readableLogger.debug( - `Closed early, ${code}, ${reason.toString()}`, - ); - const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); - this.signalReadableEnd(e); - controller.error(e); - } - }); - ws.once('error', (e) => { - if (!this._readableEnded) { - readableLogger.error(e); - this.signalReadableEnd(e); - controller.error(e); - } - }); - }, - cancel: (reason) => { - readableLogger.debug('Cancelled'); - this.signalReadableEnd(reason); - if (this._writableEnded) { - readableLogger.debug('Closing socket'); - this.signalWritableEnd(reason); - ws.close(); - } + }, - pull: () => { - readableLogger.debug('Releasing backpressure'); - ws.resume(); + pull: async (controller) => { }, + cancel: async (reason) => { + + } }, - { highWaterMark: 1 }, + new CountQueuingStrategy({ + // Allow 1 buffered message, so we can know when data is desired, and we can know when to un-pause. + highWaterMark: 1, + }) ); - this.writable = new WritableStream( + + this.writable = new WritableStream( { start: (controller) => { - this.writableController = controller; - writableLogger.info('Starting'); - ws.once('error', (e) => { - if (!this._writableEnded) { - writableLogger.error(e); - this.signalWritableEnd(e); - controller.error(e); - } - }); - ws.once('close', (code, reason) => { - if (!this._writableEnded) { - writableLogger.debug( - `Closed early, ${code}, ${reason.toString()}`, - ); - const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); - this.signalWritableEnd(e); - controller.error(e); - } - }); - }, - close: async () => { - writableLogger.debug('Closing, sending null message'); - const sendProm = promise(); - ws.send(Buffer.from([]), (err) => { - if (err == null) sendProm.resolveP(); - else sendProm.rejectP(err); - }); - await sendProm.p; - this.signalWritableEnd(); - if (this._readableEnded) { - writableLogger.debug('Closing socket'); - ws.close(); - } + }, - abort: (reason) => { - writableLogger.debug('Aborted'); - this.signalWritableEnd(reason); - if (this._readableEnded) { - writableLogger.debug('Closing socket'); - ws.close(4000, `Aborting connection with ${reason.message}`); - } + write: async (chunk: Uint8Array, controller) => { + }, - write: async (chunk, controller) => { - if (this._writableEnded) return; - writableLogger.debug(`Sending ${chunk?.toString()}`); - const wait = promise(); - ws.send(chunk, (e) => { - if (e != null && !this._writableEnded) { - // Opting to debug message here and not log an error, sending - // failure is common if we send before the close event. - writableLogger.debug('failed to send'); - const err = new webSocketErrors.ErrorClientConnectionEndedEarly( - undefined, - { - cause: e, - }, - ); - this.signalWritableEnd(err); - controller.error(err); - } - wait.resolveP(); - }); - await wait.p; + close: async () => { + }, + abort: async (reason?: any) => { + + } }, - { highWaterMark: 1 }, + { + highWaterMark: 0, + } ); - - // Setting up heartbeat - const pingTimer = setInterval(() => { - ws.ping(); - }, pingInterval); - const pingTimeoutTimeTimer = setTimeout(() => { - logger.debug('Ping timed out'); - ws.close(4002, 'Timed out'); - }, pingTimeoutTime); - const pingHandler = () => { - logger.debug('Received ping'); - ws.pong(); - }; - const pongHandler = () => { - logger.debug('Received pong'); - pingTimeoutTimeTimer.refresh(); - }; - ws.on('ping', pingHandler); - ws.on('pong', pongHandler); - ws.once('close', (code, reason) => { - ws.off('ping', pingHandler); - ws.off('pong', pongHandler); - logger.debug('WebSocket closed'); - const err = - code !== 1000 - ? new webSocketErrors.ErrorClientConnectionEndedEarly( - `ended with code ${code}, ${reason.toString()}`, - ) - : undefined; - this.signalWebSocketEnd(err); - logger.debug('Cleaning up timers'); - // Clean up timers - clearTimeout(pingTimer); - clearTimeout(pingTimeoutTimeTimer); - }); - } - - get readableEnded() { - return this._readableEnded; - } - - /** - * Resolves when the readable has ended and rejects with any errors. - */ - get readableEndedProm() { - return this._readableEndedProm.p; - } - - get writableEnded() { - return this._writableEnded; - } - - /** - * Resolves when the writable has ended and rejects with any errors. - */ - get writableEndedProm() { - return this._writableEndedProm.p; - } - - get webSocketEnded() { - return this._webSocketEnded; - } - - /** - * Resolves when the webSocket has ended and rejects with any errors. - */ - get webSocketEndedProm() { - return this._webSocketEndedProm.p; - } - - get ended() { - return this._readableEnded && this._writableEnded; - } - - /** - * Resolves when the stream has fully closed - */ - get endedProm(): Promise { - return this._endedProm; - } - - get meta() { - // Spreading to avoid modifying the data - return { - ...this.metadata, - }; - } - - /** - * Forces the active stream to end early - */ - public cancel(reason?: any): void { - // Default error - const err = reason ?? new webSocketErrors.ErrorClientConnectionEndedEarly(); - // Close the streams with the given error, - if (!this._readableEnded) { - this.readableController?.error(err); - this.signalReadableEnd(err); - } - if (!this._writableEnded) { - this.writableController?.error(err); - this.signalWritableEnd(err); - } - // Then close the websocket - if (!this._webSocketEnded) { - this.ws.close(4000, 'Ending connection'); - this.signalWebSocketEnd(err); - } - } - - /** - * Signals the end of the ReadableStream. to be used with the extended class - * to track the streams state. - */ - protected signalReadableEnd(reason?: any) { - if (this._readableEnded) return; - this._readableEnded = true; - if (reason == null) this._readableEndedProm.resolveP(); - else this._readableEndedProm.rejectP(reason); - } - - /** - * Signals the end of the WritableStream. to be used with the extended class - * to track the streams state. - */ - protected signalWritableEnd(reason?: any) { - if (this._writableEnded) return; - this._writableEnded = true; - if (reason == null) this._writableEndedProm.resolveP(); - else this._writableEndedProm.rejectP(reason); } - /** - * Signals the end of the WebSocket. to be used with the extended class - * to track the streams state. - */ - protected signalWebSocketEnd(reason?: any) { - if (this._webSocketEnded) return; - this._webSocketEnded = true; - if (reason == null) this._webSocketEndedProm.resolveP(); - else this._webSocketEndedProm.rejectP(reason); + public async destroy() { + this.logger.info(`Destroy ${this.constructor.name}`); + this.connection.streamMap.delete(this.streamId); + this.logger.info(`Destroyed ${this.constructor.name}`); } } diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 00000000..edea1ac3 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,11 @@ +import { WebSocketConfig } from "./types"; + +const serverDefault: WebSocketConfig = { + connectTimeoutTime: 120, + keepAliveIntervalTime: 1_000, + keepAliveTimeoutTime: 10_000, +} + +export { + serverDefault, +} diff --git a/src/errors.ts b/src/errors.ts index 26ed1cf9..72ea4d03 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,138 +1,44 @@ -import ErrorWebSocket from './ErrorWebSocket'; -import { sysexits } from './utils'; +import { AbstractError } from '@matrixai/errors'; -class ErrorWebSocketClient extends ErrorWebSocket {} - -class ErrorClientDestroyed extends ErrorWebSocketClient { - static description = 'ClientClient has been destroyed'; - exitCode = sysexits.USAGE; -} - -class ErrorClientInvalidHost extends ErrorWebSocketClient { - static description = 'Host must be a valid IPv4 or IPv6 address string'; - exitCode = sysexits.USAGE; -} - -class ErrorClientConnectionFailed extends ErrorWebSocketClient { - static description = 'Failed to establish connection to server'; - exitCode = sysexits.UNAVAILABLE; -} - -class ErrorClientConnectionTimedOut extends ErrorWebSocketClient { - static description = 'Connection timed out'; - exitCode = sysexits.UNAVAILABLE; +class ErrorWebSocket extends AbstractError { + static description = 'WebSocket error'; } -class ErrorClientConnectionEndedEarly extends ErrorWebSocketClient { - static description = 'Connection ended before stream ended'; - exitCode = sysexits.UNAVAILABLE; +class ErrorWebSocketServer extends ErrorWebSocket { + static description = 'WebSocket Server error'; } -class ErrorClientStreamAborted extends ErrorWebSocketClient { - static description = 'Stream was ended early with an abort signal'; - exitCode = sysexits.USAGE; -} - -class ErrorClientEndingConnections extends ErrorWebSocketClient { - static description = 'WebSocketClient is ending active connections'; - exitCode = sysexits.USAGE; -} - -class ErrorWebSocketServer extends ErrorWebSocket {} - class ErrorWebSocketServerNotRunning extends ErrorWebSocketServer { - static description = 'WebSocketServer is not running'; - exitCode = sysexits.USAGE; -} - -class ErrorServerPortUnavailable extends ErrorWebSocketServer { - static description = 'Failed to bind a free port'; - exitCode = sysexits.UNAVAILABLE; + static description = 'WebSocket Server is not running'; } -class ErrorServerSendFailed extends ErrorWebSocketServer { - static description = 'Failed to send message'; - exitCode = sysexits.UNAVAILABLE; +class ErrorWebSocketConnection extends ErrorWebSocket { + static description = 'WebSocket Connection error'; } -class ErrorServerReadableBufferLimit extends ErrorWebSocketServer { - static description = 'Readable buffer is full, messages received too quickly'; - exitCode = sysexits.USAGE; +class ErrorWebSocketConnectionNotRunning extends ErrorWebSocketConnection { + static description = 'WebSocket Connection is not running'; } -class ErrorServerConnectionEndedEarly extends ErrorWebSocketServer { - static description = 'Connection ended before stream ended'; - exitCode = sysexits.UNAVAILABLE; +class ErrorWebSocketConnectionStartTimeOut extends ErrorWebSocketConnection { + static description = 'WebSocket Connection start timeout'; } -/** - * Used for certificate verification - */ -class ErrorCertChain extends ErrorWebSocket {} - -class ErrorCertChainEmpty extends ErrorCertChain { - static description = 'Certificate chain is empty'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorCertChainUnclaimed extends ErrorCertChain { - static description = 'The target node id is not claimed by any certificate'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorCertChainBroken extends ErrorCertChain { - static description = 'The signature chain is broken'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorCertChainDateInvalid extends ErrorCertChain { - static description = 'Certificate in the chain is expired'; - exitCode = sysexits.PROTOCOL; +class ErrorWebSocketConnectionKeepAliveTimeOut extends ErrorWebSocketConnection { + static description = 'WebSocket Connection reached idle timeout'; } -class ErrorCertChainNameInvalid extends ErrorCertChain { - static description = 'Certificate is missing the common name'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorCertChainKeyInvalid extends ErrorCertChain { - static description = 'Certificate public key does not generate the Node ID'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorCertChainSignatureInvalid extends ErrorCertChain { - static description = 'Certificate self-signed signature is invalid'; - exitCode = sysexits.PROTOCOL; -} - -class ErrorConnectionNodesEmpty extends ErrorWebSocket { - static description = 'Nodes list to verify against was empty'; - exitCode = sysexits.USAGE; +class ErrorWebSocketUndefinedBehaviour extends ErrorWebSocket { + static description = 'This should never happen'; } export { - ErrorWebSocketClient, - ErrorClientDestroyed, - ErrorClientInvalidHost, - ErrorClientConnectionFailed, - ErrorClientConnectionTimedOut, - ErrorClientConnectionEndedEarly, - ErrorClientStreamAborted, - ErrorClientEndingConnections, + ErrorWebSocket, ErrorWebSocketServer, ErrorWebSocketServerNotRunning, - ErrorServerPortUnavailable, - ErrorServerSendFailed, - ErrorServerReadableBufferLimit, - ErrorServerConnectionEndedEarly, - ErrorCertChainEmpty, - ErrorCertChainUnclaimed, - ErrorCertChainBroken, - ErrorCertChainDateInvalid, - ErrorCertChainNameInvalid, - ErrorCertChainKeyInvalid, - ErrorCertChainSignatureInvalid, - ErrorConnectionNodesEmpty, + ErrorWebSocketConnection, + ErrorWebSocketConnectionNotRunning, + ErrorWebSocketConnectionStartTimeOut, + ErrorWebSocketConnectionKeepAliveTimeOut, + ErrorWebSocketUndefinedBehaviour }; - -export * from './utils/errors'; diff --git a/src/events.ts b/src/events.ts index fc70e825..042614b3 100644 --- a/src/events.ts +++ b/src/events.ts @@ -1,43 +1,79 @@ +import WebSocketConnection from './WebSocketConnection'; import type WebSocketStream from './WebSocketStream'; -class StartEvent extends Event { - public detail: { - host: string; - port: number; - }; +// Server events + +abstract class WebSocketServerEvent extends Event {} + +class WebSocketServerConnectionEvent extends Event { + public detail: WebSocketConnection; + constructor( + options: EventInit & { + detail: WebSocketConnection; + }, + ) { + super('serverConnection', options); + this.detail = options.detail; + } +} + +class WebSocketServerStartEvent extends Event { + constructor(options?: EventInit) { + super('serverStart', options); + } +} + +class WebSocketServerStopEvent extends Event { + constructor(options?: EventInit) { + super('serverStop', options); + } +} + +class WebSocketServerErrorEvent extends Event { + public detail: Error; + constructor( + options: EventInit & { + detail: Error; + }, + ) { + super('serverError', options); + this.detail = options.detail; + } +} + +// Connection events + +abstract class WebSocketConnectionEvent extends Event {} + +class WebSocketConnectionStreamEvent extends WebSocketConnectionEvent { + public detail: WebSocketStream; constructor( options: EventInit & { - detail: { - host: string; - port: number; - }; + detail: WebSocketStream; }, ) { - super('start', options); + super('connectionStream', options); this.detail = options.detail; } } -class StopEvent extends Event { +class WebSocketConnectionStopEvent extends WebSocketConnectionEvent { constructor(options?: EventInit) { - super('stop', options); + super('connectionStop', options); } } -class ConnectionEvent extends Event { - public detail: { - webSocketStream: WebSocketStream; - }; +class WebSocketConnectionErrorEvent extends WebSocketConnectionEvent { + public detail: Error; constructor( options: EventInit & { - detail: { - webSocketStream: WebSocketStream; - }; + detail: Error; }, ) { - super('connection', options); + super('connectionError', options); this.detail = options.detail; } } -export { StartEvent, StopEvent, ConnectionEvent }; +export { + WebSocketServerEvent, WebSocketServerConnectionEvent, WebSocketServerStartEvent, WebSocketServerStopEvent, WebSocketConnectionEvent, WebSocketConnectionStreamEvent, WebSocketConnectionStopEvent, WebSocketConnectionErrorEvent }; diff --git a/src/types.ts b/src/types.ts index 5b95e376..84eb369f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,18 +1,105 @@ +import type { DetailedPeerCertificate } from 'tls'; + +/** + * Opaque types are wrappers of existing types + * that require smart constructors + */ +type Opaque = T & { readonly [brand]: K }; +declare const brand: unique symbol; + +/** + * Generic callback + */ +type Callback

= [], R = any, E extends Error = Error> = { + (e: E, ...params: Partial

): R; + (e?: null | undefined, ...params: P): R; +}; + /** * Deconstructed promise */ - type PromiseDeconstructed = { +type PromiseDeconstructed = { p: Promise; resolveP: (value: T | PromiseLike) => void; rejectP: (reason?: any) => void; }; -type TLSConfig = { - keyPrivatePem: string; - certChainPem: string; +type ConnectionId = Opaque<'ConnectionId', number>; + +type StreamId = Opaque<'StreamId', number>; + +/** + * Host is always an IP address + */ +type Host = Opaque<'Host', string>; + +/** + * Hostnames are resolved to IP addresses + */ +type Hostname = Opaque<'Hostname', string>; + +/** + * Ports are numbers from 0 to 65535 + */ +type Port = Opaque<'Port', number>; + +/** + * Combination of `:` + */ +type Address = Opaque<'Address', string>; + +type RemoteInfo = { + host: Host; + port: Port; +}; + +/** + * Maps reason (most likely an exception) to a stream code. + * Use `0` to indicate unknown/default reason. + */ +type StreamReasonToCode = ( + type: 'recv' | 'send', + reason?: any, +) => number | PromiseLike; + +/** + * Maps code to a reason. 0 usually indicates unknown/default reason. + */ +type StreamCodeToReason = ( + type: 'recv' | 'send', + code: number, +) => any | PromiseLike; + +type ConnectionMetadata = { + localHost: string; + localPort: number; + remoteHost: string; + remotePort: number; + peerCert: DetailedPeerCertificate; +}; + +type VerifyCallback = (peerCert: DetailedPeerCertificate) => Promise; + +type WebSocketConfig = { + connectTimeoutTime: number, + keepAliveTimeoutTime: number, + keepAliveIntervalTime: number, }; export type { + Opaque, + Callback, PromiseDeconstructed, - TLSConfig -} + ConnectionId, + StreamId, + Host, + Hostname, + Port, + Address, + RemoteInfo, + StreamReasonToCode, + StreamCodeToReason, + ConnectionMetadata, + VerifyCallback, + WebSocketConfig +}; diff --git a/src/utils/errors.ts b/src/utils/errors.ts deleted file mode 100644 index 379e77cd..00000000 --- a/src/utils/errors.ts +++ /dev/null @@ -1,31 +0,0 @@ -import sysexits from './sysexits'; -import ErrorWebSocket from '../ErrorWebSocket'; - -class ErrorUtils extends ErrorWebSocket {} - -/** - * This is a special error that is only used for absurd situations - * Intended to placate typescript so that unreachable code type checks - * If this is thrown, this means there is a bug in the code - */ -class ErrorUtilsUndefinedBehaviour extends ErrorUtils { - static description = 'You should never see this error'; - exitCode = sysexits.SOFTWARE; -} - -class ErrorUtilsPollTimeout extends ErrorUtils { - static description = 'Poll timed out'; - exitCode = sysexits.TEMPFAIL; -} - -class ErrorUtilsNodePath extends ErrorUtils { - static description = 'Cannot derive default node path from unknown platform'; - exitCode = sysexits.USAGE; -} - -export { - ErrorUtils, - ErrorUtilsUndefinedBehaviour, - ErrorUtilsPollTimeout, - ErrorUtilsNodePath, -}; diff --git a/src/utils/index.ts b/src/utils/index.ts index 1ce90c3f..f0c4eaa5 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,3 +1,2 @@ -export { default as sysexits } from './sysexits'; export * from './utils'; -export * as errors from './errors'; +export * from './types'; diff --git a/src/utils/sysexits.ts b/src/utils/sysexits.ts deleted file mode 100644 index 935c1810..00000000 --- a/src/utils/sysexits.ts +++ /dev/null @@ -1,91 +0,0 @@ -const sysexits = Object.freeze({ - OK: 0, - GENERAL: 1, - // Sysexit standard starts at 64 to avoid conflicts - /** - * The command was used incorrectly, e.g., with the wrong number of arguments, - * a bad flag, a bad syntax in a parameter, or whatever. - */ - USAGE: 64, - /** - * The input data was incorrect in some way. This should only be used for - * user's data and not system files. - */ - DATAERR: 65, - /** - * An input file (not a system file) did not exist or was not readable. - * This could also include errors like "No message" to a mailer - * (if it cared to catch it). - */ - NOINPUT: 66, - /** - * The user specified did not exist. This might be used for mail addresses - * or remote logins. - */ - NOUSER: 67, - /** - * The host specified did not exist. This is used in mail addresses or - * network requests. - */ - NOHOST: 68, - /** - * A service is unavailable. This can occur if a support program or file - * does not exist. This can also be used as a catchall message when - * something you wanted to do does not work, but you do not know why. - */ - UNAVAILABLE: 69, - /** - * An internal software error has been detected. This should be limited to - * non-operating system related errors as possible. - */ - SOFTWARE: 70, - /** - * An operating system error has been detected. This is intended to be used - * for such things as "cannot fork", "cannot create pipe", or the like. - * It in-cludes things like getuid returning a user that does not exist in - * the passwd file. - */ - OSERR: 71, - /** - * Some system file (e.g., /etc/passwd, /var/run/utx.active, etc.) - * does not exist, cannot be opened, or has some sort of error - * (e.g., syntax error). - */ - OSFILE: 72, - /** - * A (user specified) output file cannot be created. - */ - CANTCREAT: 73, - /** - * An error occurred while doing I/O on some file. - */ - IOERR: 74, - /** - * Temporary failure, indicating something that is not really an error. - * In sendmail, this means that a mailer (e.g.) could not create a connection, - * and the request should be reattempted later. - */ - TEMPFAIL: 75, - /** - * The remote system returned something that was "not possible" during a - * protocol exchange. - */ - PROTOCOL: 76, - /** - * You did not have sufficient permission to perform the operation. This is - * not intended for file system problems, which should use EX_NOINPUT or - * EX_CANTCREAT, but rather for higher level permissions. - */ - NOPERM: 77, - /** - * Something was found in an un-configured or mis-configured state. - */ - CONFIG: 78, - CANNOT_EXEC: 126, - COMMAND_NOT_FOUND: 127, - INVALID_EXIT_ARG: 128, - // 128+ are reserved for signal exits - UNKNOWN: 255, -}); - -export default sysexits; diff --git a/src/utils/types.ts b/src/utils/types.ts new file mode 100644 index 00000000..5438defd --- /dev/null +++ b/src/utils/types.ts @@ -0,0 +1,12 @@ +/** + * Deconstructed promise + */ + type PromiseDeconstructed = { + p: Promise; + resolveP: (value: T | PromiseLike) => void; + rejectP: (reason?: any) => void; +}; + +export type { + PromiseDeconstructed +} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 4c4b7722..3dcc78fb 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,11 +1,11 @@ import type { PromiseDeconstructed, -} from '../types'; -import * as utilsErrors from './errors'; +} from './types'; +import * as errors from '../errors'; function never(): never { - throw new utilsErrors.ErrorUtilsUndefinedBehaviour(); + throw new errors.ErrorWebSocketUndefinedBehaviour(); } /** diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts new file mode 100644 index 00000000..44b127b1 --- /dev/null +++ b/tests/WebSocketServer.test.ts @@ -0,0 +1,16 @@ +import { WebSocketServer } from "@"; +import fs from 'fs'; + +describe('test', () => { + test('test', () => { + const server = new WebSocketServer({ + config: { + cert: fs.readFileSync('./cert/cert.pem').toString(), + key: fs.readFileSync('./cert/key.pem').toString(), + } + }); + server.start({ + port: 3000, + }); + }); +}); diff --git a/tests/utils.ts b/tests/utils.ts index 800a02c5..3633f175 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -4,4 +4,4 @@ async function sleep(ms: number): Promise { export { sleep -}; \ No newline at end of file +}; From 6d9a7b241389290d7833dd9086794c834dab499f Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 23 Aug 2023 21:08:13 +1000 Subject: [PATCH 004/149] feat: tests now use generated certs for TLS --- package-lock.json | 389 ++++++++++++++++++++++ package.json | 5 + tests/WebSocketServer.test.ts | 10 +- tests/utils.ts | 603 +++++++++++++++++++++++++++++++++- 4 files changed, 1001 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 786b629c..b24b85fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,10 +18,16 @@ "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", "ip-num": "^1.5.0", + "resource-counter": "^1.2.4", "ws": "^8.13.0" }, "devDependencies": { "@fast-check/jest": "^1.1.0", + "@peculiar/asn1-pkcs8": "^2.3.0", + "@peculiar/asn1-schema": "^2.3.0", + "@peculiar/asn1-x509": "^2.3.0", + "@peculiar/webcrypto": "^1.4.0", + "@peculiar/x509": "^1.8.3", "@swc/core": "^1.3.62", "@swc/jest": "^0.2.26", "@types/jest": "^28.1.3", @@ -1748,6 +1754,258 @@ "node": ">= 8" } }, + "node_modules/@peculiar/asn1-cms": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-cms/-/asn1-cms-2.3.6.tgz", + "integrity": "sha512-Kr0XsyjuElTc4NijuPYyd6YkTlbz0KCuoWnNkfPFhXjHTzbUIh/s15ixjxLj8XDrXsI1aPQp3D64uHbrs3Kuyg==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "@peculiar/asn1-x509-attr": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-cms/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-csr": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-csr/-/asn1-csr-2.3.6.tgz", + "integrity": "sha512-gCTEB/PvUxapmxo4SzGZT1JtEdevRnphRGZZmc9oJE7+pLuj2Px0Q6x+w8VvObfozA3pyPRTq+Wkocnu64+oLw==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-csr/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-ecc": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-ecc/-/asn1-ecc-2.3.6.tgz", + "integrity": "sha512-Hu1xzMJQWv8/GvzOiinaE6XiD1/kEhq2C/V89UEoWeZ2fLUcGNIvMxOr/pMyL0OmpRWj/mhCTXOZp4PP+a0aTg==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-ecc/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-pfx": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pfx/-/asn1-pfx-2.3.6.tgz", + "integrity": "sha512-bScrrpQ59mppcoZLkDEW/Wruu+daSWQxpR2vqGjg69+v7VoQ1Le/Elm10ObfNShV2eNNridNQcOQvsHMLvUOCg==", + "dev": true, + "dependencies": { + "@peculiar/asn1-cms": "^2.3.6", + "@peculiar/asn1-pkcs8": "^2.3.6", + "@peculiar/asn1-rsa": "^2.3.6", + "@peculiar/asn1-schema": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-pfx/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-pkcs8": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs8/-/asn1-pkcs8-2.3.6.tgz", + "integrity": "sha512-poqgdjsHNiyR0gnxP8l5VjRInSgpQvOM3zLULF/ZQW67uUsEiuPfplvaNJUlNqNOCd2szGo9jKW9+JmVVpWojA==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-pkcs8/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-pkcs9": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs9/-/asn1-pkcs9-2.3.6.tgz", + "integrity": "sha512-uaxSBF60glccuu5BEZvoPsaJzebVYcQRjXx2wXsGe7Grz/BXtq5RQAJ/3i9fEXawFK/zIbvbXBBpy07cnvrqhA==", + "dev": true, + "dependencies": { + "@peculiar/asn1-cms": "^2.3.6", + "@peculiar/asn1-pfx": "^2.3.6", + "@peculiar/asn1-pkcs8": "^2.3.6", + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "@peculiar/asn1-x509-attr": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-pkcs9/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-rsa": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-rsa/-/asn1-rsa-2.3.6.tgz", + "integrity": "sha512-DswjJyAXZnvESuImGNTvbNKvh1XApBVqU+r3UmrFFTAI23gv62byl0f5OFKWTNhCf66WQrd3sklpsCZc/4+jwA==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-rsa/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.3.6.tgz", + "integrity": "sha512-izNRxPoaeJeg/AyH8hER6s+H7p4itk+03QCa4sbxI3lNdseQYCuxzgsuNK8bTXChtLTjpJz6NmXKA73qLa3rCA==", + "dev": true, + "dependencies": { + "asn1js": "^3.0.5", + "pvtsutils": "^1.3.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-schema/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.3.6.tgz", + "integrity": "sha512-dRwX31R1lcbIdzbztiMvLNTDoGptxdV7HocNx87LfKU0fEWh7fTWJjx4oV+glETSy6heF/hJHB2J4RGB3vVSYg==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "asn1js": "^3.0.5", + "ipaddr.js": "^2.0.1", + "pvtsutils": "^1.3.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-x509-attr": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-attr/-/asn1-x509-attr-2.3.6.tgz", + "integrity": "sha512-x5Kax8xp3fz+JSc+4Sq0/SUXIdbJeOePibYqvjHMGkP6AoeCOVcP+gg7rZRRGkTlDSyQnAoUTgTEsfAfFEd1/g==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "asn1js": "^3.0.5", + "tslib": "^2.4.0" + } + }, + "node_modules/@peculiar/asn1-x509-attr/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/asn1-x509/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/json-schema": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@peculiar/json-schema/-/json-schema-1.1.12.tgz", + "integrity": "sha512-coUfuoMeIB7B8/NMekxaDzLhaYmp0HZNPEjYRm9goRou8UZIC3z21s0sL9AWoCw4EG876QyO3kYrc61WNF9B/w==", + "dev": true, + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@peculiar/json-schema/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/webcrypto": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@peculiar/webcrypto/-/webcrypto-1.4.3.tgz", + "integrity": "sha512-VtaY4spKTdN5LjJ04im/d/joXuvLbQdgy5Z4DXF4MFZhQ+MTrejbNMkfZBp1Bs3O5+bFqnJgyGdPuZQflvIa5A==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/json-schema": "^1.1.12", + "pvtsutils": "^1.3.2", + "tslib": "^2.5.0", + "webcrypto-core": "^1.7.7" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/@peculiar/webcrypto/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/@peculiar/x509": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/@peculiar/x509/-/x509-1.9.5.tgz", + "integrity": "sha512-6HBrlgoyH8sod0PTjQ8hzOL4/f5L94s5lwiL9Gr0P5HiSO8eeNgKoiB+s7VhDczE2aaloAgDXFjoQHVEcTg4mg==", + "dev": true, + "dependencies": { + "@peculiar/asn1-cms": "^2.3.6", + "@peculiar/asn1-csr": "^2.3.6", + "@peculiar/asn1-ecc": "^2.3.6", + "@peculiar/asn1-pkcs9": "^2.3.6", + "@peculiar/asn1-rsa": "^2.3.6", + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/asn1-x509": "^2.3.6", + "pvtsutils": "^1.3.5", + "reflect-metadata": "^0.1.13", + "tslib": "^2.6.1", + "tsyringe": "^4.8.0" + } + }, + "node_modules/@peculiar/x509/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -2697,6 +2955,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asn1js": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.5.tgz", + "integrity": "sha512-FVnvrKJwpt9LP2lAMl8qZswRNm3T4q9CON+bxldk2iwk3FFpuwhx2FfinyitizWHsVYyaY+y5JzDR0rCMV5yTQ==", + "dev": true, + "dependencies": { + "pvtsutils": "^1.3.2", + "pvutils": "^1.1.3", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/asn1js/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", @@ -2800,12 +3078,29 @@ "@babel/core": "^7.0.0" } }, + "node_modules/babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==", + "dependencies": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/bitset": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/bitset/-/bitset-5.1.1.tgz", + "integrity": "sha512-oKaRp6mzXedJ1Npo86PKhWfDelI6HxxJo+it9nAcBB0HLVvYVp+5i6yj6DT5hfFgo+TS5T57MRWtw8zhwdTs3g==", + "engines": { + "node": "*" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3044,6 +3339,13 @@ "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", "dev": true }, + "node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", + "hasInstallScript": true + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -4283,6 +4585,15 @@ "resolved": "https://registry.npmjs.org/ip-num/-/ip-num-1.5.1.tgz", "integrity": "sha512-QziFxgxq3mjIf5CuwlzXFYscHxgLqdEdJKRo2UJ5GurL5zrSRMzT/O+nK0ABimoFH8MWF8YwIiwECYsHc1LpUQ==" }, + "node_modules/ipaddr.js": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", + "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, "node_modules/is-array-buffer": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", @@ -7213,6 +7524,30 @@ } ] }, + "node_modules/pvtsutils": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.5.tgz", + "integrity": "sha512-ARvb14YB9Nm2Xi6nBq1ZX6dAM0FsJnuk+31aUp4TrcZEdKUlSqOqsxJHUPJDNE3qiIp+iUPEIeR6Je/tgV7zsA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.1" + } + }, + "node_modules/pvtsutils/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, + "node_modules/pvutils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.1.3.tgz", + "integrity": "sha512-pMpnA0qRdFp32b1sJl1wOJNxZLQ2cbQx+k6tjNtZ8CpvVhNqEPRgivZ2WOUev2YMajecdH7ctUPDvEe87nariQ==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -7251,6 +7586,17 @@ "node": ">= 0.10" } }, + "node_modules/reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==", + "dev": true + }, + "node_modules/regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", @@ -7333,6 +7679,18 @@ "node": ">=10" } }, + "node_modules/resource-counter": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/resource-counter/-/resource-counter-1.2.4.tgz", + "integrity": "sha512-DGJChvE5r4smqPE+xYNv9r1u/I9cCfRR5yfm7D6EQckdKqMyVpJ5z0s40yn0EM0puFxHg6mPORrQLQdEbJ/RnQ==", + "dependencies": { + "babel-runtime": "^6.26.0", + "bitset": "^5.0.3" + }, + "engines": { + "node": ">=6.4.0" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -7972,6 +8330,18 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, + "node_modules/tsyringe": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/tsyringe/-/tsyringe-4.8.0.tgz", + "integrity": "sha512-YB1FG+axdxADa3ncEtRnQCFq/M0lALGLxSZeVNbTU8NqhOVc51nnv2CISTcvc1kyv6EGPtXVr0v6lWeDxiijOA==", + "dev": true, + "dependencies": { + "tslib": "^1.9.3" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -8232,6 +8602,25 @@ "makeerror": "1.0.12" } }, + "node_modules/webcrypto-core": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/webcrypto-core/-/webcrypto-core-1.7.7.tgz", + "integrity": "sha512-7FjigXNsBfopEj+5DV2nhNpfic2vumtjjgPmeDKk45z+MJwXKKfhPB7118Pfzrmh4jqOMST6Ch37iPAHoImg5g==", + "dev": true, + "dependencies": { + "@peculiar/asn1-schema": "^2.3.6", + "@peculiar/json-schema": "^1.1.12", + "asn1js": "^3.0.1", + "pvtsutils": "^1.3.2", + "tslib": "^2.4.0" + } + }, + "node_modules/webcrypto-core/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 3d76dc37..281a9e41 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,11 @@ }, "devDependencies": { "@fast-check/jest": "^1.1.0", + "@peculiar/asn1-pkcs8": "^2.3.0", + "@peculiar/asn1-schema": "^2.3.0", + "@peculiar/asn1-x509": "^2.3.0", + "@peculiar/webcrypto": "^1.4.0", + "@peculiar/x509": "^1.8.3", "@swc/core": "^1.3.62", "@swc/jest": "^0.2.26", "@types/jest": "^28.1.3", diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index 44b127b1..ea5a10f6 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,12 +1,12 @@ -import { WebSocketServer } from "@"; -import fs from 'fs'; +import WebSocketServer from "@/WebSocketServer"; +import * as testsUtils from './utils'; describe('test', () => { - test('test', () => { + test('test', async () => { + const tlsConfigServer = await testsUtils.generateConfig('RSA'); const server = new WebSocketServer({ config: { - cert: fs.readFileSync('./cert/cert.pem').toString(), - key: fs.readFileSync('./cert/key.pem').toString(), + ...tlsConfigServer } }); server.start({ diff --git a/tests/utils.ts b/tests/utils.ts index 3633f175..f2a5f9bc 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -1,7 +1,608 @@ +import type { X509Certificate } from '@peculiar/x509'; +import { Crypto } from '@peculiar/webcrypto'; +import * as x509 from '@peculiar/x509'; +import { never } from '@/utils'; + +/** + * WebCrypto polyfill from @peculiar/webcrypto + * This behaves differently with respect to Ed25519 keys + * See: https://github.com/PeculiarVentures/webcrypto/issues/55 + */ +const webcrypto = new Crypto(); + +/** + * Monkey patches the global crypto object polyfill + */ +globalThis.crypto = webcrypto; + +x509.cryptoProvider.set(webcrypto); + async function sleep(ms: number): Promise { return await new Promise((r) => setTimeout(r, ms)); } +async function randomBytes(data: ArrayBuffer) { + webcrypto.getRandomValues(new Uint8Array(data)); +} + +/** + * Generates RSA keypair + */ +async function generateKeyPairRSA(): Promise<{ + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}> { + const keyPair = await webcrypto.subtle.generateKey( + { + name: 'RSASSA-PKCS1-v1_5', + modulusLength: 2048, + publicExponent: new Uint8Array([0x01, 0x00, 0x01]), + hash: 'SHA-256', + }, + true, + ['sign', 'verify'], + ); + return { + publicKey: await webcrypto.subtle.exportKey('jwk', keyPair.publicKey), + privateKey: await webcrypto.subtle.exportKey('jwk', keyPair.privateKey), + }; +} + +/** + * Generates ECDSA keypair + */ +async function generateKeyPairECDSA(): Promise<{ + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}> { + const keyPair = await webcrypto.subtle.generateKey( + { + name: 'ECDSA', + namedCurve: 'P-256', + }, + true, + ['sign', 'verify'], + ); + return { + publicKey: await webcrypto.subtle.exportKey('jwk', keyPair.publicKey), + privateKey: await webcrypto.subtle.exportKey('jwk', keyPair.privateKey), + }; +} + +/** + * Generates Ed25519 keypair + * This uses `@peculiar/webcrypto` API + */ +async function generateKeyPairEd25519(): Promise<{ + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}> { + const keyPair = (await webcrypto.subtle.generateKey( + { + name: 'EdDSA', + namedCurve: 'Ed25519', + }, + true, + ['sign', 'verify'], + )) as CryptoKeyPair; + return { + publicKey: await webcrypto.subtle.exportKey('jwk', keyPair.publicKey), + privateKey: await webcrypto.subtle.exportKey('jwk', keyPair.privateKey), + }; +} + +/** + * Imports public key. + * This uses `@peculiar/webcrypto` API for Ed25519 keys. + */ +async function importPublicKey(publicKey: JsonWebKey): Promise { + let algorithm; + switch (publicKey.kty) { + case 'RSA': + switch (publicKey.alg) { + case 'RS256': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-256', + }; + break; + case 'RS384': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-384', + }; + break; + case 'RS512': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-512', + }; + break; + default: + throw new Error(`Unsupported algorithm ${publicKey.alg}`); + } + break; + case 'EC': + switch (publicKey.crv) { + case 'P-256': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-256', + }; + break; + case 'P-384': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-384', + }; + break; + case 'P-521': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-521', + }; + break; + default: + throw new Error(`Unsupported curve ${publicKey.crv}`); + } + break; + case 'OKP': + algorithm = { + name: 'EdDSA', + namedCurve: 'Ed25519', + }; + break; + default: + throw new Error(`Unsupported key type ${publicKey.kty}`); + } + return await webcrypto.subtle.importKey('jwk', publicKey, algorithm, true, [ + 'verify', + ]); +} + +/** + * Imports private key. + * This uses `@peculiar/webcrypto` API for Ed25519 keys. + */ +async function importPrivateKey(privateKey: JsonWebKey): Promise { + let algorithm; + switch (privateKey.kty) { + case 'RSA': + switch (privateKey.alg) { + case 'RS256': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-256', + }; + break; + case 'RS384': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-384', + }; + break; + case 'RS512': + algorithm = { + name: 'RSASSA-PKCS1-v1_5', + hash: 'SHA-512', + }; + break; + default: + throw new Error(`Unsupported algorithm ${privateKey.alg}`); + } + break; + case 'EC': + switch (privateKey.crv) { + case 'P-256': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-256', + }; + break; + case 'P-384': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-384', + }; + break; + case 'P-521': + algorithm = { + name: 'ECDSA', + namedCurve: 'P-521', + }; + break; + default: + throw new Error(`Unsupported curve ${privateKey.crv}`); + } + break; + case 'OKP': + algorithm = { + name: 'EdDSA', + namedCurve: 'Ed25519', + }; + break; + default: + throw new Error(`Unsupported key type ${privateKey.kty}`); + } + return await webcrypto.subtle.importKey('jwk', privateKey, algorithm, true, [ + 'sign', + ]); +} + +async function keyPairRSAToPEM(keyPair: { + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}): Promise<{ + publicKey: string; + privateKey: string; +}> { + const publicKey = await importPublicKey(keyPair.publicKey); + const privatekey = await importPrivateKey(keyPair.privateKey); + const publicKeySPKI = await webcrypto.subtle.exportKey('spki', publicKey); + const publicKeySPKIBuffer = Buffer.from(publicKeySPKI); + const publicKeyPEMBody = + publicKeySPKIBuffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const publicKeyPEM = `-----BEGIN PUBLIC KEY-----\n${publicKeyPEMBody}\n-----END PUBLIC KEY-----\n`; + const privateKeyPKCS8 = await webcrypto.subtle.exportKey('pkcs8', privatekey); + const privateKeyPKCS8Buffer = Buffer.from(privateKeyPKCS8); + const privateKeyPEMBody = + privateKeyPKCS8Buffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const privateKeyPEM = `-----BEGIN PRIVATE KEY-----\n${privateKeyPEMBody}-----END PRIVATE KEY-----\n`; + return { + publicKey: publicKeyPEM, + privateKey: privateKeyPEM, + }; +} + +async function keyPairECDSAToPEM(keyPair: { + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}): Promise<{ + publicKey: string; + privateKey: string; +}> { + const publicKey = await importPublicKey(keyPair.publicKey); + const privatekey = await importPrivateKey(keyPair.privateKey); + const publicKeySPKI = await webcrypto.subtle.exportKey('spki', publicKey); + const publicKeySPKIBuffer = Buffer.from(publicKeySPKI); + const publicKeyPEMBody = + publicKeySPKIBuffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const publicKeyPEM = `-----BEGIN PUBLIC KEY-----\n${publicKeyPEMBody}\n-----END PUBLIC KEY-----\n`; + const privateKeyPKCS8 = await webcrypto.subtle.exportKey('pkcs8', privatekey); + const privateKeyPKCS8Buffer = Buffer.from(privateKeyPKCS8); + const privateKeyPEMBody = + privateKeyPKCS8Buffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const privateKeyPEM = `-----BEGIN PRIVATE KEY-----\n${privateKeyPEMBody}-----END PRIVATE KEY-----\n`; + return { + publicKey: publicKeyPEM, + privateKey: privateKeyPEM, + }; +} + +async function keyPairEd25519ToPEM(keyPair: { + publicKey: JsonWebKey; + privateKey: JsonWebKey; +}): Promise<{ + publicKey: string; + privateKey: string; +}> { + const publicKey = await importPublicKey(keyPair.publicKey); + const privatekey = await importPrivateKey(keyPair.privateKey); + const publicKeySPKI = await webcrypto.subtle.exportKey('spki', publicKey); + const publicKeySPKIBuffer = Buffer.from(publicKeySPKI); + const publicKeyPEMBody = + publicKeySPKIBuffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const publicKeyPEM = `-----BEGIN PUBLIC KEY-----\n${publicKeyPEMBody}\n-----END PUBLIC KEY-----\n`; + const privateKeyPKCS8 = await webcrypto.subtle.exportKey('pkcs8', privatekey); + const privateKeyPKCS8Buffer = Buffer.from(privateKeyPKCS8); + const privateKeyPEMBody = + privateKeyPKCS8Buffer + .toString('base64') + .replace(/(.{64})/g, '$1\n') + .trimEnd() + '\n'; + const privateKeyPEM = `-----BEGIN PRIVATE KEY-----\n${privateKeyPEMBody}-----END PRIVATE KEY-----\n`; + return { + publicKey: publicKeyPEM, + privateKey: privateKeyPEM, + }; +} + +const extendedKeyUsageFlags = { + serverAuth: '1.3.6.1.5.5.7.3.1', + clientAuth: '1.3.6.1.5.5.7.3.2', + codeSigning: '1.3.6.1.5.5.7.3.3', + emailProtection: '1.3.6.1.5.5.7.3.4', + timeStamping: '1.3.6.1.5.5.7.3.8', + ocspSigning: '1.3.6.1.5.5.7.3.9', +}; + +/** + * Generate x509 certificate. + * Duration is in seconds. + * X509 certificates currently use `UTCTime` format for `notBefore` and `notAfter`. + * This means: + * - Only second resolution. + * - Minimum date for validity is 1970-01-01T00:00:00Z (inclusive). + * - Maximum date for valdity is 2049-12-31T23:59:59Z (inclusive). + */ +async function generateCertificate({ + certId, + subjectKeyPair, + issuerPrivateKey, + duration, + subjectAttrsExtra = [], + issuerAttrsExtra = [], + now = new Date(), +}: { + certId: string; + subjectKeyPair: { + publicKey: JsonWebKey; + privateKey: JsonWebKey; + }; + issuerPrivateKey: JsonWebKey; + duration: number; + subjectAttrsExtra?: Array<{ [key: string]: Array }>; + issuerAttrsExtra?: Array<{ [key: string]: Array }>; + now?: Date; +}): Promise { + const certIdNum = parseInt(certId); + const iss = certIdNum === 0 ? certIdNum : certIdNum - 1; + const sub = certIdNum; + const subjectPublicCryptoKey = await importPublicKey( + subjectKeyPair.publicKey, + ); + const subjectPrivateCryptoKey = await importPrivateKey( + subjectKeyPair.privateKey, + ); + const issuerPrivateCryptoKey = await importPrivateKey(issuerPrivateKey); + if (duration < 0) { + throw new RangeError('`duration` must be positive'); + } + // X509 `UTCTime` format only has resolution of seconds + // this truncates to second resolution + const notBeforeDate = new Date(now.getTime() - (now.getTime() % 1000)); + const notAfterDate = new Date(now.getTime() - (now.getTime() % 1000)); + // If the duration is 0, then only the `now` is valid + notAfterDate.setSeconds(notAfterDate.getSeconds() + duration); + if (notBeforeDate < new Date(0)) { + throw new RangeError( + '`notBeforeDate` cannot be before 1970-01-01T00:00:00Z', + ); + } + if (notAfterDate > new Date(new Date('2050').getTime() - 1)) { + throw new RangeError('`notAfterDate` cannot be after 2049-12-31T23:59:59Z'); + } + const serialNumber = certId; + // The entire subject attributes and issuer attributes + // is constructed via `x509.Name` class + // By default this supports on a limited set of names: + // CN, L, ST, O, OU, C, DC, E, G, I, SN, T + // If custom names are desired, this needs to change to constructing + // `new x509.Name('FOO=BAR', { FOO: '1.2.3.4' })` manually + // And each custom attribute requires a registered OID + // Because the OID is what is encoded into ASN.1 + const subjectAttrs = [ + { + CN: [`${sub}`], + }, + // Filter out conflicting CN attributes + ...subjectAttrsExtra.filter((attr) => !('CN' in attr)), + ]; + const issuerAttrs = [ + { + CN: [`${iss}`], + }, + // Filter out conflicting CN attributes + ...issuerAttrsExtra.filter((attr) => !('CN' in attr)), + ]; + const signingAlgorithm: any = issuerPrivateCryptoKey.algorithm; + if (signingAlgorithm.name === 'ECDSA') { + // In ECDSA, the signature should match the curve strength + switch (signingAlgorithm.namedCurve) { + case 'P-256': + signingAlgorithm.hash = 'SHA-256'; + break; + case 'P-384': + signingAlgorithm.hash = 'SHA-384'; + break; + case 'P-521': + signingAlgorithm.hash = 'SHA-512'; + break; + default: + throw new TypeError( + `Issuer private key has an unsupported curve: ${signingAlgorithm.namedCurve}`, + ); + } + } + const certConfig = { + serialNumber, + notBefore: notBeforeDate, + notAfter: notAfterDate, + subject: subjectAttrs, + issuer: issuerAttrs, + signingAlgorithm, + publicKey: subjectPublicCryptoKey, + signingKey: subjectPrivateCryptoKey, + extensions: [ + new x509.BasicConstraintsExtension(true, undefined, true), + new x509.KeyUsagesExtension( + x509.KeyUsageFlags.keyCertSign | + x509.KeyUsageFlags.cRLSign | + x509.KeyUsageFlags.digitalSignature | + x509.KeyUsageFlags.nonRepudiation | + x509.KeyUsageFlags.keyAgreement | + x509.KeyUsageFlags.keyEncipherment | + x509.KeyUsageFlags.dataEncipherment, + true, + ), + new x509.ExtendedKeyUsageExtension([ + extendedKeyUsageFlags.serverAuth, + extendedKeyUsageFlags.clientAuth, + extendedKeyUsageFlags.codeSigning, + extendedKeyUsageFlags.emailProtection, + extendedKeyUsageFlags.timeStamping, + extendedKeyUsageFlags.ocspSigning, + ]), + await x509.SubjectKeyIdentifierExtension.create(subjectPublicCryptoKey), + ] as Array, + }; + certConfig.signingKey = issuerPrivateCryptoKey; + return await x509.X509CertificateGenerator.create(certConfig); +} + +function certToPEM(cert: X509Certificate): string { + return cert.toString('pem') + '\n'; +} + +/** + * Generate 256-bit HMAC key using webcrypto. + * Web Crypto prefers using the `CryptoKey` type. + * But to be fully generic, we use the `ArrayBuffer` type. + * In production, prefer to use libsodium as it would be faster. + */ +async function generateKeyHMAC(): Promise { + const cryptoKey = await webcrypto.subtle.generateKey( + { + name: 'HMAC', + hash: 'SHA-256', + }, + true, + ['sign', 'verify'], + ); + const key = await webcrypto.subtle.exportKey('raw', cryptoKey); + return key; +} + +/** + * Signs using the 256-bit HMAC key + * Web Crypto has to use the `CryptoKey` type. + * But to be fully generic, we use the `ArrayBuffer` type. + * In production, prefer to use libsodium as it would be faster. + */ +async function signHMAC(key: ArrayBuffer, data: ArrayBuffer) { + const cryptoKey = await webcrypto.subtle.importKey( + 'raw', + key, + { + name: 'HMAC', + hash: 'SHA-256', + }, + true, + ['sign', 'verify'], + ); + return webcrypto.subtle.sign('HMAC', cryptoKey, data); +} + +/** + * Verifies using 256-bit HMAC key + * Web Crypto prefers using the `CryptoKey` type. + * But to be fully generic, we use the `ArrayBuffer` type. + * In production, prefer to use libsodium as it would be faster. + */ +async function verifyHMAC( + key: ArrayBuffer, + data: ArrayBuffer, + sig: ArrayBuffer, +) { + const cryptoKey = await webcrypto.subtle.importKey( + 'raw', + key, + { + name: 'HMAC', + hash: 'SHA-256', + }, + true, + ['sign', 'verify'], + ); + return webcrypto.subtle.verify('HMAC', cryptoKey, sig, data); +} + +type KeyTypes = 'RSA' | 'ECDSA' | 'ED25519'; +type TLSConfigs = { + key: string; + cert: string; + ca: string; +}; + +async function generateConfig(type: KeyTypes): Promise { + let privateKeyPem: string; + let keysLeaf: { publicKey: JsonWebKey; privateKey: JsonWebKey }; + let keysCa: { publicKey: JsonWebKey; privateKey: JsonWebKey }; + switch (type) { + case 'RSA': + { + keysLeaf = await generateKeyPairRSA(); + keysCa = await generateKeyPairRSA(); + privateKeyPem = (await keyPairRSAToPEM(keysLeaf)).privateKey; + } + break; + case 'ECDSA': + { + keysLeaf = await generateKeyPairECDSA(); + keysCa = await generateKeyPairECDSA(); + privateKeyPem = (await keyPairECDSAToPEM(keysLeaf)).privateKey; + } + break; + case 'ED25519': + { + keysLeaf = await generateKeyPairEd25519(); + keysCa = await generateKeyPairEd25519(); + privateKeyPem = (await keyPairEd25519ToPEM(keysLeaf)).privateKey; + } + break; + default: + never(); + } + + const certCa = await generateCertificate({ + certId: '0', + duration: 100000, + issuerPrivateKey: keysCa.privateKey, + subjectKeyPair: keysCa, + }); + const certLeaf = await generateCertificate({ + certId: '1', + duration: 100000, + issuerPrivateKey: keysCa.privateKey, + subjectKeyPair: keysLeaf, + }); + return { + key: privateKeyPem, + cert: certToPEM(certLeaf), + ca: certToPEM(certCa), + }; +} + export { - sleep + sleep, + randomBytes, + generateKeyPairRSA, + generateKeyPairECDSA, + generateKeyPairEd25519, + keyPairRSAToPEM, + keyPairECDSAToPEM, + keyPairEd25519ToPEM, + generateCertificate, + certToPEM, + generateKeyHMAC, + signHMAC, + verifyHMAC, + generateConfig, }; + +export type { KeyTypes, TLSConfigs }; From 867ec6d4a5eeb59086d1cf8667c6885ba394d455 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 23 Aug 2023 21:08:41 +1000 Subject: [PATCH 005/149] feat: streamIds are now bigints + added conversion utility functions for streamIds --- src/WebSocketConnection.ts | 2 ++ src/WebSocketServer.ts | 10 ++++--- src/types.ts | 5 +++- src/utils/utils.ts | 55 +++++++++++++++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 493e05fc..ec374fc2 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -115,6 +115,8 @@ class WebSocketConnection extends EventTarget { ); return; } + const message: Uint8Array = data instanceof ArrayBuffer ? new Uint8Array(data) : data; + message[0]; } public static createWebSocketConnection( diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index f806ba66..a74edc03 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -63,6 +63,7 @@ class WebSocketServer extends EventTarget { config: Partial & { key: string, cert: string, + ca?: string, }; logger?: Logger, }) { @@ -85,7 +86,8 @@ class WebSocketServer extends EventTarget { this.logger.info(`Starting ${this.constructor.name}`); this.server = https.createServer({ ...this.config, - requestTimeout: this.config.connectTimeoutTime + requestTimeout: this.config.connectTimeoutTime, + }); this.webSocketServer = new ws.WebSocketServer({ server: this.server, @@ -172,13 +174,15 @@ class WebSocketServer extends EventTarget { @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) public updateConfig(config: Partial & { - key: string, - cert: string, + key?: string, + cert?: string, + ca?: string, }): void { const tlsServer = this.server as tls.Server; tlsServer.setSecureContext({ key: config.key, cert: config.cert, + ca: config.ca, }); const wsConfig = { ...this.config, diff --git a/src/types.ts b/src/types.ts index 84eb369f..7a721f1b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,7 +26,10 @@ type PromiseDeconstructed = { type ConnectionId = Opaque<'ConnectionId', number>; -type StreamId = Opaque<'StreamId', number>; +/** + * StreamId is a 62 bit unsigned integer + */ +type StreamId = Opaque<'StreamId', bigint>; /** * Host is always an IP address diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 3dcc78fb..14c86c15 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -2,6 +2,7 @@ import type { PromiseDeconstructed, } from './types'; import * as errors from '../errors'; +import { StreamId } from '@/types'; function never(): never { @@ -24,7 +25,59 @@ function promise(): PromiseDeconstructed { }; } +function toStreamId(array: Uint8Array): bigint { + const header = array[0]; + const prefix = header >> 6; + const dv = new DataView(array.buffer, array.byteOffset, array.byteLength); + + let streamId: bigint; + + switch (prefix) { + case 0b00: + streamId = BigInt(dv.getUint8(0)); + case 0b01: + streamId = BigInt(dv.getUint16(0, false)); + case 0b10: + streamId = BigInt(dv.getUint32(0, false)); + case 0b11: + streamId = dv.getBigUint64(0, false); + } + return streamId!; +} + +function fromStreamId(streamId: StreamId): Uint8Array { + const id = streamId as bigint; + let prefix: number; + if (id <= 0xFF) { + prefix = 0b00; + } + else if (id <= 0xFFFF) { + prefix = 0b01; + } + else if (id <= 0xFFFFFFFF) { + prefix = 0b10; + } + else { + prefix = 0b11; + } + const array = new Uint8Array(1 << prefix); + const dv = new DataView(array.buffer, array.byteOffset, array.byteLength); + switch (prefix) { + case 0b00: + dv.setUint8(0, Number(id)); + case 0b01: + dv.setUint16(0, Number(id), false) + case 0b10: + dv.setUint32(0, Number(id), false); + case 0b11: + dv.setBigUint64(0, id, false); + } + return array; +} + export { never, - promise + promise, + toStreamId, + fromStreamId, }; From 63657005de212956bbc3b4b5545d65e7f1964236 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 24 Aug 2023 13:24:37 +1000 Subject: [PATCH 006/149] fix: stream id to/from parser --- package-lock.json | 2 +- package.json | 2 +- src/WebSocketConnection.ts | 8 ++--- src/utils/utils.ts | 61 ++++++++++++++++++++++++-------------- tests/utils.test.ts | 17 +++++++++++ 5 files changed, 62 insertions(+), 28 deletions(-) create mode 100644 tests/utils.test.ts diff --git a/package-lock.json b/package-lock.json index b24b85fc..ee53680a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "ws": "^8.13.0" }, "devDependencies": { - "@fast-check/jest": "^1.1.0", + "@fast-check/jest": "^1.7.1", "@peculiar/asn1-pkcs8": "^2.3.0", "@peculiar/asn1-schema": "^2.3.0", "@peculiar/asn1-x509": "^2.3.0", diff --git a/package.json b/package.json index 281a9e41..e9edba5c 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "ws": "^8.13.0" }, "devDependencies": { - "@fast-check/jest": "^1.1.0", + "@fast-check/jest": "^1.7.1", "@peculiar/asn1-pkcs8": "^2.3.0", "@peculiar/asn1-schema": "^2.3.0", "@peculiar/asn1-x509": "^2.3.0", diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index ec374fc2..b4be982c 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -67,27 +67,27 @@ class WebSocketConnection extends EventTarget { * Client initiated bidirectional stream starts at 0. * Increment by 4 to get the next ID. */ - protected streamIdClientBidi: StreamId = 0b00 as StreamId; + protected streamIdClientBidi: StreamId = 0b00n as StreamId; /** * Server initiated bidirectional stream starts at 1. * Increment by 4 to get the next ID. */ - protected streamIdServerBidi: StreamId = 0b01 as StreamId; + protected streamIdServerBidi: StreamId = 0b01n as StreamId; /** * Client initiated unidirectional stream starts at 2. * Increment by 4 to get the next ID. * Currently unsupported. */ - protected _streamIdClientUni: StreamId = 0b10 as StreamId; + protected _streamIdClientUni: StreamId = 0b10n as StreamId; /** * Server initiated unidirectional stream starts at 3. * Increment by 4 to get the next ID. * Currently unsupported. */ - protected _streamIdServerUni: StreamId = 0b11 as StreamId; + protected _streamIdServerUni: StreamId = 0b11n as StreamId; protected keepAliveTimeOutTimer?: Timer; protected keepAliveIntervalTimer?: Timer; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 14c86c15..2987dacd 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -26,52 +26,69 @@ function promise(): PromiseDeconstructed { } function toStreamId(array: Uint8Array): bigint { + let streamId: bigint; + + // get header and prefix const header = array[0]; const prefix = header >> 6; - const dv = new DataView(array.buffer, array.byteOffset, array.byteLength); - let streamId: bigint; + // copy bytearray and remove prefix + const arrayCopy = array.slice(); + arrayCopy[0] &= 0b00111111; + + const dv = new DataView(arrayCopy.buffer); switch (prefix) { case 0b00: streamId = BigInt(dv.getUint8(0)); + break; case 0b01: streamId = BigInt(dv.getUint16(0, false)); + break; case 0b10: streamId = BigInt(dv.getUint32(0, false)); case 0b11: streamId = dv.getBigUint64(0, false); + break; } return streamId!; } function fromStreamId(streamId: StreamId): Uint8Array { const id = streamId as bigint; - let prefix: number; - if (id <= 0xFF) { - prefix = 0b00; + + let array: Uint8Array; + let dv: DataView; + let prefixMask = 0; + + if (id < 0x40) { + array = new Uint8Array(1); + dv = new DataView(array.buffer); + dv.setUint8(0, Number(id)); } - else if (id <= 0xFFFF) { - prefix = 0b01; + else if (id < 0x4000) { + array = new Uint8Array(2); + dv = new DataView(array.buffer); + dv.setUint16(0, Number(id)); + prefixMask = 0b01_000000; } - else if (id <= 0xFFFFFFFF) { - prefix = 0b10; + else if (id < 0x40000000) { + array = new Uint8Array(4); + dv = new DataView(array.buffer); + dv.setUint32(0, Number(id)); + prefixMask = 0b10_000000; } else { - prefix = 0b11; - } - const array = new Uint8Array(1 << prefix); - const dv = new DataView(array.buffer, array.byteOffset, array.byteLength); - switch (prefix) { - case 0b00: - dv.setUint8(0, Number(id)); - case 0b01: - dv.setUint16(0, Number(id), false) - case 0b10: - dv.setUint32(0, Number(id), false); - case 0b11: - dv.setBigUint64(0, id, false); + array = new Uint8Array(8); + dv = new DataView(array.buffer); + dv.setBigUint64(0, id); + prefixMask = 0b11_000000; } + + let header = dv.getUint8(0); + header |= prefixMask; + dv.setUint8(0, header); + return array; } diff --git a/tests/utils.test.ts b/tests/utils.test.ts new file mode 100644 index 00000000..2f070ed2 --- /dev/null +++ b/tests/utils.test.ts @@ -0,0 +1,17 @@ +import { StreamId } from '@/types'; +import * as utils from '@/utils'; +import { fc, testProp } from '@fast-check/jest'; + +const MAX_62_BIT_UINT = 2n ** 62n - 1n; + +describe('utils', () => { + testProp( + 'from/to StreamId', + [fc.bigUint().filter((n) => n <= MAX_62_BIT_UINT)], + (input) => { + const array = utils.fromStreamId(input as StreamId); + const id = utils.toStreamId(array); + expect(id).toBe(input); + } + ); +}) From 0a76685075aca339dfea0aee90102b073f4f5080 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:45:22 +1000 Subject: [PATCH 007/149] fix: toStreamId properly copies buffer --- src/utils/utils.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 2987dacd..3272c493 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -25,7 +25,7 @@ function promise(): PromiseDeconstructed { }; } -function toStreamId(array: Uint8Array): bigint { +function toStreamId(array: Uint8Array): StreamId { let streamId: bigint; // get header and prefix @@ -33,13 +33,15 @@ function toStreamId(array: Uint8Array): bigint { const prefix = header >> 6; // copy bytearray and remove prefix - const arrayCopy = array.slice(); + const arrayCopy = new Uint8Array(array.length); + arrayCopy.set(array); arrayCopy[0] &= 0b00111111; - const dv = new DataView(arrayCopy.buffer); + const dv = new DataView(arrayCopy.buffer, arrayCopy.byteOffset); switch (prefix) { case 0b00: + console.log(dv.getUint8(0)); streamId = BigInt(dv.getUint8(0)); break; case 0b01: @@ -51,7 +53,7 @@ function toStreamId(array: Uint8Array): bigint { streamId = dv.getBigUint64(0, false); break; } - return streamId!; + return streamId! as StreamId; } function fromStreamId(streamId: StreamId): Uint8Array { From 43a4498fa3f47732ada0e78a99e0a4d3d2a88a18 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:53:32 +1000 Subject: [PATCH 008/149] feat: WebSocketStream WritableStream --- src/WebSocketConnection.ts | 96 ++++++++++++++++++++++++++------ src/WebSocketStream.ts | 102 +++++++++++++++++++++++++++++++--- src/errors.ts | 29 ++++++++++ src/events.ts | 12 +++- src/types.ts | 8 ++- src/utils/utils.ts | 24 ++++++-- tests/WebSocketServer.test.ts | 9 +++ tests/utils.test.ts | 2 +- 8 files changed, 252 insertions(+), 30 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index b4be982c..7876ca3b 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -9,9 +9,8 @@ import { Host, RemoteInfo, StreamId, VerifyCallback, WebSocketConfig } from './t import WebSocketClient from './WebSocketClient'; import WebSocketServer from './WebSocketServer'; import WebSocketStream from './WebSocketStream'; -import Counter from 'resource-counter'; import * as errors from './errors'; -import { promise } from './utils'; +import { fromStreamId, promise, toStreamId } from './utils'; import { Timer } from '@matrixai/timer'; import * as events from './events'; import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; @@ -97,6 +96,13 @@ class WebSocketConnection extends EventTarget { protected logger: Logger; protected _remoteHost: Host; + /** + * Bubble up stream destroy event + */ + protected handleWebSocketStreamDestroyEvent = () => { + this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); + }; + /** * Connection closed promise. * This can resolve or reject. @@ -106,7 +112,8 @@ class WebSocketConnection extends EventTarget { protected resolveClosedP: () => void; protected rejectClosedP: (reason?: any) => void; - protected messageHandler = (data: ws.RawData, isBinary: boolean) => { + protected messageHandler = async (data: ws.RawData, isBinary: boolean) => { + console.log(isBinary); if (!isBinary || data instanceof Array) { this.dispatchEvent( new events.WebSocketConnectionErrorEvent({ @@ -115,8 +122,39 @@ class WebSocketConnection extends EventTarget { ); return; } - const message: Uint8Array = data instanceof ArrayBuffer ? new Uint8Array(data) : data; - message[0]; + let message: Uint8Array = data instanceof ArrayBuffer ? new Uint8Array(data) : data; + + const { data: streamId, remainder } = toStreamId(message); + message = remainder; + + let stream = this.streamMap.get(streamId); + if (stream == null) { + stream = await WebSocketStream.createWebSocketStream({ + connection: this, + streamId, + logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), + }); + stream.addEventListener( + 'streamDestroy', + this.handleWebSocketStreamDestroyEvent, + { once: true } + ); + this.dispatchEvent( + new events.WebSocketConnectionStreamEvent({ + detail: stream, + }) + ); + } + + stream!.streamRecv(message); + } + + protected pingHandler = () => { + this.socket.pong(); + } + + protected pongHandler = () => { + this.setKeepAliveTimeoutTimer(); } public static createWebSocketConnection( @@ -283,13 +321,10 @@ class WebSocketConnection extends EventTarget { void this.stop({ force: true }); } }); - this.socket.on('ping', () => { - this.socket.pong(); - }); - this.socket.on('pong', () => { - this.setKeepAliveTimeoutTimer(); - }); + this.socket.on('message', this.messageHandler); + this.socket.on('ping', this.pingHandler); + this.socket.on('pong', this.pongHandler); this.logger.info(`Started ${this.constructor.name}`); } @@ -303,21 +338,46 @@ class WebSocketConnection extends EventTarget { } else if (this.type === 'server' && streamType === 'bidi') { streamId = this.streamIdServerBidi; } - const wsStream = await WebSocketStream.createWebSocketStream({ + const stream = await WebSocketStream.createWebSocketStream({ streamId: streamId!, connection: this, logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); + stream.addEventListener( + 'streamDestroy', + this.handleWebSocketStreamDestroyEvent, + { once: true } + ); // Ok the stream is opened and working if (this.type === 'client' && streamType === 'bidi') { - this.streamIdClientBidi = (this.streamIdClientBidi + 4) as StreamId; + this.streamIdClientBidi = (this.streamIdClientBidi + 4n) as StreamId; } else if (this.type === 'server' && streamType === 'bidi') { - this.streamIdServerBidi = (this.streamIdServerBidi + 4) as StreamId; + this.streamIdServerBidi = (this.streamIdServerBidi + 4n) as StreamId; } - return wsStream; + return stream; }); } + public async streamSend(streamId: StreamId, data: Uint8Array) + { + const sendProm = promise(); + + const encodedStreamId = fromStreamId(streamId); + const array = new Uint8Array(encodedStreamId.length + data.length); + array.set(encodedStreamId, 0); + array.set(data, encodedStreamId.length); + if (data != null) { + array.set(data, encodedStreamId.length); + } + + this.socket.send(array, (err) => { + if (err == null) sendProm.resolveP(); + else sendProm.rejectP(err); + }); + + await sendProm.p; + } + public async stop({ force = false } : { @@ -331,6 +391,7 @@ class WebSocketConnection extends EventTarget { this.logger.debug('streams destroyed'); this.stopKeepAliveIntervalTimer(); + // Socket Cleanup if (this.socket.readyState === ws.CLOSED) { this.resolveClosedP(); } @@ -339,6 +400,9 @@ class WebSocketConnection extends EventTarget { } await this.closedP; this.logger.debug('closedP'); + this.socket.off('message', this.messageHandler); + this.socket.off('ping', this.pingHandler); + this.socket.off('pong', this.pongHandler); this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); if (this.type === 'server') { @@ -366,7 +430,7 @@ class WebSocketConnection extends EventTarget { this.keepAliveTimeOutTimer != null && this.keepAliveTimeOutTimer.status === null ) { - logger.debug(`resetting timer with ${timeout} delay`); + // logger.debug(`resetting timer with ${timeout} delay`); this.keepAliveTimeOutTimer.reset(timeout); } else { logger.debug(`timeout created with delay ${timeout}`); diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 1f1e1f1f..b32c9e97 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -2,7 +2,9 @@ import { CreateDestroy } from "@matrixai/async-init/dist/CreateDestroy"; import Logger from "@matrixai/logger"; import { } from "stream/web"; import { StreamId } from "./types"; +import { promise, StreamCode } from "./utils"; import WebSocketConnection from "./WebSocketConnection"; +import * as errors from './errors'; interface WebSocketStream extends CreateDestroy {} @CreateDestroy() @@ -14,6 +16,11 @@ class WebSocketStream public readable: ReadableStream; public writable: WritableStream; + protected _readableEnded = false; + protected _readableEndedProm = promise(); + protected _writableEnded = false; + protected _writableEndedProm = promise(); + protected logger: Logger; protected connection: WebSocketConnection; protected readableController: @@ -58,10 +65,17 @@ class WebSocketStream this.readable = new ReadableStream( { - start: (controller) => { + start: async (controller) => { + try { + await this.streamSend(StreamCode.ACK) + } + catch (err) { + controller.error(err); + } }, pull: async (controller) => { + }, cancel: async (reason) => { @@ -75,30 +89,104 @@ class WebSocketStream this.writable = new WritableStream( { - start: (controller) => { - + start: async (controller) => { }, write: async (chunk: Uint8Array, controller) => { - + try { + await this.streamSend(StreamCode.DATA, chunk) + } + catch (err) { + controller.error(err); + } }, close: async () => { - + await this.streamSend(StreamCode.CLOSE); + this.signalWritableEnd(); }, abort: async (reason?: any) => { - + this.signalWritableEnd(reason); } }, { - highWaterMark: 0, + highWaterMark: 1, } ); } public async destroy() { this.logger.info(`Destroy ${this.constructor.name}`); + // force close any open streams + this.cancel(); + // Removing stream from the connection's stream map this.connection.streamMap.delete(this.streamId); this.logger.info(`Destroyed ${this.constructor.name}`); } + + public async streamSend(code: StreamCode) + public async streamSend(code: StreamCode.ACK, payloadSize: number) + public async streamSend(code: StreamCode.DATA, data: Uint8Array) + public async streamSend( + code: StreamCode, + data_?: Uint8Array | number, + ) { + let data: Uint8Array | undefined; + if (code === StreamCode.ACK && typeof data_ === 'number') { + data = new Uint8Array([data_]); + } + else { + data = data_ as Uint8Array | undefined; + } + let arrayLength = 1 + (data?.length ?? 0); + const array = new Uint8Array(arrayLength); + array.set([code], 0); + if (data != null) { + array.set(data, 1); + } + await this.connection.streamSend(this.streamId, array); + } + + public async streamRecv(message: Uint8Array) { + this.logger.debug(message); + } + + /** + * Forces the active stream to end early + */ + public cancel(reason?: any): void { + // Default error + const err = reason ?? new errors.ErrorWebSocketStreamCancel(); + // Close the streams with the given error, + if (!this._readableEnded) { + this.readableController?.error(err); + this.signalReadableEnd(err); + } + if (!this._writableEnded) { + this.writableController?.error(err); + this.signalWritableEnd(err); + } + } + + /** + * Signals the end of the ReadableStream. to be used with the extended class + * to track the streams state. + */ + protected signalReadableEnd(reason?: any) { + if (this._readableEnded) return; + this._readableEnded = true; + if (reason == null) this._readableEndedProm.resolveP(); + else this._readableEndedProm.rejectP(reason); + } + + /** + * Signals the end of the WritableStream. to be used with the extended class + * to track the streams state. + */ + protected signalWritableEnd(reason?: any) { + if (this._writableEnded) return; + this._writableEnded = true; + if (reason == null) this._writableEndedProm.resolveP(); + else this._writableEndedProm.rejectP(reason); + } } export default WebSocketStream; diff --git a/src/errors.ts b/src/errors.ts index 72ea4d03..ef7a1ea2 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -4,6 +4,9 @@ class ErrorWebSocket extends AbstractError { static description = 'WebSocket error'; } + +// Server + class ErrorWebSocketServer extends ErrorWebSocket { static description = 'WebSocket Server error'; } @@ -12,6 +15,8 @@ class ErrorWebSocketServerNotRunning extends ErrorWebSocketServer { static description = 'WebSocket Server is not running'; } +// Connection + class ErrorWebSocketConnection extends ErrorWebSocket { static description = 'WebSocket Connection error'; } @@ -28,6 +33,26 @@ class ErrorWebSocketConnectionKeepAliveTimeOut extends ErrorWebSocketConnecti static description = 'WebSocket Connection reached idle timeout'; } +// Stream + +class ErrorWebSocketStream extends ErrorWebSocket { + static description = 'WebSocket Stream error'; +} + +class ErrorWebSocketStreamDestroyed extends ErrorWebSocketStream { + static description = 'WebSocket Stream is destroyed'; +} + +class ErrorWebSocketStreamClose extends ErrorWebSocketStream { + static description = 'WebSocket Stream force close'; +} + +class ErrorWebSocketStreamCancel extends ErrorWebSocketStream { + static description = 'WebSocket Stream was cancelled without a provided reason'; +} + +// Misc + class ErrorWebSocketUndefinedBehaviour extends ErrorWebSocket { static description = 'This should never happen'; } @@ -40,5 +65,9 @@ export { ErrorWebSocketConnectionNotRunning, ErrorWebSocketConnectionStartTimeOut, ErrorWebSocketConnectionKeepAliveTimeOut, + ErrorWebSocketStream, + ErrorWebSocketStreamDestroyed, + ErrorWebSocketStreamClose, + ErrorWebSocketStreamCancel, ErrorWebSocketUndefinedBehaviour }; diff --git a/src/events.ts b/src/events.ts index 042614b3..37ef5bde 100644 --- a/src/events.ts +++ b/src/events.ts @@ -75,5 +75,15 @@ class WebSocketConnectionErrorEvent extends WebSocketConnectionEvent { } } +// Stream events + +abstract class WebSocketStreamEvent extends Event {} + +class WebSocketStreamDestroyEvent extends WebSocketStreamEvent { + constructor(options?: EventInit) { + super('streamDestroy', options); + } +} + export { - WebSocketServerEvent, WebSocketServerConnectionEvent, WebSocketServerStartEvent, WebSocketServerStopEvent, WebSocketConnectionEvent, WebSocketConnectionStreamEvent, WebSocketConnectionStopEvent, WebSocketConnectionErrorEvent }; + WebSocketServerEvent, WebSocketServerConnectionEvent, WebSocketServerStartEvent, WebSocketServerStopEvent, WebSocketConnectionEvent, WebSocketConnectionStreamEvent, WebSocketConnectionStopEvent, WebSocketConnectionErrorEvent, WebSocketStreamEvent, WebSocketStreamDestroyEvent, }; diff --git a/src/types.ts b/src/types.ts index 7a721f1b..6a98b609 100644 --- a/src/types.ts +++ b/src/types.ts @@ -89,6 +89,11 @@ type WebSocketConfig = { keepAliveIntervalTime: number, }; +interface Parsed { + data: T; + remainder: Uint8Array; +} + export type { Opaque, Callback, @@ -104,5 +109,6 @@ export type { StreamCodeToReason, ConnectionMetadata, VerifyCallback, - WebSocketConfig + WebSocketConfig, + Parsed }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 3272c493..e94c8d12 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -2,7 +2,7 @@ import type { PromiseDeconstructed, } from './types'; import * as errors from '../errors'; -import { StreamId } from '@/types'; +import { Parsed, StreamId } from '@/types'; function never(): never { @@ -25,7 +25,7 @@ function promise(): PromiseDeconstructed { }; } -function toStreamId(array: Uint8Array): StreamId { +function toStreamId(array: Uint8Array): Parsed { let streamId: bigint; // get header and prefix @@ -39,21 +39,29 @@ function toStreamId(array: Uint8Array): StreamId { const dv = new DataView(arrayCopy.buffer, arrayCopy.byteOffset); + let readBytes = 0; + switch (prefix) { case 0b00: - console.log(dv.getUint8(0)); + readBytes = 1; streamId = BigInt(dv.getUint8(0)); break; case 0b01: + readBytes = 2; streamId = BigInt(dv.getUint16(0, false)); break; case 0b10: + readBytes = 4; streamId = BigInt(dv.getUint32(0, false)); case 0b11: + readBytes = 8; streamId = dv.getBigUint64(0, false); break; } - return streamId! as StreamId; + return { + data: streamId! as StreamId, + remainder: array.subarray(readBytes) + }; } function fromStreamId(streamId: StreamId): Uint8Array { @@ -94,9 +102,17 @@ function fromStreamId(streamId: StreamId): Uint8Array { return array; } +enum StreamCode { + DATA = 0, + ACK = 1, + ERROR = 2, + CLOSE = 3, +} + export { never, promise, toStreamId, fromStreamId, + StreamCode }; diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index ea5a10f6..7e73a9ba 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,3 +1,4 @@ +import { WebSocketServerConnectionEvent } from "@/events"; import WebSocketServer from "@/WebSocketServer"; import * as testsUtils from './utils'; @@ -12,5 +13,13 @@ describe('test', () => { server.start({ port: 3000, }); + server.addEventListener("serverConnection", async (event: WebSocketServerConnectionEvent) => { + const connection = event.detail; + const stream = await connection.streamNew("bidi"); + const writer = stream.writable.getWriter(); + await writer.ready; + writer.write(new Uint8Array([1, 2, 3])); + writer.write(new Uint8Array([1, 2, 3])); + }) }); }); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 2f070ed2..5cd2b64c 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -10,7 +10,7 @@ describe('utils', () => { [fc.bigUint().filter((n) => n <= MAX_62_BIT_UINT)], (input) => { const array = utils.fromStreamId(input as StreamId); - const id = utils.toStreamId(array); + const { data: id } = utils.toStreamId(array); expect(id).toBe(input); } ); From dcd3c21e451299b1679ee2247391de0ad8dd8d43 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:38:01 +1000 Subject: [PATCH 009/149] lintfix --- src/WebSocketClient.ts | 10 +- src/WebSocketConnection.ts | 257 +++++++++++++++++----------------- src/WebSocketServer.ts | 51 ++++--- src/WebSocketStream.bak.ts | 4 +- src/WebSocketStream.ts | 69 ++++----- src/config.ts | 8 +- src/errors.ts | 18 ++- src/events.ts | 14 +- src/types.ts | 8 +- src/utils/types.ts | 6 +- src/utils/utils.ts | 31 ++-- tests/WebSocketServer.test.ts | 27 ++-- tests/utils.test.ts | 8 +- 13 files changed, 251 insertions(+), 260 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 3fd0a6b1..00da526f 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,5 +1,6 @@ import type { DetailedPeerCertificate, TLSSocket } from 'tls'; import type { ContextTimed } from '@matrixai/contexts'; +import type { VerifyCallback } from './types'; import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; @@ -8,7 +9,6 @@ import { Timer } from '@matrixai/timer'; import WebSocketStream from './WebSocketStream'; import * as webSocketErrors from './errors'; import { promise } from './utils'; -import { VerifyCallback } from './types'; import WebSocketConnection from './WebSocketConnection'; interface WebSocketClient extends createDestroy.CreateDestroy {} @@ -35,7 +35,7 @@ class WebSocketClient { pingIntervalTime = 1_000, pingTimeoutTimeTime = 10_000, logger = new Logger(this.name), - verifyCallback + verifyCallback, }: { host: string; port: number; @@ -53,7 +53,7 @@ class WebSocketClient { connectionTimeoutTime, pingIntervalTime, pingTimeoutTimeTime, - verifyCallback + verifyCallback, ); logger.info(`Created ${this.name}`); return clientClient; @@ -69,7 +69,7 @@ class WebSocketClient { protected connectionTimeoutTime: number, protected pingIntervalTime: number, protected pingTimeoutTimeTime: number, - protected verifyCallback?: VerifyCallback + protected verifyCallback?: VerifyCallback, ) { if (Validator.isValidIPv4String(host)[0]) { this.host = host; @@ -182,7 +182,7 @@ class WebSocketClient { localPort: request.connection.localPort ?? 0, remoteHost: request.connection.remoteAddress ?? '', remotePort: request.connection.remotePort ?? 0, - peerCert + peerCert, }); } catch (e) { authenticateProm.rejectP(e); diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 7876ca3b..cc5d64ef 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -1,19 +1,25 @@ -import { PromiseCancellable } from '@matrixai/async-cancellable'; +import type { PromiseCancellable } from '@matrixai/async-cancellable'; +import type { ContextTimed, ContextTimedInput } from '@matrixai/contexts'; +import type { + Host, + RemoteInfo, + StreamId, + VerifyCallback, + WebSocketConfig, +} from './types'; +import type WebSocketClient from './WebSocketClient'; +import type WebSocketServer from './WebSocketServer'; import { startStop } from '@matrixai/async-init'; -import { ContextTimed, ContextTimedInput } from '@matrixai/contexts'; import { Lock } from '@matrixai/async-locks'; import { context, timedCancellable } from '@matrixai/contexts/dist/decorators'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; -import { Host, RemoteInfo, StreamId, VerifyCallback, WebSocketConfig } from './types'; -import WebSocketClient from './WebSocketClient'; -import WebSocketServer from './WebSocketServer'; +import { Timer } from '@matrixai/timer'; +import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { fromStreamId, promise, toStreamId } from './utils'; -import { Timer } from '@matrixai/timer'; import * as events from './events'; -import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -60,32 +66,32 @@ class WebSocketConnection extends EventTarget { /** * Stream ID increment lock. */ - protected streamIdLock: Lock = new Lock(); + protected streamIdLock: Lock = new Lock(); - /** - * Client initiated bidirectional stream starts at 0. - * Increment by 4 to get the next ID. - */ + /** + * Client initiated bidirectional stream starts at 0. + * Increment by 4 to get the next ID. + */ protected streamIdClientBidi: StreamId = 0b00n as StreamId; - /** - * Server initiated bidirectional stream starts at 1. - * Increment by 4 to get the next ID. - */ + /** + * Server initiated bidirectional stream starts at 1. + * Increment by 4 to get the next ID. + */ protected streamIdServerBidi: StreamId = 0b01n as StreamId; - /** - * Client initiated unidirectional stream starts at 2. - * Increment by 4 to get the next ID. - * Currently unsupported. - */ + /** + * Client initiated unidirectional stream starts at 2. + * Increment by 4 to get the next ID. + * Currently unsupported. + */ protected _streamIdClientUni: StreamId = 0b10n as StreamId; - /** - * Server initiated unidirectional stream starts at 3. - * Increment by 4 to get the next ID. - * Currently unsupported. - */ + /** + * Server initiated unidirectional stream starts at 3. + * Increment by 4 to get the next ID. + * Currently unsupported. + */ protected _streamIdServerUni: StreamId = 0b11n as StreamId; protected keepAliveTimeOutTimer?: Timer; @@ -117,12 +123,13 @@ class WebSocketConnection extends EventTarget { if (!isBinary || data instanceof Array) { this.dispatchEvent( new events.WebSocketConnectionErrorEvent({ - detail: new errors.ErrorWebSocketUndefinedBehaviour() - }) + detail: new errors.ErrorWebSocketUndefinedBehaviour(), + }), ); return; } - let message: Uint8Array = data instanceof ArrayBuffer ? new Uint8Array(data) : data; + let message: Uint8Array = + data instanceof ArrayBuffer ? new Uint8Array(data) : data; const { data: streamId, remainder } = toStreamId(message); message = remainder; @@ -137,97 +144,92 @@ class WebSocketConnection extends EventTarget { stream.addEventListener( 'streamDestroy', this.handleWebSocketStreamDestroyEvent, - { once: true } + { once: true }, ); this.dispatchEvent( new events.WebSocketConnectionStreamEvent({ detail: stream, - }) + }), ); } stream!.streamRecv(message); - } + }; protected pingHandler = () => { this.socket.pong(); - } + }; protected pongHandler = () => { this.setKeepAliveTimeoutTimer(); - } + }; public static createWebSocketConnection( - args: { - type: 'client'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: undefined - client?: WebSocketClient; - verifyCallback?: VerifyCallback; - logger?: Logger; - } | { - type: 'server'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: WebSocketServer; - client?: undefined; - verifyCallback?: undefined; - logger?: Logger; - }, + args: + | { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined; + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } + | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }, ctx?: Partial, ): PromiseCancellable; - @timedCancellable( - true, - Infinity, - errors.ErrorWebSocketConnectionStartTimeOut - ) + @timedCancellable(true, Infinity, errors.ErrorWebSocketConnectionStartTimeOut) public static async createWebSocketConnection( - args: { - type: 'client'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: undefined - client?: WebSocketClient; - verifyCallback?: VerifyCallback; - logger?: Logger; - } | { - type: 'server'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: WebSocketServer; - client?: undefined; - verifyCallback?: undefined; - logger?: Logger; - }, + args: + | { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined; + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } + | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }, @context ctx: ContextTimed, ): Promise { // Setting up abort/cancellation logic const abortProm = promise(); const abortHandler = () => { abortProm.rejectP(ctx.signal.reason); - } + }; ctx.signal.addEventListener('abort', abortHandler); const connection = new this(args); try { - await Promise.race([ - connection.start(), - abortProm.p, - ]); - } - catch (e) { + await Promise.race([connection.start(), abortProm.p]); + } catch (e) { await connection.stop({ force: true }); throw e; - } - finally { + } finally { ctx.signal.removeEventListener('abort', abortHandler); } if (connection.config.keepAliveIntervalTime != null) { @@ -247,27 +249,29 @@ class WebSocketConnection extends EventTarget { client, verifyCallback, logger, - }: { - type: 'client'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: undefined - client?: WebSocketClient; - verifyCallback?: VerifyCallback; - logger?: Logger; - } | { - type: 'server'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: WebSocketServer; - client?: undefined; - verifyCallback?: undefined; - logger?: Logger; - }) { + }: + | { + type: 'client'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: undefined; + client?: WebSocketClient; + verifyCallback?: VerifyCallback; + logger?: Logger; + } + | { + type: 'server'; + connectionId: number; + remoteInfo: RemoteInfo; + config: WebSocketConfig; + socket: ws.WebSocket; + server?: WebSocketServer; + client?: undefined; + verifyCallback?: undefined; + logger?: Logger; + }) { super(); this.logger = logger ?? new Logger(`${this.constructor.name}`); this.connectionId = connectionId; @@ -286,7 +290,6 @@ class WebSocketConnection extends EventTarget { this.closedP = closedP; this.resolveClosedP = resolveClosedP; this.rejectClosedP = rejectClosedP; - } public async start(): Promise { this.logger.info(`Start ${this.constructor.name}`); @@ -330,7 +333,9 @@ class WebSocketConnection extends EventTarget { } @ready(new errors.ErrorWebSocketConnectionNotRunning()) - public async streamNew(streamType: 'bidi' = 'bidi'): Promise { + public async streamNew( + streamType: 'bidi' = 'bidi', + ): Promise { return await this.streamIdLock.withF(async () => { let streamId: StreamId; if (this.type === 'client' && streamType === 'bidi') { @@ -346,7 +351,7 @@ class WebSocketConnection extends EventTarget { stream.addEventListener( 'streamDestroy', this.handleWebSocketStreamDestroyEvent, - { once: true } + { once: true }, ); // Ok the stream is opened and working if (this.type === 'client' && streamType === 'bidi') { @@ -358,8 +363,7 @@ class WebSocketConnection extends EventTarget { }); } - public async streamSend(streamId: StreamId, data: Uint8Array) - { + public async streamSend(streamId: StreamId, data: Uint8Array) { const sendProm = promise(); const encodedStreamId = fromStreamId(streamId); @@ -370,7 +374,7 @@ class WebSocketConnection extends EventTarget { array.set(data, encodedStreamId.length); } - this.socket.send(array, (err) => { + this.socket.send(array, (err) => { if (err == null) sendProm.resolveP(); else sendProm.rejectP(err); }); @@ -378,11 +382,7 @@ class WebSocketConnection extends EventTarget { await sendProm.p; } - public async stop({ - force = false - } : { - force: boolean - }) { + public async stop({ force = false }: { force: boolean }) { this.logger.info(`Stop ${this.constructor.name}`); // Cleaning up existing streams // ... @@ -394,8 +394,7 @@ class WebSocketConnection extends EventTarget { // Socket Cleanup if (this.socket.readyState === ws.CLOSED) { this.resolveClosedP(); - } - else { + } else { this.socket.close(); } await this.closedP; @@ -418,9 +417,11 @@ class WebSocketConnection extends EventTarget { const logger = this.logger.getChild('timer'); const timeout = this.config.keepAliveTimeoutTime; const keepAliveTimeOutHandler = () => { - this.dispatchEvent(new events.WebSocketConnectionErrorEvent({ - detail: new errors.ErrorWebSocketConnectionKeepAliveTimeOut(), - })); + this.dispatchEvent( + new events.WebSocketConnectionErrorEvent({ + detail: new errors.ErrorWebSocketConnectionKeepAliveTimeOut(), + }), + ); if (this[startStop.running] && this[startStop.status] !== 'stopping') { void this.stop({ force: true }); } @@ -430,7 +431,7 @@ class WebSocketConnection extends EventTarget { this.keepAliveTimeOutTimer != null && this.keepAliveTimeOutTimer.status === null ) { - // logger.debug(`resetting timer with ${timeout} delay`); + // Logger.debug(`resetting timer with ${timeout} delay`); this.keepAliveTimeOutTimer.reset(timeout); } else { logger.debug(`timeout created with delay ${timeout}`); diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index a74edc03..7d129c80 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -1,15 +1,15 @@ import type { IncomingMessage, ServerResponse } from 'http'; import type tls from 'tls'; +import type { Host, Port, WebSocketConfig } from './types'; import https from 'https'; import { startStop, status } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; +import Counter from 'resource-counter'; import * as errors from './errors'; import * as webSocketEvents from './events'; import { never, promise } from './utils'; -import { Host, Port, WebSocketConfig } from './types'; import WebSocketConnection from './WebSocketConnection'; -import Counter from 'resource-counter'; import { serverDefault } from './config'; import * as utils from './utils'; @@ -42,9 +42,13 @@ class WebSocketServer extends EventTarget { ); } else if (event instanceof webSocketEvents.WebSocketConnectionStopEvent) { this.dispatchEvent(new webSocketEvents.WebSocketConnectionStopEvent()); - } else if (event instanceof webSocketEvents.WebSocketConnectionStreamEvent) { + } else if ( + event instanceof webSocketEvents.WebSocketConnectionStreamEvent + ) { this.dispatchEvent( - new webSocketEvents.WebSocketConnectionStreamEvent({ detail: event.detail }), + new webSocketEvents.WebSocketConnectionStreamEvent({ + detail: event.detail, + }), ); } else { utils.never(); @@ -61,11 +65,11 @@ class WebSocketServer extends EventTarget { logger, }: { config: Partial & { - key: string, - cert: string, - ca?: string, + key: string; + cert: string; + ca?: string; }; - logger?: Logger, + logger?: Logger; }) { super(); const wsConfig = { @@ -87,7 +91,6 @@ class WebSocketServer extends EventTarget { this.server = https.createServer({ ...this.config, requestTimeout: this.config.connectTimeoutTime, - }); this.webSocketServer = new ws.WebSocketServer({ server: this.server, @@ -108,24 +111,18 @@ class WebSocketServer extends EventTarget { this._port = address.port; this.logger.debug(`Listening on port ${this._port}`); this._host = address.address ?? '127.0.0.1'; - this.dispatchEvent( - new webSocketEvents.WebSocketServerStartEvent(), - ); + this.dispatchEvent(new webSocketEvents.WebSocketServerStartEvent()); this.logger.info(`Started ${this.constructor.name}`); } - public async stop({ - force = false, - }: { - force?: boolean; - }): Promise { + public async stop({ force = false }: { force?: boolean }): Promise { this.logger.info(`Stopping ${this.constructor.name}`); const destroyProms: Array> = []; for (const webSocketConnection of this.connectionMap.values()) { destroyProms.push( webSocketConnection.stop({ - force - }) + force, + }), ); } this.logger.debug('Awaiting connections to destroy'); @@ -173,11 +170,13 @@ class WebSocketServer extends EventTarget { } @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) - public updateConfig(config: Partial & { - key?: string, - cert?: string, - ca?: string, - }): void { + public updateConfig( + config: Partial & { + key?: string; + cert?: string; + ca?: string; + }, + ): void { const tlsServer = this.server as tls.Server; tlsServer.setSecureContext({ key: config.key, @@ -211,7 +210,7 @@ class WebSocketServer extends EventTarget { config: this.config, socket: webSocket, logger: this.logger.getChild( - `${WebSocketConnection.name} ${connectionId}` + `${WebSocketConnection.name} ${connectionId}`, ), server: this, }); @@ -251,7 +250,7 @@ class WebSocketServer extends EventTarget { this.dispatchEvent( new webSocketEvents.WebSocketServerConnectionEvent({ - detail: connection + detail: connection, }), ); }; diff --git a/src/WebSocketStream.bak.ts b/src/WebSocketStream.bak.ts index db3175fb..ace34c60 100644 --- a/src/WebSocketStream.bak.ts +++ b/src/WebSocketStream.bak.ts @@ -3,13 +3,13 @@ import type { ReadableStreamController, WritableStreamDefaultController, } from 'stream/web'; -import type * as ws from 'ws'; import type Logger from '@matrixai/logger'; +import type WebSocketConnection from './WebSocketConnection'; import { WritableStream, ReadableStream } from 'stream/web'; +import * as ws from 'ws'; import * as webSocketErrors from './errors'; import * as utilsErrors from './utils/errors'; import { promise } from './utils'; -import WebSocketConnection from './WebSocketConnection'; class WebSocketStream implements ReadableWritablePair { public readable: ReadableStream; diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index b32c9e97..1d49c9e2 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,9 +1,9 @@ -import { CreateDestroy } from "@matrixai/async-init/dist/CreateDestroy"; -import Logger from "@matrixai/logger"; -import { } from "stream/web"; -import { StreamId } from "./types"; -import { promise, StreamCode } from "./utils"; -import WebSocketConnection from "./WebSocketConnection"; +import { CreateDestroy } from '@matrixai/async-init/dist/CreateDestroy'; +import Logger from '@matrixai/logger'; +import {} from 'stream/web'; +import type { StreamId } from './types'; +import { promise, StreamCode } from './utils'; +import type WebSocketConnection from './WebSocketConnection'; import * as errors from './errors'; interface WebSocketStream extends CreateDestroy {} @@ -24,15 +24,14 @@ class WebSocketStream protected logger: Logger; protected connection: WebSocketConnection; protected readableController: - | ReadableStreamController - | undefined; + | ReadableStreamController + | undefined; protected writableController: WritableStreamDefaultController | undefined; - public static async createWebSocketStream({ streamId, connection, - logger = new Logger(`${this.name} ${streamId}`) + logger = new Logger(`${this.name} ${streamId}`), }: { streamId: StreamId; connection: WebSocketConnection; @@ -42,7 +41,7 @@ class WebSocketStream const stream = new this({ streamId, connection, - logger + logger, }); connection.streamMap.set(streamId, stream); logger.info(`Created ${this.name}`); @@ -52,7 +51,7 @@ class WebSocketStream constructor({ streamId, connection, - logger + logger, }: { streamId: StreamId; connection: WebSocketConnection; @@ -67,35 +66,27 @@ class WebSocketStream { start: async (controller) => { try { - await this.streamSend(StreamCode.ACK) - } - catch (err) { + await this.streamSend(StreamCode.ACK); + } catch (err) { controller.error(err); } - }, - pull: async (controller) => { - - }, - cancel: async (reason) => { - - } + pull: async (controller) => {}, + cancel: async (reason) => {}, }, new CountQueuingStrategy({ // Allow 1 buffered message, so we can know when data is desired, and we can know when to un-pause. highWaterMark: 1, - }) + }), ); this.writable = new WritableStream( { - start: async (controller) => { - }, + start: async (controller) => {}, write: async (chunk: Uint8Array, controller) => { try { - await this.streamSend(StreamCode.DATA, chunk) - } - catch (err) { + await this.streamSend(StreamCode.DATA, chunk); + } catch (err) { controller.error(err); } }, @@ -105,38 +96,34 @@ class WebSocketStream }, abort: async (reason?: any) => { this.signalWritableEnd(reason); - } + }, }, { highWaterMark: 1, - } + }, ); } public async destroy() { this.logger.info(`Destroy ${this.constructor.name}`); - // force close any open streams + // Force close any open streams this.cancel(); // Removing stream from the connection's stream map this.connection.streamMap.delete(this.streamId); this.logger.info(`Destroyed ${this.constructor.name}`); } - public async streamSend(code: StreamCode) - public async streamSend(code: StreamCode.ACK, payloadSize: number) - public async streamSend(code: StreamCode.DATA, data: Uint8Array) - public async streamSend( - code: StreamCode, - data_?: Uint8Array | number, - ) { + public async streamSend(code: StreamCode); + public async streamSend(code: StreamCode.ACK, payloadSize: number); + public async streamSend(code: StreamCode.DATA, data: Uint8Array); + public async streamSend(code: StreamCode, data_?: Uint8Array | number) { let data: Uint8Array | undefined; if (code === StreamCode.ACK && typeof data_ === 'number') { data = new Uint8Array([data_]); - } - else { + } else { data = data_ as Uint8Array | undefined; } - let arrayLength = 1 + (data?.length ?? 0); + const arrayLength = 1 + (data?.length ?? 0); const array = new Uint8Array(arrayLength); array.set([code], 0); if (data != null) { diff --git a/src/config.ts b/src/config.ts index edea1ac3..ab78f274 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,11 +1,9 @@ -import { WebSocketConfig } from "./types"; +import type { WebSocketConfig } from './types'; const serverDefault: WebSocketConfig = { connectTimeoutTime: 120, keepAliveIntervalTime: 1_000, keepAliveTimeoutTime: 10_000, -} +}; -export { - serverDefault, -} +export { serverDefault }; diff --git a/src/errors.ts b/src/errors.ts index ef7a1ea2..77cca759 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -4,7 +4,6 @@ class ErrorWebSocket extends AbstractError { static description = 'WebSocket error'; } - // Server class ErrorWebSocketServer extends ErrorWebSocket { @@ -21,15 +20,21 @@ class ErrorWebSocketConnection extends ErrorWebSocket { static description = 'WebSocket Connection error'; } -class ErrorWebSocketConnectionNotRunning extends ErrorWebSocketConnection { +class ErrorWebSocketConnectionNotRunning< + T, +> extends ErrorWebSocketConnection { static description = 'WebSocket Connection is not running'; } -class ErrorWebSocketConnectionStartTimeOut extends ErrorWebSocketConnection { +class ErrorWebSocketConnectionStartTimeOut< + T, +> extends ErrorWebSocketConnection { static description = 'WebSocket Connection start timeout'; } -class ErrorWebSocketConnectionKeepAliveTimeOut extends ErrorWebSocketConnection { +class ErrorWebSocketConnectionKeepAliveTimeOut< + T, +> extends ErrorWebSocketConnection { static description = 'WebSocket Connection reached idle timeout'; } @@ -48,7 +53,8 @@ class ErrorWebSocketStreamClose extends ErrorWebSocketStream { } class ErrorWebSocketStreamCancel extends ErrorWebSocketStream { - static description = 'WebSocket Stream was cancelled without a provided reason'; + static description = + 'WebSocket Stream was cancelled without a provided reason'; } // Misc @@ -69,5 +75,5 @@ export { ErrorWebSocketStreamDestroyed, ErrorWebSocketStreamClose, ErrorWebSocketStreamCancel, - ErrorWebSocketUndefinedBehaviour + ErrorWebSocketUndefinedBehaviour, }; diff --git a/src/events.ts b/src/events.ts index 37ef5bde..618dac61 100644 --- a/src/events.ts +++ b/src/events.ts @@ -1,5 +1,5 @@ -import WebSocketConnection from './WebSocketConnection'; import type WebSocketStream from './WebSocketStream'; +import type WebSocketConnection from './WebSocketConnection'; // Server events @@ -86,4 +86,14 @@ class WebSocketStreamDestroyEvent extends WebSocketStreamEvent { } export { - WebSocketServerEvent, WebSocketServerConnectionEvent, WebSocketServerStartEvent, WebSocketServerStopEvent, WebSocketConnectionEvent, WebSocketConnectionStreamEvent, WebSocketConnectionStopEvent, WebSocketConnectionErrorEvent, WebSocketStreamEvent, WebSocketStreamDestroyEvent, }; + WebSocketServerEvent, + WebSocketServerConnectionEvent, + WebSocketServerStartEvent, + WebSocketServerStopEvent, + WebSocketConnectionEvent, + WebSocketConnectionStreamEvent, + WebSocketConnectionStopEvent, + WebSocketConnectionErrorEvent, + WebSocketStreamEvent, + WebSocketStreamDestroyEvent, +}; diff --git a/src/types.ts b/src/types.ts index 6a98b609..e56fa6d3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -84,9 +84,9 @@ type ConnectionMetadata = { type VerifyCallback = (peerCert: DetailedPeerCertificate) => Promise; type WebSocketConfig = { - connectTimeoutTime: number, - keepAliveTimeoutTime: number, - keepAliveIntervalTime: number, + connectTimeoutTime: number; + keepAliveTimeoutTime: number; + keepAliveIntervalTime: number; }; interface Parsed { @@ -110,5 +110,5 @@ export type { ConnectionMetadata, VerifyCallback, WebSocketConfig, - Parsed + Parsed, }; diff --git a/src/utils/types.ts b/src/utils/types.ts index 5438defd..594a6925 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -1,12 +1,10 @@ /** * Deconstructed promise */ - type PromiseDeconstructed = { +type PromiseDeconstructed = { p: Promise; resolveP: (value: T | PromiseLike) => void; rejectP: (reason?: any) => void; }; -export type { - PromiseDeconstructed -} +export type { PromiseDeconstructed }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index e94c8d12..b567d8ce 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,9 +1,6 @@ -import type { - PromiseDeconstructed, -} from './types'; +import type { PromiseDeconstructed } from './types'; +import type { Parsed, StreamId } from '@/types'; import * as errors from '../errors'; -import { Parsed, StreamId } from '@/types'; - function never(): never { throw new errors.ErrorWebSocketUndefinedBehaviour(); @@ -28,11 +25,11 @@ function promise(): PromiseDeconstructed { function toStreamId(array: Uint8Array): Parsed { let streamId: bigint; - // get header and prefix + // Get header and prefix const header = array[0]; const prefix = header >> 6; - // copy bytearray and remove prefix + // Copy bytearray and remove prefix const arrayCopy = new Uint8Array(array.length); arrayCopy.set(array); arrayCopy[0] &= 0b00111111; @@ -53,6 +50,7 @@ function toStreamId(array: Uint8Array): Parsed { case 0b10: readBytes = 4; streamId = BigInt(dv.getUint32(0, false)); + break; case 0b11: readBytes = 8; streamId = dv.getBigUint64(0, false); @@ -60,7 +58,7 @@ function toStreamId(array: Uint8Array): Parsed { } return { data: streamId! as StreamId, - remainder: array.subarray(readBytes) + remainder: array.subarray(readBytes), }; } @@ -75,20 +73,17 @@ function fromStreamId(streamId: StreamId): Uint8Array { array = new Uint8Array(1); dv = new DataView(array.buffer); dv.setUint8(0, Number(id)); - } - else if (id < 0x4000) { + } else if (id < 0x4000) { array = new Uint8Array(2); dv = new DataView(array.buffer); dv.setUint16(0, Number(id)); prefixMask = 0b01_000000; - } - else if (id < 0x40000000) { + } else if (id < 0x40000000) { array = new Uint8Array(4); dv = new DataView(array.buffer); dv.setUint32(0, Number(id)); prefixMask = 0b10_000000; - } - else { + } else { array = new Uint8Array(8); dv = new DataView(array.buffer); dv.setBigUint64(0, id); @@ -109,10 +104,4 @@ enum StreamCode { CLOSE = 3, } -export { - never, - promise, - toStreamId, - fromStreamId, - StreamCode -}; +export { never, promise, toStreamId, fromStreamId, StreamCode }; diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index 7e73a9ba..9d18525e 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,5 +1,5 @@ -import { WebSocketServerConnectionEvent } from "@/events"; -import WebSocketServer from "@/WebSocketServer"; +import type { WebSocketServerConnectionEvent } from '@/events'; +import WebSocketServer from '@/WebSocketServer'; import * as testsUtils from './utils'; describe('test', () => { @@ -7,19 +7,22 @@ describe('test', () => { const tlsConfigServer = await testsUtils.generateConfig('RSA'); const server = new WebSocketServer({ config: { - ...tlsConfigServer - } + ...tlsConfigServer, + }, }); server.start({ port: 3000, }); - server.addEventListener("serverConnection", async (event: WebSocketServerConnectionEvent) => { - const connection = event.detail; - const stream = await connection.streamNew("bidi"); - const writer = stream.writable.getWriter(); - await writer.ready; - writer.write(new Uint8Array([1, 2, 3])); - writer.write(new Uint8Array([1, 2, 3])); - }) + server.addEventListener( + 'serverConnection', + async (event: WebSocketServerConnectionEvent) => { + const connection = event.detail; + const stream = await connection.streamNew('bidi'); + const writer = stream.writable.getWriter(); + await writer.ready; + writer.write(new Uint8Array([1, 2, 3])); + writer.write(new Uint8Array([1, 2, 3])); + }, + ); }); }); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 5cd2b64c..19ee9f18 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -1,6 +1,6 @@ -import { StreamId } from '@/types'; -import * as utils from '@/utils'; +import type { StreamId } from '@/types'; import { fc, testProp } from '@fast-check/jest'; +import * as utils from '@/utils'; const MAX_62_BIT_UINT = 2n ** 62n - 1n; @@ -12,6 +12,6 @@ describe('utils', () => { const array = utils.fromStreamId(input as StreamId); const { data: id } = utils.toStreamId(array); expect(id).toBe(input); - } + }, ); -}) +}); From 90e1360d62786d19d5256c443ec25edf1a667f77 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:59:47 +1000 Subject: [PATCH 010/149] feat: WebSocketStream chunking --- src/WebSocketConnection.ts | 7 ++- src/WebSocketStream.ts | 102 +++++++++++++++++++++++++++---------- 2 files changed, 81 insertions(+), 28 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index cc5d64ef..8dab5ff9 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -119,7 +119,6 @@ class WebSocketConnection extends EventTarget { protected rejectClosedP: (reason?: any) => void; protected messageHandler = async (data: ws.RawData, isBinary: boolean) => { - console.log(isBinary); if (!isBinary || data instanceof Array) { this.dispatchEvent( new events.WebSocketConnectionErrorEvent({ @@ -385,9 +384,13 @@ class WebSocketConnection extends EventTarget { public async stop({ force = false }: { force: boolean }) { this.logger.info(`Stop ${this.constructor.name}`); // Cleaning up existing streams - // ... + const streamsDestroyP: Array> = []; this.logger.debug('triggering stream destruction'); + for (const stream of this.streamMap.values()) { + streamsDestroyP.push(stream.destroy()); + } this.logger.debug('waiting for streams to destroy'); + await Promise.all(streamsDestroyP); this.logger.debug('streams destroyed'); this.stopKeepAliveIntervalTimer(); diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 1d49c9e2..29231da1 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -5,6 +5,7 @@ import type { StreamId } from './types'; import { promise, StreamCode } from './utils'; import type WebSocketConnection from './WebSocketConnection'; import * as errors from './errors'; +import * as events from './events'; interface WebSocketStream extends CreateDestroy {} @CreateDestroy() @@ -23,10 +24,11 @@ class WebSocketStream protected logger: Logger; protected connection: WebSocketConnection; - protected readableController: - | ReadableStreamController - | undefined; - protected writableController: WritableStreamDefaultController | undefined; + protected readableController: ReadableStreamController; + protected writableController: WritableStreamDefaultController; + + protected writableDesiredSize = 0; + protected writableDesiredSizeProm = promise(); public static async createWebSocketStream({ streamId, @@ -62,34 +64,60 @@ class WebSocketStream this.connection = connection; this.logger = logger; - this.readable = new ReadableStream( + this.readable = new ReadableStream( { start: async (controller) => { + this.readableController = controller; try { await this.streamSend(StreamCode.ACK); } catch (err) { controller.error(err); } }, - pull: async (controller) => {}, + pull: async (controller) => { + + }, cancel: async (reason) => {}, }, - new CountQueuingStrategy({ - // Allow 1 buffered message, so we can know when data is desired, and we can know when to un-pause. - highWaterMark: 1, - }), + new ByteLengthQueuingStrategy({ + highWaterMark: 0xFFFFFFFF, + }) ); + const writableWrite = async (chunk: Uint8Array, controller: WritableStreamDefaultController) => { + await this.writableDesiredSizeProm.p; + let data: Uint8Array; + const isChunkable = chunk.length > this.writableDesiredSize; + if (isChunkable) { + data = chunk.subarray(0, this.writableDesiredSize); + } + else { + data = chunk; + } + const newWritableDesiredSize = this.writableDesiredSize - data.length; + const oldProm = this.writableDesiredSizeProm; + try { + if (this.writableDesiredSize <= 0) { + this.writableDesiredSizeProm = promise(); + } + await this.streamSend(StreamCode.DATA, chunk); + this.writableDesiredSize = newWritableDesiredSize; + if (isChunkable) { + await writableWrite(chunk.subarray(this.writableDesiredSize), controller); + } + } + catch { + this.writableDesiredSizeProm = oldProm; + // TODO: Handle error + } + } + this.writable = new WritableStream( { - start: async (controller) => {}, - write: async (chunk: Uint8Array, controller) => { - try { - await this.streamSend(StreamCode.DATA, chunk); - } catch (err) { - controller.error(err); - } + start: (controller) => { + this.writableController = controller; }, + write: writableWrite, close: async () => { await this.streamSend(StreamCode.CLOSE); this.signalWritableEnd(); @@ -99,8 +127,8 @@ class WebSocketStream }, }, { - highWaterMark: 1, - }, + highWaterMark: 1 + } ); } @@ -133,23 +161,45 @@ class WebSocketStream } public async streamRecv(message: Uint8Array) { - this.logger.debug(message); + const code = message[0] as StreamCode; + const data = message.subarray(1); + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + if (code === StreamCode.DATA) { + const data = message.subarray(1); + this.readableController.enqueue(data); + this.readableController.desiredSize; + } + else if (code === StreamCode.ACK) { + console.log(data); + const bufferSize = dv.getUint32(0, false); + console.log(bufferSize); + this.writableDesiredSize = bufferSize; + this.writableDesiredSizeProm.resolveP(); + + } + else if (code === StreamCode.ERROR) { + this.readableController?.error(new errors.ErrorWebSocketStream()); + } + else if (code === StreamCode.CLOSE) { + // close the stream + } + + this.readableController?.enqueue(data); } /** * Forces the active stream to end early */ public cancel(reason?: any): void { - // Default error - const err = reason ?? new errors.ErrorWebSocketStreamCancel(); + reason = reason ?? new errors.ErrorWebSocketStreamCancel(); // Close the streams with the given error, if (!this._readableEnded) { - this.readableController?.error(err); - this.signalReadableEnd(err); + this.readableController.error(reason); + this.signalReadableEnd(reason); } if (!this._writableEnded) { - this.writableController?.error(err); - this.signalWritableEnd(err); + this.writableController.error(reason); + this.signalWritableEnd(reason); } } From 7fea3e66b41cb5a4f967e720183ed17d1b0205c2 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:25:21 +1000 Subject: [PATCH 011/149] feat: WebSocketClient connections are now managed by WebSocketConnectionMap --- src/WebSocketClient.ts | 25 ++++++++++++++----------- src/errors.ts | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 00da526f..e0e326d1 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -7,9 +7,10 @@ import WebSocket from 'ws'; import { Validator } from 'ip-num'; import { Timer } from '@matrixai/timer'; import WebSocketStream from './WebSocketStream'; -import * as webSocketErrors from './errors'; +import * as errors from './errors'; import { promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; +import Counter from 'resource-counter'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() @@ -45,7 +46,7 @@ class WebSocketClient { logger?: Logger; verifyCallback?: VerifyCallback; }): Promise { - logger.info(`Creating ${this.name}`); + logger.info(`Create ${this.name} to ${host}:${port}`); const clientClient = new this( logger, host, @@ -60,7 +61,9 @@ class WebSocketClient { } protected host: string; - protected activeConnections: Set = new Set(); + + public readonly connectionIdCounter = new Counter(0); + public readonly connectionMap: Map = new Map(); constructor( protected logger: Logger, @@ -76,7 +79,7 @@ class WebSocketClient { } else if (Validator.isValidIPv6String(host)[0]) { this.host = `[${host}]`; } else { - throw new webSocketErrors.ErrorClientInvalidHost(); + throw new errors.ErrorWebSocketClientInvalidHost(); } } @@ -85,7 +88,7 @@ class WebSocketClient { if (force) { for (const activeConnection of this.activeConnections) { activeConnection.cancel( - new webSocketErrors.ErrorClientEndingConnections( + new errors.ErrorClientEndingConnections( 'Destroying WebSocketClient', ), ); @@ -98,11 +101,11 @@ class WebSocketClient { this.logger.info(`Destroyed ${this.constructor.name}`); } - @createDestroy.ready(new webSocketErrors.ErrorClientDestroyed()) + @createDestroy.ready(new errors.ErrorWebSocketClientDestroyed()) public async stopConnections() { for (const activeConnection of this.activeConnections) { activeConnection.cancel( - new webSocketErrors.ErrorClientEndingConnections(), + new errors.ErrorClientEndingConnections(), ); } for (const activeConnection of this.activeConnections) { @@ -111,7 +114,7 @@ class WebSocketClient { } } - @createDestroy.ready(new webSocketErrors.ErrorClientDestroyed()) + @createDestroy.ready(new errors.ErrorWebSocketClientDestroyed()) public async startConnection( ctx: Partial = {}, ): Promise { @@ -127,7 +130,7 @@ class WebSocketClient { void timer.then( () => { abortRaceProm.rejectP( - new webSocketErrors.ErrorClientConnectionTimedOut(), + new errors.ErrorClientConnectionTimedOut(), ); }, () => {}, // Ignore cancellation errors @@ -163,7 +166,7 @@ class WebSocketClient { // Handle connection failure const openErrorHandler = (e) => { connectProm.rejectP( - new webSocketErrors.ErrorClientConnectionFailed(undefined, { + new errors.ErrorClientConnectionFailed(undefined, { cause: e, }), ); @@ -236,7 +239,7 @@ class WebSocketClient { ); const abortStream = () => { webSocketStreamClient.cancel( - new webSocketErrors.ErrorClientStreamAborted(undefined, { + new errors.ErrorClientStreamAborted(undefined, { cause: signal?.reason, }), ); diff --git a/src/errors.ts b/src/errors.ts index 77cca759..376d67be 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -14,6 +14,24 @@ class ErrorWebSocketServerNotRunning extends ErrorWebSocketServer { static description = 'WebSocket Server is not running'; } +// Client + +class ErrorWebSocketClient extends ErrorWebSocket { + static description = 'WebSocket Client error'; +} + +class ErrorWebSocketClientCreateTimeOut extends ErrorWebSocketClient { + static description = 'WebSocketC Client create timeout'; +} + +class ErrorWebSocketClientDestroyed extends ErrorWebSocketClient { + static description = 'WebSocket Client is destroyed'; +} + +class ErrorWebSocketClientInvalidHost extends ErrorWebSocketClient { + static description = 'WebSocket Client cannot be created with the specified host'; +} + // Connection class ErrorWebSocketConnection extends ErrorWebSocket { @@ -67,6 +85,10 @@ export { ErrorWebSocket, ErrorWebSocketServer, ErrorWebSocketServerNotRunning, + ErrorWebSocketClient, + ErrorWebSocketClientCreateTimeOut, + ErrorWebSocketClientDestroyed, + ErrorWebSocketClientInvalidHost, ErrorWebSocketConnection, ErrorWebSocketConnectionNotRunning, ErrorWebSocketConnectionStartTimeOut, From 0404cda76007f3fe35dc2d610202938522f68f5e Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:25:45 +1000 Subject: [PATCH 012/149] fix: WebSocketStream error handling --- src/WebSocketStream.ts | 47 ++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 29231da1..f30c99fb 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -68,14 +68,14 @@ class WebSocketStream { start: async (controller) => { this.readableController = controller; - try { - await this.streamSend(StreamCode.ACK); - } catch (err) { - controller.error(err); - } }, pull: async (controller) => { - + if (controller.desiredSize == null) { + controller.error(new errors.ErrorWebSocketUndefinedBehaviour()); + } + if (controller.desiredSize! > 0) { + await this.streamSend(StreamCode.ACK, controller.desiredSize!).catch((e) => controller.error(e)); + } }, cancel: async (reason) => {}, }, @@ -100,7 +100,7 @@ class WebSocketStream if (this.writableDesiredSize <= 0) { this.writableDesiredSizeProm = promise(); } - await this.streamSend(StreamCode.DATA, chunk); + await this.streamSend(StreamCode.DATA, chunk).catch((e) => controller.error(e));; this.writableDesiredSize = newWritableDesiredSize; if (isChunkable) { await writableWrite(chunk.subarray(this.writableDesiredSize), controller); @@ -119,7 +119,6 @@ class WebSocketStream }, write: writableWrite, close: async () => { - await this.streamSend(StreamCode.CLOSE); this.signalWritableEnd(); }, abort: async (reason?: any) => { @@ -141,10 +140,10 @@ class WebSocketStream this.logger.info(`Destroyed ${this.constructor.name}`); } - public async streamSend(code: StreamCode); - public async streamSend(code: StreamCode.ACK, payloadSize: number); - public async streamSend(code: StreamCode.DATA, data: Uint8Array); - public async streamSend(code: StreamCode, data_?: Uint8Array | number) { + public async streamSend(code: StreamCode): Promise; + public async streamSend(code: StreamCode.ACK, payloadSize: number): Promise; + public async streamSend(code: StreamCode.DATA, data: Uint8Array): Promise; + public async streamSend(code: StreamCode, data_?: Uint8Array | number): Promise { let data: Uint8Array | undefined; if (code === StreamCode.ACK && typeof data_ === 'number') { data = new Uint8Array([data_]); @@ -157,7 +156,13 @@ class WebSocketStream if (data != null) { array.set(data, 1); } - await this.connection.streamSend(this.streamId, array); + try { + await this.connection.streamSend(this.streamId, array); + } + catch (e) { + this.signalWritableEnd(e); + throw e; + } } public async streamRecv(message: Uint8Array) { @@ -165,26 +170,28 @@ class WebSocketStream const data = message.subarray(1); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); if (code === StreamCode.DATA) { - const data = message.subarray(1); + if (this.readableController.desiredSize == null) { + this.readableController.error(new errors.ErrorWebSocketUndefinedBehaviour()); + return; + } + if (data.length > this.readableController.desiredSize) { + this.readableController.error(new errors.ErrorWebSocketStream()); + return; + } this.readableController.enqueue(data); - this.readableController.desiredSize; } else if (code === StreamCode.ACK) { - console.log(data); const bufferSize = dv.getUint32(0, false); - console.log(bufferSize); this.writableDesiredSize = bufferSize; this.writableDesiredSizeProm.resolveP(); } else if (code === StreamCode.ERROR) { - this.readableController?.error(new errors.ErrorWebSocketStream()); + this.readableController.error(new errors.ErrorWebSocketStream()); } else if (code === StreamCode.CLOSE) { // close the stream } - - this.readableController?.enqueue(data); } /** From 7877c6ad7e6e311f764dba09d54a6e5d604dc8a3 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:39:32 +1000 Subject: [PATCH 013/149] feat: WebSocketConnection is now more generic --- src/WebSocketConnection.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 8dab5ff9..c2b58e89 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -20,6 +20,7 @@ import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { fromStreamId, promise, toStreamId } from './utils'; import * as events from './events'; +import { Counter } from 'resource-counter'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -97,8 +98,10 @@ class WebSocketConnection extends EventTarget { protected keepAliveTimeOutTimer?: Timer; protected keepAliveIntervalTimer?: Timer; - protected client?: WebSocketClient; - protected server?: WebSocketServer; + protected connectionMapHolder: { + connectionIdCounter: Counter; + connectionMap: Map; + }; protected logger: Logger; protected _remoteHost: Host; @@ -277,8 +280,7 @@ class WebSocketConnection extends EventTarget { this.socket = socket; this.config = config; this.type = type; - this.server = server; - this.client = client; + this.connectionMapHolder = server ?? client!; this._remoteHost = remoteInfo.host; const { @@ -314,7 +316,7 @@ class WebSocketConnection extends EventTarget { // Set the connection up if (this.type === 'server') { - this.server!.connectionMap.set(this.connectionId, this); + this.connectionMapHolder.connectionMap.set(this.connectionId, this); } this.socket.once('close', () => { @@ -408,8 +410,8 @@ class WebSocketConnection extends EventTarget { this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); if (this.type === 'server') { - this.server!.connectionMap.delete(this.connectionId); - this.server!.connectionIdCounter.deallocate(this.connectionId); + this.connectionMapHolder!.connectionMap.delete(this.connectionId); + this.connectionMapHolder!.connectionIdCounter.deallocate(this.connectionId); } this.dispatchEvent(new events.WebSocketConnectionStopEvent()); From 8a80720452b01291672095a488dd2ffed14f0604 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:50:34 +1000 Subject: [PATCH 014/149] feat: created WebSocketConnectionMap --- src/WebSocketClient.ts | 3 ++- src/WebSocketConnection.ts | 13 ++++++------- src/WebSocketConnectionMap.ts | 27 +++++++++++++++++++++++++++ src/WebSocketServer.ts | 6 +++--- tests/WebSocketServer.test.ts | 4 ++++ 5 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 src/WebSocketConnectionMap.ts diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index e0e326d1..905031f2 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -11,6 +11,7 @@ import * as errors from './errors'; import { promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; import Counter from 'resource-counter'; +import WebSocketConnectionMap from './WebSocketConnectionMap'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() @@ -63,7 +64,7 @@ class WebSocketClient { protected host: string; public readonly connectionIdCounter = new Counter(0); - public readonly connectionMap: Map = new Map(); + public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); constructor( protected logger: Logger, diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index c2b58e89..3db010a0 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -21,6 +21,7 @@ import * as errors from './errors'; import { fromStreamId, promise, toStreamId } from './utils'; import * as events from './events'; import { Counter } from 'resource-counter'; +import WebSocketConnectionMap from './WebSocketConnectionMap'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -98,9 +99,8 @@ class WebSocketConnection extends EventTarget { protected keepAliveTimeOutTimer?: Timer; protected keepAliveIntervalTimer?: Timer; - protected connectionMapHolder: { - connectionIdCounter: Counter; - connectionMap: Map; + protected parentInstance: { + connectionMap: WebSocketConnectionMap }; protected logger: Logger; protected _remoteHost: Host; @@ -280,7 +280,7 @@ class WebSocketConnection extends EventTarget { this.socket = socket; this.config = config; this.type = type; - this.connectionMapHolder = server ?? client!; + this.parentInstance = server ?? client!; this._remoteHost = remoteInfo.host; const { @@ -316,7 +316,7 @@ class WebSocketConnection extends EventTarget { // Set the connection up if (this.type === 'server') { - this.connectionMapHolder.connectionMap.set(this.connectionId, this); + this.parentInstance.connectionMap.set(this.connectionId, this); } this.socket.once('close', () => { @@ -410,8 +410,7 @@ class WebSocketConnection extends EventTarget { this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); if (this.type === 'server') { - this.connectionMapHolder!.connectionMap.delete(this.connectionId); - this.connectionMapHolder!.connectionIdCounter.deallocate(this.connectionId); + this.parentInstance!.connectionMap.delete(this.connectionId); } this.dispatchEvent(new events.WebSocketConnectionStopEvent()); diff --git a/src/WebSocketConnectionMap.ts b/src/WebSocketConnectionMap.ts new file mode 100644 index 00000000..3b84fd21 --- /dev/null +++ b/src/WebSocketConnectionMap.ts @@ -0,0 +1,27 @@ +import WebSocketConnection from "./WebSocketConnection"; +import { Counter } from 'resource-counter'; + +class WebSocketConnectionMap extends Map { + counter: Counter + public constructor() { + super(); + this.counter = new Counter(0); + } + public allocateId(): number { + return this.counter.allocate(); + } + public add(conn: WebSocketConnection): this { + const key = this.allocateId(); + return this.set(key, conn); + } + public delete(key: number): boolean { + this.counter.deallocate(key); + return super.delete(key); + } + public clear(): void { + this.counter = new Counter(0); + return super.clear(); + } +} + +export default WebSocketConnectionMap; diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 7d129c80..1c560136 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -12,6 +12,7 @@ import { never, promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; import { serverDefault } from './config'; import * as utils from './utils'; +import WebSocketConnectionMap from './WebSocketConnectionMap'; /** * Events: @@ -28,8 +29,7 @@ class WebSocketServer extends EventTarget { protected webSocketServer: ws.WebSocketServer; protected _port: number; protected _host: string; - public readonly connectionIdCounter = new Counter(0); - public readonly connectionMap: Map = new Map(); + public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); protected handleWebSocketConnectionEvents = ( event: webSocketEvents.WebSocketConnectionEvent, @@ -199,7 +199,7 @@ class WebSocketServer extends EventTarget { request: IncomingMessage, ) => { const httpSocket = request.connection; - const connectionId = this.connectionIdCounter.allocate(); + const connectionId = this.connectionMap.allocateId(); const connection = await WebSocketConnection.createWebSocketConnection({ type: 'server', connectionId: connectionId, diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index 9d18525e..5626bb75 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,8 +1,12 @@ import type { WebSocketServerConnectionEvent } from '@/events'; import WebSocketServer from '@/WebSocketServer'; +import { WebSocket } from 'ws'; import * as testsUtils from './utils'; describe('test', () => { + beforeEach(() => { + + }); test('test', async () => { const tlsConfigServer = await testsUtils.generateConfig('RSA'); const server = new WebSocketServer({ From 76be5f11ee27914ce734da7169729a55d9683e86 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:38:10 +1000 Subject: [PATCH 015/149] feat: WebSocketClient connection configuration --- src/WebSocketClient.ts | 106 ++++++++++++++++++++++--------------- src/WebSocketConnection.ts | 63 ++++++++++++++++++++-- src/WebSocketServer.ts | 2 + src/config.ts | 8 ++- 4 files changed, 131 insertions(+), 48 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 905031f2..3821e837 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,6 +1,6 @@ import type { DetailedPeerCertificate, TLSSocket } from 'tls'; import type { ContextTimed } from '@matrixai/contexts'; -import type { VerifyCallback } from './types'; +import type { Host, Port, VerifyCallback, WebSocketConfig } from './types'; import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; @@ -12,10 +12,11 @@ import { promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; import Counter from 'resource-counter'; import WebSocketConnectionMap from './WebSocketConnectionMap'; +import { clientDefault } from './config'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() -class WebSocketClient { +class WebSocketClient extends EventTarget { /** * @param obj * @param obj.host - Target host address to connect to @@ -33,55 +34,82 @@ class WebSocketClient { static async createWebSocketClient({ host, port, - connectionTimeoutTime = Infinity, - pingIntervalTime = 1_000, - pingTimeoutTimeTime = 10_000, - logger = new Logger(this.name), + config, + logger = new Logger(`${this.name}`), verifyCallback, }: { host: string; port: number; - connectionTimeoutTime?: number; - pingIntervalTime?: number; - pingTimeoutTimeTime?: number; + config?: Partial; logger?: Logger; verifyCallback?: VerifyCallback; }): Promise { logger.info(`Create ${this.name} to ${host}:${port}`); - const clientClient = new this( - logger, - host, - port, - connectionTimeoutTime, - pingIntervalTime, - pingTimeoutTimeTime, + const wsConfig = { + ...clientDefault, + ...config, + }; + + let host_: Host; + if (Validator.isValidIPv4String(host)[0]) { + host_ = host as Host; + } else if (Validator.isValidIPv6String(host)[0]) { + host_ = `[${host}]` as Host; + } else { + throw new errors.ErrorWebSocketClientInvalidHost(); + } + let port_: Port; + if (port >= 0 && port <= 65535) { + port_ = port as Port; + } + else { + throw new errors.ErrorWebSocketClientInvalidHost(); + } + + const address = `wss://${host_}:${port_}`; + + const client = new this({ + address, + logger + }); + + const webSocket = new WebSocket(address, { + rejectUnauthorized: verifyCallback != null, + }); + + const connectionId = client.connectionMap.allocateId(); + const connection = WebSocketConnection.createWebSocketConnection({ + type: 'client', + connectionId, + remoteInfo: { + host: host_, + port: port_, + }, + config: wsConfig, + socket: webSocket, verifyCallback, - ); + client: client, + }) logger.info(`Created ${this.name}`); - return clientClient; + return client; } - protected host: string; + protected address: string; + protected logger: Logger; public readonly connectionIdCounter = new Counter(0); public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); - constructor( - protected logger: Logger, - host: string, - protected port: number, - protected connectionTimeoutTime: number, - protected pingIntervalTime: number, - protected pingTimeoutTimeTime: number, - protected verifyCallback?: VerifyCallback, - ) { - if (Validator.isValidIPv4String(host)[0]) { - this.host = host; - } else if (Validator.isValidIPv6String(host)[0]) { - this.host = `[${host}]`; - } else { - throw new errors.ErrorWebSocketClientInvalidHost(); - } + constructor({ + address, + logger, + } : { + address: string, + logger: Logger + }) { + super(); + this.address = address; + this.logger = logger; } public async destroy(force: boolean = false) { @@ -153,13 +181,7 @@ class WebSocketClient { const address = `wss://${this.host}:${this.port}`; this.logger.info(`Connecting to ${address}`); const connectProm = promise(); - const authenticateProm = promise<{ - localHost: string; - localPort: number; - remoteHost: string; - remotePort: number; - peerCert: DetailedPeerCertificate; - }>(); + // Let ws handle authentication if no custom verify callback is provided. const ws = new WebSocket(address, { rejectUnauthorized: this.verifyCallback != null, diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 3db010a0..beab39ea 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -2,6 +2,7 @@ import type { PromiseCancellable } from '@matrixai/async-cancellable'; import type { ContextTimed, ContextTimedInput } from '@matrixai/contexts'; import type { Host, + PromiseDeconstructed, RemoteInfo, StreamId, VerifyCallback, @@ -22,6 +23,7 @@ import { fromStreamId, promise, toStreamId } from './utils'; import * as events from './events'; import { Counter } from 'resource-counter'; import WebSocketConnectionMap from './WebSocketConnectionMap'; +import { DetailedPeerCertificate, TLSSocket } from 'tls'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -121,6 +123,8 @@ class WebSocketConnection extends EventTarget { protected resolveClosedP: () => void; protected rejectClosedP: (reason?: any) => void; + protected verifyCallback: ((peerCert: DetailedPeerCertificate) => Promise) | undefined; + protected messageHandler = async (data: ws.RawData, isBinary: boolean) => { if (!isBinary || data instanceof Array) { this.dispatchEvent( @@ -282,6 +286,7 @@ class WebSocketConnection extends EventTarget { this.type = type; this.parentInstance = server ?? client!; this._remoteHost = remoteInfo.host; + this.verifyCallback = verifyCallback; const { p: closedP, @@ -294,7 +299,10 @@ class WebSocketConnection extends EventTarget { } public async start(): Promise { this.logger.info(`Start ${this.constructor.name}`); + const promises: Array> = []; + const connectProm = promise(); + if (this.socket.readyState === ws.OPEN) { connectProm.resolveP(); } @@ -311,13 +319,58 @@ class WebSocketConnection extends EventTarget { connectProm.resolveP(); }; this.socket.once('open', openHandler); - await connectProm; - this.socket.off('open', openHandler); + promises.push(connectProm.p); + + if (this.type === 'client') { + const authenticateProm = promise<{ + localHost: string; + localPort: number; + remoteHost: string; + remotePort: number; + peerCert: DetailedPeerCertificate; + }>(); + this.socket.once('upgrade', async (request) => { + const tlsSocket = request.socket as TLSSocket; + const peerCert = tlsSocket.getPeerCertificate(true); + try { + if (this.verifyCallback != null) { + await this.verifyCallback(peerCert); + } + authenticateProm.resolveP({ + localHost: request.connection.localAddress ?? '', + localPort: request.connection.localPort ?? 0, + remoteHost: request.connection.remoteAddress ?? '', + remotePort: request.connection.remotePort ?? 0, + peerCert, + }); + } + catch (e) { + authenticateProm.rejectP(e); + } + }); + promises.push(authenticateProm.p); + } - // Set the connection up - if (this.type === 'server') { - this.parentInstance.connectionMap.set(this.connectionId, this); + // Wait for open + try { + await Promise.all(promises); + } + catch (e) { + this.socket.removeAllListeners('error'); + this.socket.removeAllListeners('upgrade'); + this.socket.removeAllListeners('open'); + // Close the ws if it's open at this stage + this.socket.terminate(); + throw e; } + finally { + this.socket.removeAllListeners('upgrade'); + this.socket.off('open', openHandler); + this.socket.off('error', openErrorHandler); + } + + // Set the connection up + this.parentInstance.connectionMap.set(this.connectionId, this); this.socket.once('close', () => { this.resolveClosedP(); diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 1c560136..47a3f003 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -213,6 +213,8 @@ class WebSocketServer extends EventTarget { `${WebSocketConnection.name} ${connectionId}`, ), server: this, + }, { + timer: this.config.connectTimeoutTime }); // Handling connection events diff --git a/src/config.ts b/src/config.ts index ab78f274..55425b8c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,4 +6,10 @@ const serverDefault: WebSocketConfig = { keepAliveTimeoutTime: 10_000, }; -export { serverDefault }; +const clientDefault: WebSocketConfig = { + connectTimeoutTime: Infinity, + keepAliveIntervalTime: 1_000, + keepAliveTimeoutTime: 10_000, +} + +export { serverDefault, clientDefault }; From ed4d444936e5f65979c8330b7cb3c76e5d741127 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:52:19 +1000 Subject: [PATCH 016/149] feat: WebSocketStream buffering and tests --- src/WebSocketConnection.ts | 2 + src/WebSocketStream.ts | 33 ++++++++++------ src/config.ts | 2 + src/types.ts | 4 ++ tests/WebSocketStream.test.ts | 73 +++++++++++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 tests/WebSocketStream.test.ts diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index beab39ea..85b64a44 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -145,6 +145,7 @@ class WebSocketConnection extends EventTarget { stream = await WebSocketStream.createWebSocketStream({ connection: this, streamId, + bufferSize: this.config.streamBufferSize, logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); stream.addEventListener( @@ -400,6 +401,7 @@ class WebSocketConnection extends EventTarget { const stream = await WebSocketStream.createWebSocketStream({ streamId: streamId!, connection: this, + bufferSize: this.config.streamBufferSize, logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); stream.addEventListener( diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index f30c99fb..108dbd42 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -33,16 +33,19 @@ class WebSocketStream public static async createWebSocketStream({ streamId, connection, + bufferSize, logger = new Logger(`${this.name} ${streamId}`), }: { streamId: StreamId; connection: WebSocketConnection; - logger: Logger; + bufferSize: number; + logger?: Logger; }): Promise { logger.info(`Create ${this.name}`); const stream = new this({ streamId, connection, + bufferSize, logger, }); connection.streamMap.set(streamId, stream); @@ -53,10 +56,12 @@ class WebSocketStream constructor({ streamId, connection, + bufferSize, logger, }: { streamId: StreamId; connection: WebSocketConnection; + bufferSize: number; logger: Logger; }) { super(); @@ -80,34 +85,38 @@ class WebSocketStream cancel: async (reason) => {}, }, new ByteLengthQueuingStrategy({ - highWaterMark: 0xFFFFFFFF, + highWaterMark: bufferSize, }) ); const writableWrite = async (chunk: Uint8Array, controller: WritableStreamDefaultController) => { await this.writableDesiredSizeProm.p; + this.logger.debug(`${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`); let data: Uint8Array; const isChunkable = chunk.length > this.writableDesiredSize; if (isChunkable) { + this.logger.debug(`this chunk will be split into sizes of ${this.writableDesiredSize} bytes`); data = chunk.subarray(0, this.writableDesiredSize); } else { data = chunk; } - const newWritableDesiredSize = this.writableDesiredSize - data.length; - const oldProm = this.writableDesiredSizeProm; try { - if (this.writableDesiredSize <= 0) { + if (this.writableDesiredSize === data.length) { + this.logger.debug(`this chunk will trigger receiver to send an ACK`); + // Resolve and reset the promise to wait for another ACK + this.writableDesiredSizeProm.resolveP(); this.writableDesiredSizeProm = promise(); } - await this.streamSend(StreamCode.DATA, chunk).catch((e) => controller.error(e));; - this.writableDesiredSize = newWritableDesiredSize; + const bytesWritten = this.writableDesiredSize; + await this.streamSend(StreamCode.DATA, data).catch((e) => controller.error(e)); + this.writableDesiredSize =- data.length; if (isChunkable) { - await writableWrite(chunk.subarray(this.writableDesiredSize), controller); + await writableWrite(chunk.subarray(bytesWritten), controller); } } catch { - this.writableDesiredSizeProm = oldProm; + this.writableDesiredSizeProm.resolveP(); // TODO: Handle error } } @@ -146,7 +155,9 @@ class WebSocketStream public async streamSend(code: StreamCode, data_?: Uint8Array | number): Promise { let data: Uint8Array | undefined; if (code === StreamCode.ACK && typeof data_ === 'number') { - data = new Uint8Array([data_]); + data = new Uint8Array(4); + const dv = new DataView(data.buffer); + dv.setUint32(0, data_, false); } else { data = data_ as Uint8Array | undefined; } @@ -184,7 +195,7 @@ class WebSocketStream const bufferSize = dv.getUint32(0, false); this.writableDesiredSize = bufferSize; this.writableDesiredSizeProm.resolveP(); - + this.logger.debug(`received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`); } else if (code === StreamCode.ERROR) { this.readableController.error(new errors.ErrorWebSocketStream()); diff --git a/src/config.ts b/src/config.ts index 55425b8c..0927136d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,12 +4,14 @@ const serverDefault: WebSocketConfig = { connectTimeoutTime: 120, keepAliveIntervalTime: 1_000, keepAliveTimeoutTime: 10_000, + streamBufferSize: 1024 * 1024 // 1MB }; const clientDefault: WebSocketConfig = { connectTimeoutTime: Infinity, keepAliveIntervalTime: 1_000, keepAliveTimeoutTime: 10_000, + streamBufferSize: 1024 * 1024 // 1MB } export { serverDefault, clientDefault }; diff --git a/src/types.ts b/src/types.ts index e56fa6d3..a0cc062a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -87,6 +87,10 @@ type WebSocketConfig = { connectTimeoutTime: number; keepAliveTimeoutTime: number; keepAliveIntervalTime: number; + /** + * Maximum number of bytes for the readable stream + */ + streamBufferSize: number; }; interface Parsed { diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts new file mode 100644 index 00000000..9007c520 --- /dev/null +++ b/tests/WebSocketStream.test.ts @@ -0,0 +1,73 @@ +import { StreamId } from "@/types"; +import WebSocketStream from "@/WebSocketStream"; +import WebSocketConnection from "@/WebSocketConnection"; +import * as events from "@/events"; +import { promise } from "@/utils"; + +const DEFAULT_BUFFER_SIZE = 1024; + +jest.mock('@/WebSocketConnection', () => { + return jest.fn().mockImplementation(( + ) => { + const instance = new EventTarget() as EventTarget & { + connectedConnection: WebSocketConnection | undefined; + connectTo: (connection: WebSocketConnection) => void; + streamSend: (streamId: StreamId, data: Uint8Array) => Promise; + streamMap: Map; + }; + instance.connectedConnection = undefined; + instance.connectTo = (connectedConnection: any) => { + instance.connectedConnection = connectedConnection; + connectedConnection.connectedConnection = instance; + }; + instance.streamMap = new Map(); + instance.streamSend = async (streamId: StreamId, data: Uint8Array) => { + let stream = instance.connectedConnection!.streamMap.get(streamId); + if (stream == null) { + stream = await WebSocketStream.createWebSocketStream({ + streamId, + bufferSize: DEFAULT_BUFFER_SIZE, + connection: instance.connectedConnection!, + }); + instance.connectedConnection!.dispatchEvent(new events.WebSocketConnectionStreamEvent({ + detail: stream, + })); + } + stream.streamRecv(data); + }; + return instance; + }); +}); + +const connectionMock = jest.mocked(WebSocketConnection, true); + +describe(WebSocketStream.name, () => { + let connection1: WebSocketConnection; + let connection2: WebSocketConnection; + let stream1: WebSocketStream; + let stream2: WebSocketStream; + beforeEach(async () => { + connectionMock.mockClear(); + connection1 = new (WebSocketConnection as any)(); + connection2 = new (WebSocketConnection as any)(); + (connection1 as any).connectTo(connection2); + stream1 = await WebSocketStream.createWebSocketStream({ + streamId: 0n as StreamId, + bufferSize: DEFAULT_BUFFER_SIZE, + connection: connection1 + }); + const createStream2Prom = promise(); + connection2.addEventListener("connectionStream", (e: events.WebSocketConnectionStreamEvent) => { + createStream2Prom.resolveP(e.detail); + }); + stream2 = await createStream2Prom.p; + }); + test('buffering', async () => { + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE+1); + const writeProm = stream2Writable.getWriter().write(buffer); + await stream1Readable.getReader().read() + await writeProm; + }); +}); From 9108f2cc41a1bf1d4b667882d14ad808e8012c10 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:36:01 +1000 Subject: [PATCH 017/149] feat: WebSocketStream error handling --- src/WebSocketStream.ts | 179 ++++++++++++++++++++++++++++++----------- 1 file changed, 132 insertions(+), 47 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 108dbd42..d38299c6 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,8 +1,7 @@ -import { CreateDestroy } from '@matrixai/async-init/dist/CreateDestroy'; +import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; -import {} from 'stream/web'; -import type { StreamId } from './types'; -import { promise, StreamCode } from './utils'; +import type { StreamCodeToReason, StreamId, StreamReasonToCode } from './types'; +import { never, promise, StreamCode } from './utils'; import type WebSocketConnection from './WebSocketConnection'; import * as errors from './errors'; import * as events from './events'; @@ -18,27 +17,33 @@ class WebSocketStream public writable: WritableStream; protected _readableEnded = false; - protected _readableEndedProm = promise(); protected _writableEnded = false; - protected _writableEndedProm = promise(); protected logger: Logger; protected connection: WebSocketConnection; + protected reasonToCode: StreamReasonToCode; + protected codeToReason: StreamCodeToReason; protected readableController: ReadableStreamController; protected writableController: WritableStreamDefaultController; protected writableDesiredSize = 0; protected writableDesiredSizeProm = promise(); + protected destroyProm = promise(); + public static async createWebSocketStream({ streamId, connection, bufferSize, + reasonToCode = () => 0, + codeToReason = (type, code) => new Error(`${type.toString()} ${code.toString()}`), logger = new Logger(`${this.name} ${streamId}`), }: { streamId: StreamId; connection: WebSocketConnection; bufferSize: number; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; logger?: Logger; }): Promise { logger.info(`Create ${this.name}`); @@ -46,6 +51,8 @@ class WebSocketStream streamId, connection, bufferSize, + reasonToCode, + codeToReason, logger, }); connection.streamMap.set(streamId, stream); @@ -57,32 +64,39 @@ class WebSocketStream streamId, connection, bufferSize, + reasonToCode, + codeToReason, logger, }: { streamId: StreamId; connection: WebSocketConnection; bufferSize: number; + reasonToCode: StreamReasonToCode; + codeToReason: StreamCodeToReason; logger: Logger; }) { super(); + this.logger = logger; this.streamId = streamId; this.connection = connection; - this.logger = logger; + this.reasonToCode = reasonToCode; + this.codeToReason = codeToReason; this.readable = new ReadableStream( { start: async (controller) => { this.readableController = controller; + this.logger.debug('started'); }, pull: async (controller) => { - if (controller.desiredSize == null) { - controller.error(new errors.ErrorWebSocketUndefinedBehaviour()); - } - if (controller.desiredSize! > 0) { - await this.streamSend(StreamCode.ACK, controller.desiredSize!).catch((e) => controller.error(e)); + if (controller.desiredSize != null && controller.desiredSize > 0) { + await this.streamSend(StreamCode.ACK, controller.desiredSize!); } }, - cancel: async (reason) => {}, + cancel: async (reason) => { + this.logger.debug(`readable aborted with [${reason.message}]`); + this.signalReadableEnd(true, reason); + }, }, new ByteLengthQueuingStrategy({ highWaterMark: bufferSize, @@ -101,23 +115,24 @@ class WebSocketStream else { data = chunk; } + const oldProm = this.writableDesiredSizeProm; try { if (this.writableDesiredSize === data.length) { this.logger.debug(`this chunk will trigger receiver to send an ACK`); - // Resolve and reset the promise to wait for another ACK - this.writableDesiredSizeProm.resolveP(); + // Reset the promise to wait for another ACK this.writableDesiredSizeProm = promise(); } const bytesWritten = this.writableDesiredSize; - await this.streamSend(StreamCode.DATA, data).catch((e) => controller.error(e)); + await this.streamSend(StreamCode.DATA, data); + // Decrement the desired size and resolved the old promise as to not block application exit this.writableDesiredSize =- data.length; + oldProm.resolveP(); if (isChunkable) { await writableWrite(chunk.subarray(bytesWritten), controller); } } catch { - this.writableDesiredSizeProm.resolveP(); - // TODO: Handle error + this.writableDesiredSizeProm = oldProm; } } @@ -131,7 +146,7 @@ class WebSocketStream this.signalWritableEnd(); }, abort: async (reason?: any) => { - this.signalWritableEnd(reason); + this.signalWritableEnd(true, reason); }, }, { @@ -140,19 +155,47 @@ class WebSocketStream ); } + public get readableEnded(): boolean { + return this._readableEnded; + } + + public get writableEnded(): boolean { + return this.writableEnded; + } + + public get destroyedP() { + return this.destroyProm.p; + } + public async destroy() { this.logger.info(`Destroy ${this.constructor.name}`); // Force close any open streams - this.cancel(); + this.writableDesiredSizeProm.resolveP(); + this.cancel(new errors.ErrorWebSocketStreamClose()); // Removing stream from the connection's stream map this.connection.streamMap.delete(this.streamId); + this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); this.logger.info(`Destroyed ${this.constructor.name}`); } - public async streamSend(code: StreamCode): Promise; - public async streamSend(code: StreamCode.ACK, payloadSize: number): Promise; - public async streamSend(code: StreamCode.DATA, data: Uint8Array): Promise; - public async streamSend(code: StreamCode, data_?: Uint8Array | number): Promise { + /** + * Send a code with no payload on the stream. + * @param code - The stream code to send. + */ + protected async streamSend(code: StreamCode): Promise; + /** + * Send an ACK frame with a payloadSize. + * @param code - ACK + * @param payloadSize + */ + protected async streamSend(code: StreamCode.ACK, payloadSize: number): Promise; + /** + * Send a DATA frame with a payload on the stream. + * @param code - DATA + * @param data - The payload to send. + */ + protected async streamSend(code: StreamCode.DATA, data: Uint8Array): Promise; + protected async streamSend(code: StreamCode, data_?: Uint8Array | number): Promise { let data: Uint8Array | undefined; if (code === StreamCode.ACK && typeof data_ === 'number') { data = new Uint8Array(4); @@ -167,26 +210,23 @@ class WebSocketStream if (data != null) { array.set(data, 1); } - try { - await this.connection.streamSend(this.streamId, array); - } - catch (e) { - this.signalWritableEnd(e); - throw e; - } + await this.connection.streamSend(this.streamId, array); } public async streamRecv(message: Uint8Array) { + if (message.length === 0) { + this.logger.debug('received empty message, closing stream'); + this.signalReadableEnd(true, new errors.ErrorWebSocketStream()); + return; + } const code = message[0] as StreamCode; const data = message.subarray(1); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); if (code === StreamCode.DATA) { - if (this.readableController.desiredSize == null) { - this.readableController.error(new errors.ErrorWebSocketUndefinedBehaviour()); - return; - } - if (data.length > this.readableController.desiredSize) { - this.readableController.error(new errors.ErrorWebSocketStream()); + if (this.readableController.desiredSize != null && data.length > this.readableController.desiredSize) { + if (!this._readableEnded) { + this.signalReadableEnd(true, new errors.ErrorWebSocketStream()); + } return; } this.readableController.enqueue(data); @@ -198,10 +238,10 @@ class WebSocketStream this.logger.debug(`received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`); } else if (code === StreamCode.ERROR) { - this.readableController.error(new errors.ErrorWebSocketStream()); + this.cancel(new errors.ErrorWebSocketStream()); } else if (code === StreamCode.CLOSE) { - // close the stream + this.cancel(); } } @@ -213,11 +253,11 @@ class WebSocketStream // Close the streams with the given error, if (!this._readableEnded) { this.readableController.error(reason); - this.signalReadableEnd(reason); + this.signalReadableEnd(true, reason); } if (!this._writableEnded) { this.writableController.error(reason); - this.signalWritableEnd(reason); + this.signalWritableEnd(true, reason); } } @@ -225,22 +265,67 @@ class WebSocketStream * Signals the end of the ReadableStream. to be used with the extended class * to track the streams state. */ - protected signalReadableEnd(reason?: any) { + protected signalReadableEnd(isError: boolean = false, reason?: any) { + if (isError) this.logger.debug(`readable ended with error ${reason.message}`); if (this._readableEnded) return; + this.logger.debug(`end readable`); + // indicate that receiving side is closed this._readableEnded = true; - if (reason == null) this._readableEndedProm.resolveP(); - else this._readableEndedProm.rejectP(reason); + if (isError) { + this.readableController.error(reason); + } + if (this._readableEnded && this._writableEnded) { + this.destroyProm.resolveP(); + void this.streamSend(StreamCode.CLOSE); + if (this[status] !== 'destroying') void this.destroy(); + } + this.logger.debug(`readable ended`); } /** * Signals the end of the WritableStream. to be used with the extended class * to track the streams state. */ - protected signalWritableEnd(reason?: any) { + protected signalWritableEnd( + isError: boolean = false, + reason?: any, + ) { + if (isError) this.logger.debug(`writable ended with error ${reason.message}`); if (this._writableEnded) return; + // indicate that sending side is closed this._writableEnded = true; - if (reason == null) this._writableEndedProm.resolveP(); - else this._writableEndedProm.rejectP(reason); + if (isError) { + this.readableController.error(reason); + } + if (this._readableEnded && this._writableEnded) { + this.destroyProm.resolveP(); + void this.streamSend(StreamCode.CLOSE); + if (this[status] !== 'destroying') void this.destroy(); + } + this.logger.debug(`writable ended`); + } + + /** + * This will process any errors from a `streamSend` or `streamRecv`, extract the code and covert to a reason. + * Will return null if the error was not an expected stream ending error. + */ + protected async processSendStreamError( + e: Error, + type: 'recv' | 'send', + ): Promise { + let match = + e.message.match(/StreamStopped\((.+)\)/) ?? + e.message.match(/StreamReset\((.+)\)/); + if (match != null) { + const code = parseInt(match[1]); + return await this.codeToReason(type, code); + } + match = e.message.match(/InvalidStreamState\((.+)\)/); + if (match != null) { + // `InvalidStreamState()` returns the stream ID and not any actual error code + return never('Should never reach an [InvalidState(StreamId)] error'); + } + return null; } } From 0b9bb934793aa826d805b2e2774d29863c64f13e Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:36:34 +1000 Subject: [PATCH 018/149] feat: WebSocketConnection stopping with error codes --- src/WebSocketConnection.ts | 48 ++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 85b64a44..2b5430a5 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -234,7 +234,7 @@ class WebSocketConnection extends EventTarget { try { await Promise.race([connection.start(), abortProm.p]); } catch (e) { - await connection.stop({ force: true }); + await connection.stop(); throw e; } finally { ctx.signal.removeEventListener('abort', abortHandler); @@ -376,7 +376,7 @@ class WebSocketConnection extends EventTarget { this.socket.once('close', () => { this.resolveClosedP(); if (this[startStop.running] && this[startStop.status] !== 'stopping') { - void this.stop({ force: true }); + void this.stop(); } }); @@ -419,26 +419,40 @@ class WebSocketConnection extends EventTarget { }); } + /** + * Send data to the other side of the connection. + * This will not will not error out, but will rather close the connection assuming any further communication is expected to fail. + * @param streamId The stream id to send the data on + * @param data The data to send, this will include the stream message type. + * @internal + */ public async streamSend(streamId: StreamId, data: Uint8Array) { - const sendProm = promise(); - const encodedStreamId = fromStreamId(streamId); const array = new Uint8Array(encodedStreamId.length + data.length); array.set(encodedStreamId, 0); array.set(data, encodedStreamId.length); - if (data != null) { - array.set(data, encodedStreamId.length); - } - - this.socket.send(array, (err) => { - if (err == null) sendProm.resolveP(); - else sendProm.rejectP(err); - }); - await sendProm.p; + try { + const sendProm = promise(); + this.socket.send(array, (err) => { + if (err == null) sendProm.resolveP(); + else sendProm.rejectP(err); + }); + await sendProm.p; + } + catch (err) { + await this.stop(); + } } - public async stop({ force = false }: { force: boolean }) { + + public async stop({ + errorCode = 1000, + errorMessage = '', + }: { + errorCode?: number, + errorMessage?: string, + } = {}) { this.logger.info(`Stop ${this.constructor.name}`); // Cleaning up existing streams const streamsDestroyP: Array> = []; @@ -455,7 +469,7 @@ class WebSocketConnection extends EventTarget { if (this.socket.readyState === ws.CLOSED) { this.resolveClosedP(); } else { - this.socket.close(); + this.socket.close(errorCode, errorMessage); } await this.closedP; this.logger.debug('closedP'); @@ -465,7 +479,7 @@ class WebSocketConnection extends EventTarget { this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); if (this.type === 'server') { - this.parentInstance!.connectionMap.delete(this.connectionId); + this.parentInstance.connectionMap.delete(this.connectionId); } this.dispatchEvent(new events.WebSocketConnectionStopEvent()); @@ -482,7 +496,7 @@ class WebSocketConnection extends EventTarget { }), ); if (this[startStop.running] && this[startStop.status] !== 'stopping') { - void this.stop({ force: true }); + void this.stop(); } }; // If there was an existing timer, we cancel it and set a new one From 4bb95e37d7bd7115a89b97c39bacd047d4ad6d4d Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:37:24 +1000 Subject: [PATCH 019/149] feat: mocking for WebSocketStream tests --- tests/WebSocketStream.test.ts | 68 +++++++++++++++++++++++++++-------- tests/utils.test.ts | 4 +-- tests/utils.ts | 13 +++++++ 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 9007c520..74370037 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -3,6 +3,8 @@ import WebSocketStream from "@/WebSocketStream"; import WebSocketConnection from "@/WebSocketConnection"; import * as events from "@/events"; import { promise } from "@/utils"; +import { fc, testProp } from "@fast-check/jest"; +import * as testUtils from './utils'; const DEFAULT_BUFFER_SIZE = 1024; @@ -44,14 +46,15 @@ const connectionMock = jest.mocked(WebSocketConnection, true); describe(WebSocketStream.name, () => { let connection1: WebSocketConnection; let connection2: WebSocketConnection; - let stream1: WebSocketStream; - let stream2: WebSocketStream; beforeEach(async () => { connectionMock.mockClear(); connection1 = new (WebSocketConnection as any)(); connection2 = new (WebSocketConnection as any)(); (connection1 as any).connectTo(connection2); - stream1 = await WebSocketStream.createWebSocketStream({ + }); + + async function createStreamPair(connection1, connection2) { + const stream1 = await WebSocketStream.createWebSocketStream({ streamId: 0n as StreamId, bufferSize: DEFAULT_BUFFER_SIZE, connection: connection1 @@ -59,15 +62,52 @@ describe(WebSocketStream.name, () => { const createStream2Prom = promise(); connection2.addEventListener("connectionStream", (e: events.WebSocketConnectionStreamEvent) => { createStream2Prom.resolveP(e.detail); - }); - stream2 = await createStream2Prom.p; - }); - test('buffering', async () => { - const stream1Readable = stream1.readable; - const stream2Writable = stream2.writable; - const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE+1); - const writeProm = stream2Writable.getWriter().write(buffer); - await stream1Readable.getReader().read() - await writeProm; - }); + }, { once: true }); + const stream2 = await createStream2Prom.p; + return [stream1, stream2]; + } + // testProp( + // 'normal', + // [fc.infiniteStream(fc.uint8Array({minLength: 1, maxLength: DEFAULT_BUFFER_SIZE}))], + // async (iterable) => { + + // const writingTest = async () => { + // const stream2Writable = stream2.writable; + // const stream = testUtils.toReadableStream(iterable); + // await stream.pipeTo(stream2Writable); + // } + + // const readingTest = async () => { + // const stream1Readable = stream1.readable; + // await stream1Readable.pipeTo([]); + // } + + // await Promise.all([writingTest(), readingTest()]); + + // // const stream2Writable = stream2.writable; + // // const buffer = new Uint8Array(2); + // // const writeProm = stream2Writable.getWriter().write(buffer); + // // await stream1Readable.getReader().read(); + // // await writeProm; + // // await stream1.destroy(); + // } + // ); + test( + 'normal', + async () => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); + + const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE); + testUtils.randomBytes(buffer); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + const writeProm = stream2Writable.getWriter().write(buffer); + await stream1Readable.getReader().read(); + await writeProm; + + await stream1.destroy(); + await stream2.destroy(); + } + ); }); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 19ee9f18..3f386a88 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -2,12 +2,12 @@ import type { StreamId } from '@/types'; import { fc, testProp } from '@fast-check/jest'; import * as utils from '@/utils'; -const MAX_62_BIT_UINT = 2n ** 62n - 1n; +const MAX_62_BIT_UINT = ; describe('utils', () => { testProp( 'from/to StreamId', - [fc.bigUint().filter((n) => n <= MAX_62_BIT_UINT)], + [fc.bigUint({ max: 2n ** 62n - 1n })], (input) => { const array = utils.fromStreamId(input as StreamId); const { data: id } = utils.toStreamId(array); diff --git a/tests/utils.ts b/tests/utils.ts index f2a5f9bc..e9a71099 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -588,6 +588,18 @@ async function generateConfig(type: KeyTypes): Promise { }; } +function toReadableStream(iterator: IterableIterator) { + return new ReadableStream({ + async start(controller) { + for await (const chunk of iterator) { + controller.enqueue(chunk); + } + controller.close(); + } + }); +} + + export { sleep, randomBytes, @@ -603,6 +615,7 @@ export { signHMAC, verifyHMAC, generateConfig, + toReadableStream, }; export type { KeyTypes, TLSConfigs }; From 22dd2601bf361a654c7a1807ff88665d35f3b307 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:47:01 +1000 Subject: [PATCH 020/149] feat: WebSocketStream bidi closing --- src/WebSocketConnection.ts | 33 ++++++- src/WebSocketStream.ts | 181 +++++++++++++++++++--------------- src/errors.ts | 16 +-- src/types.ts | 4 +- src/utils/utils.ts | 40 +++++--- tests/WebSocketStream.test.ts | 1 - 6 files changed, 163 insertions(+), 112 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 2b5430a5..094466ff 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -2,9 +2,10 @@ import type { PromiseCancellable } from '@matrixai/async-cancellable'; import type { ContextTimed, ContextTimedInput } from '@matrixai/contexts'; import type { Host, - PromiseDeconstructed, RemoteInfo, + StreamCodeToReason, StreamId, + StreamReasonToCode, VerifyCallback, WebSocketConfig, } from './types'; @@ -60,6 +61,18 @@ class WebSocketConnection extends EventTarget { protected config: WebSocketConfig; + /** + * Converts reason to code. + * Used during `QUICStream` creation. + */ + protected reasonToCode: StreamReasonToCode; + + /** + * Converts code to reason. + * Used during `QUICStream` creation. + */ + protected codeToReason: StreamCodeToReason; + /** * Internal stream map. * This is also used by `WebSocketStream`. @@ -181,6 +194,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: undefined; client?: WebSocketClient; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: VerifyCallback; logger?: Logger; } @@ -192,6 +207,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: WebSocketServer; client?: undefined; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: undefined; logger?: Logger; }, @@ -208,6 +225,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: undefined; client?: WebSocketClient; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: VerifyCallback; logger?: Logger; } @@ -219,6 +238,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: WebSocketServer; client?: undefined; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: undefined; logger?: Logger; }, @@ -254,6 +275,8 @@ class WebSocketConnection extends EventTarget { socket, server, client, + reasonToCode = () => 0n, + codeToReason = (type, code) => new Error(`${type} ${code}`), verifyCallback, logger, }: @@ -265,6 +288,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: undefined; client?: WebSocketClient; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: VerifyCallback; logger?: Logger; } @@ -276,6 +301,8 @@ class WebSocketConnection extends EventTarget { socket: ws.WebSocket; server?: WebSocketServer; client?: undefined; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; verifyCallback?: undefined; logger?: Logger; }) { @@ -287,6 +314,8 @@ class WebSocketConnection extends EventTarget { this.type = type; this.parentInstance = server ?? client!; this._remoteHost = remoteInfo.host; + this.reasonToCode = reasonToCode; + this.codeToReason = codeToReason; this.verifyCallback = verifyCallback; const { @@ -402,6 +431,8 @@ class WebSocketConnection extends EventTarget { streamId: streamId!, connection: this, bufferSize: this.config.streamBufferSize, + codeToReason: this.codeToReason, + reasonToCode: this.reasonToCode, logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); stream.addEventListener( diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index d38299c6..25de370e 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,7 +1,7 @@ import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; import type { StreamCodeToReason, StreamId, StreamReasonToCode } from './types'; -import { never, promise, StreamCode } from './utils'; +import { fromVarInt, never, promise, StreamType, StreamShutdown, toVarInt } from './utils'; import type WebSocketConnection from './WebSocketConnection'; import * as errors from './errors'; import * as events from './events'; @@ -35,7 +35,7 @@ class WebSocketStream streamId, connection, bufferSize, - reasonToCode = () => 0, + reasonToCode = () => 0n, codeToReason = (type, code) => new Error(`${type.toString()} ${code.toString()}`), logger = new Logger(`${this.name} ${streamId}`), }: { @@ -90,12 +90,12 @@ class WebSocketStream }, pull: async (controller) => { if (controller.desiredSize != null && controller.desiredSize > 0) { - await this.streamSend(StreamCode.ACK, controller.desiredSize!); + await this.streamSend(StreamType.ACK, controller.desiredSize!); } }, cancel: async (reason) => { this.logger.debug(`readable aborted with [${reason.message}]`); - this.signalReadableEnd(true, reason); + await this.signalReadableEnd(true, reason); }, }, new ByteLengthQueuingStrategy({ @@ -123,7 +123,7 @@ class WebSocketStream this.writableDesiredSizeProm = promise(); } const bytesWritten = this.writableDesiredSize; - await this.streamSend(StreamCode.DATA, data); + await this.streamSend(StreamType.DATA, data); // Decrement the desired size and resolved the old promise as to not block application exit this.writableDesiredSize =- data.length; oldProm.resolveP(); @@ -143,10 +143,10 @@ class WebSocketStream }, write: writableWrite, close: async () => { - this.signalWritableEnd(); + await this.signalWritableEnd(); }, abort: async (reason?: any) => { - this.signalWritableEnd(true, reason); + await this.signalWritableEnd(true, reason); }, }, { @@ -171,42 +171,62 @@ class WebSocketStream this.logger.info(`Destroy ${this.constructor.name}`); // Force close any open streams this.writableDesiredSizeProm.resolveP(); - this.cancel(new errors.ErrorWebSocketStreamClose()); + this.cancel(); // Removing stream from the connection's stream map this.connection.streamMap.delete(this.streamId); this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); this.logger.info(`Destroyed ${this.constructor.name}`); } - /** - * Send a code with no payload on the stream. - * @param code - The stream code to send. - */ - protected async streamSend(code: StreamCode): Promise; /** * Send an ACK frame with a payloadSize. * @param code - ACK - * @param payloadSize + * @param payloadSize - The number of bytes that the receiver can accept. */ - protected async streamSend(code: StreamCode.ACK, payloadSize: number): Promise; - /** - * Send a DATA frame with a payload on the stream. - * @param code - DATA - * @param data - The payload to send. - */ - protected async streamSend(code: StreamCode.DATA, data: Uint8Array): Promise; - protected async streamSend(code: StreamCode, data_?: Uint8Array | number): Promise { + protected async streamSend(type: StreamType.ACK, payloadSize: number): Promise; + /** + * Send a DATA frame with a payload on the stream. + * @param code - DATA + * @param data - The payload to send. + */ + protected async streamSend(type: StreamType.DATA, data: Uint8Array): Promise; + /** + * Send an ERROR frame with a payload on the stream. + * @param code - CLOSE + * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. + */ + protected async streamSend(type: StreamType.ERROR, shutdown: StreamShutdown, code: bigint): Promise; + /** + * Send a CLOSE frame with a payload on the stream. + * @param code - CLOSE + * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. + */ + protected async streamSend(type: StreamType.CLOSE, shutdown: StreamShutdown): Promise; + protected async streamSend(type: StreamType, data_?: Uint8Array | number, code?: bigint): Promise { let data: Uint8Array | undefined; - if (code === StreamCode.ACK && typeof data_ === 'number') { + if (type === StreamType.ACK && typeof data_ === 'number') { data = new Uint8Array(4); const dv = new DataView(data.buffer); dv.setUint32(0, data_, false); - } else { - data = data_ as Uint8Array | undefined; + } else if (type === StreamType.DATA) { + data = data_ as Uint8Array; + } + else if (type === StreamType.ERROR) { + const errorCode = fromVarInt(code!); + data = new Uint8Array(1 + errorCode.length); + const dv = new DataView(data.buffer); + dv.setUint8(0, data_ as StreamShutdown); + data.set(errorCode, 1); + } + else if (type === StreamType.CLOSE) { + data = new Uint8Array([data_ as StreamShutdown]); + } + else { + never(); } const arrayLength = 1 + (data?.length ?? 0); const array = new Uint8Array(arrayLength); - array.set([code], 0); + array.set([type], 0); if (data != null) { array.set(data, 1); } @@ -214,34 +234,43 @@ class WebSocketStream } public async streamRecv(message: Uint8Array) { - if (message.length === 0) { - this.logger.debug('received empty message, closing stream'); - this.signalReadableEnd(true, new errors.ErrorWebSocketStream()); - return; - } - const code = message[0] as StreamCode; + const type = message[0] as StreamType; const data = message.subarray(1); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); - if (code === StreamCode.DATA) { - if (this.readableController.desiredSize != null && data.length > this.readableController.desiredSize) { - if (!this._readableEnded) { - this.signalReadableEnd(true, new errors.ErrorWebSocketStream()); - } - return; - } - this.readableController.enqueue(data); - } - else if (code === StreamCode.ACK) { + if (type === StreamType.ACK) { const bufferSize = dv.getUint32(0, false); this.writableDesiredSize = bufferSize; this.writableDesiredSizeProm.resolveP(); this.logger.debug(`received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`); } - else if (code === StreamCode.ERROR) { - this.cancel(new errors.ErrorWebSocketStream()); + else if (type === StreamType.DATA) { + if (this.readableController.desiredSize != null && data.length > this.readableController.desiredSize) { + await this.signalReadableEnd(true, new errors.ErrorWebSocketStreamReaderBufferOverload()); + return; + } + this.readableController.enqueue(data); + } + else if (type === StreamType.ERROR || type === StreamType.CLOSE) { + const shutdown = dv.getUint8(0) as StreamShutdown; + let isError = false; + let reason: any; + if (type === StreamType.ERROR) { + isError = true; + const errorCode = toVarInt(data.subarray(1)).data; + reason = await this.codeToReason('recv', errorCode); + } + if (shutdown === StreamShutdown.Read) { + await this.signalReadableEnd(isError, reason); + } + else if (shutdown === StreamShutdown.Write) { + await this.signalWritableEnd(isError, reason); + } + else { + never(); + } } - else if (code === StreamCode.CLOSE) { - this.cancel(); + else { + never(); } } @@ -249,15 +278,16 @@ class WebSocketStream * Forces the active stream to end early */ public cancel(reason?: any): void { - reason = reason ?? new errors.ErrorWebSocketStreamCancel(); + console.log(reason) + const isError = reason != null; // Close the streams with the given error, if (!this._readableEnded) { this.readableController.error(reason); - this.signalReadableEnd(true, reason); + void this.signalReadableEnd(isError, reason); } if (!this._writableEnded) { this.writableController.error(reason); - this.signalWritableEnd(true, reason); + void this.signalWritableEnd(isError, reason); } } @@ -265,18 +295,23 @@ class WebSocketStream * Signals the end of the ReadableStream. to be used with the extended class * to track the streams state. */ - protected signalReadableEnd(isError: boolean = false, reason?: any) { - if (isError) this.logger.debug(`readable ended with error ${reason.message}`); + protected async signalReadableEnd(isError: boolean = false, reason?: any) { + if (isError) this.logger.debug(`ending readable with error ${reason.message}`); + else this.logger.debug(`ending readable`); if (this._readableEnded) return; - this.logger.debug(`end readable`); // indicate that receiving side is closed this._readableEnded = true; + // shutdown the write side of the other stream if (isError) { - this.readableController.error(reason); + const code = await this.reasonToCode('send', reason); + this.streamSend(StreamType.ERROR, StreamShutdown.Write, code); + } + else { + this.streamSend(StreamType.CLOSE, StreamShutdown.Write); } + this.readableController.error(reason); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); - void this.streamSend(StreamCode.CLOSE); if (this[status] !== 'destroying') void this.destroy(); } this.logger.debug(`readable ended`); @@ -286,47 +321,31 @@ class WebSocketStream * Signals the end of the WritableStream. to be used with the extended class * to track the streams state. */ - protected signalWritableEnd( + protected async signalWritableEnd( isError: boolean = false, reason?: any, ) { - if (isError) this.logger.debug(`writable ended with error ${reason.message}`); + if (isError) this.logger.debug(`ending writable with error ${reason.message}`); + else this.logger.debug(`ending writable`); if (this._writableEnded) return; // indicate that sending side is closed this._writableEnded = true; + // shutdown the read side of the other stream if (isError) { - this.readableController.error(reason); + const code = await this.reasonToCode('send', reason); + this.streamSend(StreamType.ERROR, StreamShutdown.Read, code); + + } + else { + this.streamSend(StreamType.CLOSE, StreamShutdown.Read); } + this.writableController.error(reason); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); - void this.streamSend(StreamCode.CLOSE); if (this[status] !== 'destroying') void this.destroy(); } this.logger.debug(`writable ended`); } - - /** - * This will process any errors from a `streamSend` or `streamRecv`, extract the code and covert to a reason. - * Will return null if the error was not an expected stream ending error. - */ - protected async processSendStreamError( - e: Error, - type: 'recv' | 'send', - ): Promise { - let match = - e.message.match(/StreamStopped\((.+)\)/) ?? - e.message.match(/StreamReset\((.+)\)/); - if (match != null) { - const code = parseInt(match[1]); - return await this.codeToReason(type, code); - } - match = e.message.match(/InvalidStreamState\((.+)\)/); - if (match != null) { - // `InvalidStreamState()` returns the stream ID and not any actual error code - return never('Should never reach an [InvalidState(StreamId)] error'); - } - return null; - } } export default WebSocketStream; diff --git a/src/errors.ts b/src/errors.ts index 376d67be..f91b6a47 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -62,17 +62,12 @@ class ErrorWebSocketStream extends ErrorWebSocket { static description = 'WebSocket Stream error'; } -class ErrorWebSocketStreamDestroyed extends ErrorWebSocketStream { - static description = 'WebSocket Stream is destroyed'; -} - -class ErrorWebSocketStreamClose extends ErrorWebSocketStream { - static description = 'WebSocket Stream force close'; +class ErrorWebSocketStreamReaderBufferOverload extends ErrorWebSocket { + static description = 'WebSocket Stream readable buffer has overloaded'; } -class ErrorWebSocketStreamCancel extends ErrorWebSocketStream { - static description = - 'WebSocket Stream was cancelled without a provided reason'; +class ErrorWebSocketStreamDestroyed extends ErrorWebSocketStream { + static description = 'WebSocket Stream is destroyed'; } // Misc @@ -94,8 +89,7 @@ export { ErrorWebSocketConnectionStartTimeOut, ErrorWebSocketConnectionKeepAliveTimeOut, ErrorWebSocketStream, + ErrorWebSocketStreamReaderBufferOverload, ErrorWebSocketStreamDestroyed, - ErrorWebSocketStreamClose, - ErrorWebSocketStreamCancel, ErrorWebSocketUndefinedBehaviour, }; diff --git a/src/types.ts b/src/types.ts index a0cc062a..2e621f21 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,14 +63,14 @@ type RemoteInfo = { type StreamReasonToCode = ( type: 'recv' | 'send', reason?: any, -) => number | PromiseLike; +) => bigint | PromiseLike; /** * Maps code to a reason. 0 usually indicates unknown/default reason. */ type StreamCodeToReason = ( type: 'recv' | 'send', - code: number, + code: bigint, ) => any | PromiseLike; type ConnectionMetadata = { diff --git a/src/utils/utils.ts b/src/utils/utils.ts index b567d8ce..e26af2d2 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -2,8 +2,8 @@ import type { PromiseDeconstructed } from './types'; import type { Parsed, StreamId } from '@/types'; import * as errors from '../errors'; -function never(): never { - throw new errors.ErrorWebSocketUndefinedBehaviour(); +function never(message?: string): never { + throw new errors.ErrorWebSocketUndefinedBehaviour(message); } /** @@ -22,7 +22,8 @@ function promise(): PromiseDeconstructed { }; } -function toStreamId(array: Uint8Array): Parsed { + +function toVarInt(array: Uint8Array): Parsed { let streamId: bigint; // Get header and prefix @@ -57,36 +58,34 @@ function toStreamId(array: Uint8Array): Parsed { break; } return { - data: streamId! as StreamId, + data: streamId!, remainder: array.subarray(readBytes), }; } -function fromStreamId(streamId: StreamId): Uint8Array { - const id = streamId as bigint; - +function fromVarInt(varInt: bigint): Uint8Array { let array: Uint8Array; let dv: DataView; let prefixMask = 0; - if (id < 0x40) { + if (varInt < 0x40) { array = new Uint8Array(1); dv = new DataView(array.buffer); - dv.setUint8(0, Number(id)); - } else if (id < 0x4000) { + dv.setUint8(0, Number(varInt)); + } else if (varInt < 0x4000) { array = new Uint8Array(2); dv = new DataView(array.buffer); - dv.setUint16(0, Number(id)); + dv.setUint16(0, Number(varInt)); prefixMask = 0b01_000000; - } else if (id < 0x40000000) { + } else if (varInt < 0x40000000) { array = new Uint8Array(4); dv = new DataView(array.buffer); - dv.setUint32(0, Number(id)); + dv.setUint32(0, Number(varInt)); prefixMask = 0b10_000000; } else { array = new Uint8Array(8); dv = new DataView(array.buffer); - dv.setBigUint64(0, id); + dv.setBigUint64(0, varInt); prefixMask = 0b11_000000; } @@ -97,11 +96,20 @@ function fromStreamId(streamId: StreamId): Uint8Array { return array; } -enum StreamCode { +const fromStreamId = fromVarInt as (streamId: StreamId) => Uint8Array; +const toStreamId = toVarInt as (array: Uint8Array) => Parsed; + +enum StreamType { DATA = 0, ACK = 1, ERROR = 2, CLOSE = 3, } -export { never, promise, toStreamId, fromStreamId, StreamCode }; +enum StreamShutdown { + Read = 0, + Write = 1 +} + + +export { never, promise, toVarInt, fromVarInt, toStreamId, fromStreamId, StreamType, StreamShutdown }; diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 74370037..e89a4ff7 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -107,7 +107,6 @@ describe(WebSocketStream.name, () => { await writeProm; await stream1.destroy(); - await stream2.destroy(); } ); }); From a0b4f46fe8ff4b8bc309af7e37a2e96c6290345b Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:52:34 +1000 Subject: [PATCH 021/149] feat: added ability to specify codeToReason and reasonToCode from WebSocketServer --- src/WebSocketServer.ts | 19 +++++++++++++++---- tests/WebSocketConnection.test.ts | 0 tests/utils.test.ts | 2 -- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 tests/WebSocketConnection.test.ts diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 47a3f003..90de8741 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -1,6 +1,6 @@ import type { IncomingMessage, ServerResponse } from 'http'; import type tls from 'tls'; -import type { Host, Port, WebSocketConfig } from './types'; +import type { Host, Port, StreamCodeToReason, StreamReasonToCode, WebSocketConfig } from './types'; import https from 'https'; import { startStop, status } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; @@ -27,9 +27,12 @@ class WebSocketServer extends EventTarget { protected config: WebSocketConfig; protected server: https.Server; protected webSocketServer: ws.WebSocketServer; + protected reasonToCode: StreamReasonToCode | undefined; + protected codeToReason: StreamCodeToReason | undefined; + public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); + protected _port: number; protected _host: string; - public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); protected handleWebSocketConnectionEvents = ( event: webSocketEvents.WebSocketConnectionEvent, @@ -62,6 +65,8 @@ class WebSocketServer extends EventTarget { */ constructor({ config, + reasonToCode, + codeToReason, logger, }: { config: Partial & { @@ -69,6 +74,8 @@ class WebSocketServer extends EventTarget { cert: string; ca?: string; }; + reasonToCode?: StreamReasonToCode; + codeToReason?: StreamCodeToReason; logger?: Logger; }) { super(); @@ -78,6 +85,8 @@ class WebSocketServer extends EventTarget { }; this.logger = logger ?? new Logger(this.constructor.name); this.config = wsConfig; + this.reasonToCode = reasonToCode; + this.codeToReason = codeToReason; } public async start({ @@ -121,7 +130,7 @@ class WebSocketServer extends EventTarget { for (const webSocketConnection of this.connectionMap.values()) { destroyProms.push( webSocketConnection.stop({ - force, + errorMessage: 'cleaning up connections', }), ); } @@ -207,8 +216,10 @@ class WebSocketServer extends EventTarget { host: (httpSocket.remoteAddress ?? '') as Host, port: (httpSocket.remotePort ?? 0) as Port, }, - config: this.config, socket: webSocket, + config: this.config, + reasonToCode: this.reasonToCode, + codeToReason: this.codeToReason, logger: this.logger.getChild( `${WebSocketConnection.name} ${connectionId}`, ), diff --git a/tests/WebSocketConnection.test.ts b/tests/WebSocketConnection.test.ts new file mode 100644 index 00000000..e69de29b diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 3f386a88..b47f375a 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -2,8 +2,6 @@ import type { StreamId } from '@/types'; import { fc, testProp } from '@fast-check/jest'; import * as utils from '@/utils'; -const MAX_62_BIT_UINT = ; - describe('utils', () => { testProp( 'from/to StreamId', From 513856e2c5f80fdc6d2d87ff1bbd335aada1b213 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 17:01:03 +1000 Subject: [PATCH 022/149] lintfix --- src/WebSocketClient.ts | 39 +++---- src/WebSocketConnection.ts | 36 +++---- src/WebSocketConnectionMap.ts | 4 +- src/WebSocketServer.ts | 49 +++++---- src/WebSocketStream.ts | 191 ++++++++++++++++++++-------------- src/config.ts | 6 +- src/errors.ts | 3 +- src/events.ts | 1 + src/utils/utils.ts | 15 ++- tests/WebSocketServer.test.ts | 12 +-- tests/WebSocketStream.test.ts | 63 +++++------ tests/utils.ts | 3 +- 12 files changed, 227 insertions(+), 195 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 3821e837..6e1aa6f9 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -6,11 +6,11 @@ import Logger from '@matrixai/logger'; import WebSocket from 'ws'; import { Validator } from 'ip-num'; import { Timer } from '@matrixai/timer'; +import Counter from 'resource-counter'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; -import Counter from 'resource-counter'; import WebSocketConnectionMap from './WebSocketConnectionMap'; import { clientDefault } from './config'; @@ -61,8 +61,7 @@ class WebSocketClient extends EventTarget { let port_: Port; if (port >= 0 && port <= 65535) { port_ = port as Port; - } - else { + } else { throw new errors.ErrorWebSocketClientInvalidHost(); } @@ -70,7 +69,7 @@ class WebSocketClient extends EventTarget { const client = new this({ address, - logger + logger, }); const webSocket = new WebSocket(address, { @@ -89,7 +88,7 @@ class WebSocketClient extends EventTarget { socket: webSocket, verifyCallback, client: client, - }) + }); logger.info(`Created ${this.name}`); return client; } @@ -97,16 +96,10 @@ class WebSocketClient extends EventTarget { protected address: string; protected logger: Logger; - public readonly connectionIdCounter = new Counter(0); - public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); + public readonly connectionMap: WebSocketConnectionMap = + new WebSocketConnectionMap(); - constructor({ - address, - logger, - } : { - address: string, - logger: Logger - }) { + constructor({ address, logger }: { address: string; logger: Logger }) { super(); this.address = address; this.logger = logger; @@ -115,15 +108,11 @@ class WebSocketClient extends EventTarget { public async destroy(force: boolean = false) { this.logger.info(`Destroying ${this.constructor.name}`); if (force) { - for (const activeConnection of this.activeConnections) { - activeConnection.cancel( - new errors.ErrorClientEndingConnections( - 'Destroying WebSocketClient', - ), - ); + for (const activeConnection of this.connectionMap.values()) { + activeConnection.stop({ force }); } } - for (const activeConnection of this.activeConnections) { + for (const activeConnection of this.connectionMap.values()) { // Ignore errors here, we only care that it finishes await activeConnection.endedProm.catch(() => {}); } @@ -133,9 +122,7 @@ class WebSocketClient extends EventTarget { @createDestroy.ready(new errors.ErrorWebSocketClientDestroyed()) public async stopConnections() { for (const activeConnection of this.activeConnections) { - activeConnection.cancel( - new errors.ErrorClientEndingConnections(), - ); + activeConnection.cancel(new errors.ErrorClientEndingConnections()); } for (const activeConnection of this.activeConnections) { // Ignore errors here, we only care that it finished @@ -158,9 +145,7 @@ class WebSocketClient extends EventTarget { }); void timer.then( () => { - abortRaceProm.rejectP( - new errors.ErrorClientConnectionTimedOut(), - ); + abortRaceProm.rejectP(new errors.ErrorClientConnectionTimedOut()); }, () => {}, // Ignore cancellation errors ); diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 094466ff..78b6005f 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -11,6 +11,8 @@ import type { } from './types'; import type WebSocketClient from './WebSocketClient'; import type WebSocketServer from './WebSocketServer'; +import type { DetailedPeerCertificate, TLSSocket } from 'tls'; +import type WebSocketConnectionMap from './WebSocketConnectionMap'; import { startStop } from '@matrixai/async-init'; import { Lock } from '@matrixai/async-locks'; import { context, timedCancellable } from '@matrixai/contexts/dist/decorators'; @@ -22,9 +24,6 @@ import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { fromStreamId, promise, toStreamId } from './utils'; import * as events from './events'; -import { Counter } from 'resource-counter'; -import WebSocketConnectionMap from './WebSocketConnectionMap'; -import { DetailedPeerCertificate, TLSSocket } from 'tls'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -115,7 +114,7 @@ class WebSocketConnection extends EventTarget { protected keepAliveIntervalTimer?: Timer; protected parentInstance: { - connectionMap: WebSocketConnectionMap + connectionMap: WebSocketConnectionMap; }; protected logger: Logger; protected _remoteHost: Host; @@ -136,7 +135,9 @@ class WebSocketConnection extends EventTarget { protected resolveClosedP: () => void; protected rejectClosedP: (reason?: any) => void; - protected verifyCallback: ((peerCert: DetailedPeerCertificate) => Promise) | undefined; + protected verifyCallback: + | ((peerCert: DetailedPeerCertificate) => Promise) + | undefined; protected messageHandler = async (data: ws.RawData, isBinary: boolean) => { if (!isBinary || data instanceof Array) { @@ -173,7 +174,7 @@ class WebSocketConnection extends EventTarget { ); } - stream!.streamRecv(message); + await stream!.streamRecv(message); }; protected pingHandler = () => { @@ -373,8 +374,7 @@ class WebSocketConnection extends EventTarget { remotePort: request.connection.remotePort ?? 0, peerCert, }); - } - catch (e) { + } catch (e) { authenticateProm.rejectP(e); } }); @@ -384,16 +384,14 @@ class WebSocketConnection extends EventTarget { // Wait for open try { await Promise.all(promises); - } - catch (e) { + } catch (e) { this.socket.removeAllListeners('error'); this.socket.removeAllListeners('upgrade'); this.socket.removeAllListeners('open'); // Close the ws if it's open at this stage this.socket.terminate(); throw e; - } - finally { + } finally { this.socket.removeAllListeners('upgrade'); this.socket.off('open', openHandler); this.socket.off('error', openErrorHandler); @@ -451,10 +449,10 @@ class WebSocketConnection extends EventTarget { } /** - * Send data to the other side of the connection. + * Send data to the other side of the connection with a streamId. * This will not will not error out, but will rather close the connection assuming any further communication is expected to fail. - * @param streamId The stream id to send the data on - * @param data The data to send, this will include the stream message type. + * @param streamId - The stream id to send the data on + * @param data - The data to send, this will include the stream message type. * @internal */ public async streamSend(streamId: StreamId, data: Uint8Array) { @@ -470,19 +468,17 @@ class WebSocketConnection extends EventTarget { else sendProm.rejectP(err); }); await sendProm.p; - } - catch (err) { + } catch (err) { await this.stop(); } } - public async stop({ errorCode = 1000, errorMessage = '', }: { - errorCode?: number, - errorMessage?: string, + errorCode?: number; + errorMessage?: string; } = {}) { this.logger.info(`Stop ${this.constructor.name}`); // Cleaning up existing streams diff --git a/src/WebSocketConnectionMap.ts b/src/WebSocketConnectionMap.ts index 3b84fd21..feeedd90 100644 --- a/src/WebSocketConnectionMap.ts +++ b/src/WebSocketConnectionMap.ts @@ -1,8 +1,8 @@ -import WebSocketConnection from "./WebSocketConnection"; +import type WebSocketConnection from './WebSocketConnection'; import { Counter } from 'resource-counter'; class WebSocketConnectionMap extends Map { - counter: Counter + counter: Counter; public constructor() { super(); this.counter = new Counter(0); diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 90de8741..5e90dee3 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -1,11 +1,16 @@ import type { IncomingMessage, ServerResponse } from 'http'; import type tls from 'tls'; -import type { Host, Port, StreamCodeToReason, StreamReasonToCode, WebSocketConfig } from './types'; +import type { + Host, + Port, + StreamCodeToReason, + StreamReasonToCode, + WebSocketConfig, +} from './types'; import https from 'https'; import { startStop, status } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; -import Counter from 'resource-counter'; import * as errors from './errors'; import * as webSocketEvents from './events'; import { never, promise } from './utils'; @@ -29,7 +34,8 @@ class WebSocketServer extends EventTarget { protected webSocketServer: ws.WebSocketServer; protected reasonToCode: StreamReasonToCode | undefined; protected codeToReason: StreamCodeToReason | undefined; - public readonly connectionMap: WebSocketConnectionMap = new WebSocketConnectionMap(); + public readonly connectionMap: WebSocketConnectionMap = + new WebSocketConnectionMap(); protected _port: number; protected _host: string; @@ -209,24 +215,27 @@ class WebSocketServer extends EventTarget { ) => { const httpSocket = request.connection; const connectionId = this.connectionMap.allocateId(); - const connection = await WebSocketConnection.createWebSocketConnection({ - type: 'server', - connectionId: connectionId, - remoteInfo: { - host: (httpSocket.remoteAddress ?? '') as Host, - port: (httpSocket.remotePort ?? 0) as Port, + const connection = await WebSocketConnection.createWebSocketConnection( + { + type: 'server', + connectionId: connectionId, + remoteInfo: { + host: (httpSocket.remoteAddress ?? '') as Host, + port: (httpSocket.remotePort ?? 0) as Port, + }, + socket: webSocket, + config: this.config, + reasonToCode: this.reasonToCode, + codeToReason: this.codeToReason, + logger: this.logger.getChild( + `${WebSocketConnection.name} ${connectionId}`, + ), + server: this, }, - socket: webSocket, - config: this.config, - reasonToCode: this.reasonToCode, - codeToReason: this.codeToReason, - logger: this.logger.getChild( - `${WebSocketConnection.name} ${connectionId}`, - ), - server: this, - }, { - timer: this.config.connectTimeoutTime - }); + { + timer: this.config.connectTimeoutTime, + }, + ); // Handling connection events connection.addEventListener( diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 25de370e..3b7ef904 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,8 +1,15 @@ -import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; -import Logger from '@matrixai/logger'; import type { StreamCodeToReason, StreamId, StreamReasonToCode } from './types'; -import { fromVarInt, never, promise, StreamType, StreamShutdown, toVarInt } from './utils'; import type WebSocketConnection from './WebSocketConnection'; +import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; +import Logger from '@matrixai/logger'; +import { + fromVarInt, + never, + promise, + StreamType, + StreamShutdown, + toVarInt, +} from './utils'; import * as errors from './errors'; import * as events from './events'; @@ -36,7 +43,8 @@ class WebSocketStream connection, bufferSize, reasonToCode = () => 0n, - codeToReason = (type, code) => new Error(`${type.toString()} ${code.toString()}`), + codeToReason = (type, code) => + new Error(`${type.toString()} ${code.toString()}`), logger = new Logger(`${this.name} ${streamId}`), }: { streamId: StreamId; @@ -100,19 +108,25 @@ class WebSocketStream }, new ByteLengthQueuingStrategy({ highWaterMark: bufferSize, - }) + }), ); - const writableWrite = async (chunk: Uint8Array, controller: WritableStreamDefaultController) => { + const writableWrite = async ( + chunk: Uint8Array, + controller: WritableStreamDefaultController, + ) => { await this.writableDesiredSizeProm.p; - this.logger.debug(`${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`); + this.logger.debug( + `${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`, + ); let data: Uint8Array; const isChunkable = chunk.length > this.writableDesiredSize; if (isChunkable) { - this.logger.debug(`this chunk will be split into sizes of ${this.writableDesiredSize} bytes`); + this.logger.debug( + `this chunk will be split into sizes of ${this.writableDesiredSize} bytes`, + ); data = chunk.subarray(0, this.writableDesiredSize); - } - else { + } else { data = chunk; } const oldProm = this.writableDesiredSizeProm; @@ -125,16 +139,15 @@ class WebSocketStream const bytesWritten = this.writableDesiredSize; await this.streamSend(StreamType.DATA, data); // Decrement the desired size and resolved the old promise as to not block application exit - this.writableDesiredSize =- data.length; + this.writableDesiredSize = -data.length; oldProm.resolveP(); if (isChunkable) { await writableWrite(chunk.subarray(bytesWritten), controller); } - } - catch { + } catch { this.writableDesiredSizeProm = oldProm; } - } + }; this.writable = new WritableStream( { @@ -150,8 +163,8 @@ class WebSocketStream }, }, { - highWaterMark: 1 - } + highWaterMark: 1, + }, ); } @@ -183,26 +196,43 @@ class WebSocketStream * @param code - ACK * @param payloadSize - The number of bytes that the receiver can accept. */ - protected async streamSend(type: StreamType.ACK, payloadSize: number): Promise; - /** - * Send a DATA frame with a payload on the stream. - * @param code - DATA - * @param data - The payload to send. - */ - protected async streamSend(type: StreamType.DATA, data: Uint8Array): Promise; - /** - * Send an ERROR frame with a payload on the stream. - * @param code - CLOSE - * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. - */ - protected async streamSend(type: StreamType.ERROR, shutdown: StreamShutdown, code: bigint): Promise; - /** - * Send a CLOSE frame with a payload on the stream. - * @param code - CLOSE - * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. - */ - protected async streamSend(type: StreamType.CLOSE, shutdown: StreamShutdown): Promise; - protected async streamSend(type: StreamType, data_?: Uint8Array | number, code?: bigint): Promise { + protected async streamSend( + type: StreamType.ACK, + payloadSize: number, + ): Promise; + /** + * Send a DATA frame with a payload on the stream. + * @param code - DATA + * @param data - The payload to send. + */ + protected async streamSend( + type: StreamType.DATA, + data: Uint8Array, + ): Promise; + /** + * Send an ERROR frame with a payload on the stream. + * @param code - CLOSE + * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. + */ + protected async streamSend( + type: StreamType.ERROR, + shutdown: StreamShutdown, + code: bigint, + ): Promise; + /** + * Send a CLOSE frame with a payload on the stream. + * @param code - CLOSE + * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. + */ + protected async streamSend( + type: StreamType.CLOSE, + shutdown: StreamShutdown, + ): Promise; + protected async streamSend( + type: StreamType, + data_?: Uint8Array | number, + code?: bigint, + ): Promise { let data: Uint8Array | undefined; if (type === StreamType.ACK && typeof data_ === 'number') { data = new Uint8Array(4); @@ -210,19 +240,16 @@ class WebSocketStream dv.setUint32(0, data_, false); } else if (type === StreamType.DATA) { data = data_ as Uint8Array; - } - else if (type === StreamType.ERROR) { + } else if (type === StreamType.ERROR) { const errorCode = fromVarInt(code!); data = new Uint8Array(1 + errorCode.length); const dv = new DataView(data.buffer); dv.setUint8(0, data_ as StreamShutdown); data.set(errorCode, 1); - } - else if (type === StreamType.CLOSE) { + } else if (type === StreamType.CLOSE) { data = new Uint8Array([data_ as StreamShutdown]); - } - else { - never(); + } else { + never(); } const arrayLength = 1 + (data?.length ?? 0); const array = new Uint8Array(arrayLength); @@ -233,6 +260,12 @@ class WebSocketStream await this.connection.streamSend(this.streamId, array); } + /** + * Put a message frame into a stream. + * This will not will not error out, but will rather close the ReadableStream assuming any further reads are expected to fail. + * @param message - The message to put into the stream. + * @internal + */ public async streamRecv(message: Uint8Array) { const type = message[0] as StreamType; const data = message.subarray(1); @@ -241,16 +274,22 @@ class WebSocketStream const bufferSize = dv.getUint32(0, false); this.writableDesiredSize = bufferSize; this.writableDesiredSizeProm.resolveP(); - this.logger.debug(`received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`); - } - else if (type === StreamType.DATA) { - if (this.readableController.desiredSize != null && data.length > this.readableController.desiredSize) { - await this.signalReadableEnd(true, new errors.ErrorWebSocketStreamReaderBufferOverload()); + this.logger.debug( + `received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`, + ); + } else if (type === StreamType.DATA) { + if ( + this.readableController.desiredSize != null && + data.length > this.readableController.desiredSize + ) { + await this.signalReadableEnd( + true, + new errors.ErrorWebSocketStreamReaderBufferOverload(), + ); return; } this.readableController.enqueue(data); - } - else if (type === StreamType.ERROR || type === StreamType.CLOSE) { + } else if (type === StreamType.ERROR || type === StreamType.CLOSE) { const shutdown = dv.getUint8(0) as StreamShutdown; let isError = false; let reason: any; @@ -261,15 +300,12 @@ class WebSocketStream } if (shutdown === StreamShutdown.Read) { await this.signalReadableEnd(isError, reason); - } - else if (shutdown === StreamShutdown.Write) { + } else if (shutdown === StreamShutdown.Write) { await this.signalWritableEnd(isError, reason); - } - else { + } else { never(); } - } - else { + } else { never(); } } @@ -278,7 +314,6 @@ class WebSocketStream * Forces the active stream to end early */ public cancel(reason?: any): void { - console.log(reason) const isError = reason != null; // Close the streams with the given error, if (!this._readableEnded) { @@ -296,18 +331,20 @@ class WebSocketStream * to track the streams state. */ protected async signalReadableEnd(isError: boolean = false, reason?: any) { - if (isError) this.logger.debug(`ending readable with error ${reason.message}`); - else this.logger.debug(`ending readable`); + if (isError) { + this.logger.debug(`ending readable with error ${reason.message}`); + } else { + this.logger.debug(`ending readable`); + } if (this._readableEnded) return; - // indicate that receiving side is closed + // Indicate that receiving side is closed this._readableEnded = true; - // shutdown the write side of the other stream + // Shutdown the write side of the other stream if (isError) { const code = await this.reasonToCode('send', reason); - this.streamSend(StreamType.ERROR, StreamShutdown.Write, code); - } - else { - this.streamSend(StreamType.CLOSE, StreamShutdown.Write); + await this.streamSend(StreamType.ERROR, StreamShutdown.Write, code); + } else { + await this.streamSend(StreamType.CLOSE, StreamShutdown.Write); } this.readableController.error(reason); if (this._readableEnded && this._writableEnded) { @@ -321,23 +358,21 @@ class WebSocketStream * Signals the end of the WritableStream. to be used with the extended class * to track the streams state. */ - protected async signalWritableEnd( - isError: boolean = false, - reason?: any, - ) { - if (isError) this.logger.debug(`ending writable with error ${reason.message}`); - else this.logger.debug(`ending writable`); + protected async signalWritableEnd(isError: boolean = false, reason?: any) { + if (isError) { + this.logger.debug(`ending writable with error ${reason.message}`); + } else { + this.logger.debug(`ending writable`); + } if (this._writableEnded) return; - // indicate that sending side is closed + // Indicate that sending side is closed this._writableEnded = true; - // shutdown the read side of the other stream + // Shutdown the read side of the other stream if (isError) { const code = await this.reasonToCode('send', reason); - this.streamSend(StreamType.ERROR, StreamShutdown.Read, code); - - } - else { - this.streamSend(StreamType.CLOSE, StreamShutdown.Read); + await this.streamSend(StreamType.ERROR, StreamShutdown.Read, code); + } else { + await this.streamSend(StreamType.CLOSE, StreamShutdown.Read); } this.writableController.error(reason); if (this._readableEnded && this._writableEnded) { diff --git a/src/config.ts b/src/config.ts index 0927136d..7761b25e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,14 +4,14 @@ const serverDefault: WebSocketConfig = { connectTimeoutTime: 120, keepAliveIntervalTime: 1_000, keepAliveTimeoutTime: 10_000, - streamBufferSize: 1024 * 1024 // 1MB + streamBufferSize: 1024 * 1024, // 1MB }; const clientDefault: WebSocketConfig = { connectTimeoutTime: Infinity, keepAliveIntervalTime: 1_000, keepAliveTimeoutTime: 10_000, - streamBufferSize: 1024 * 1024 // 1MB -} + streamBufferSize: 1024 * 1024, // 1MB +}; export { serverDefault, clientDefault }; diff --git a/src/errors.ts b/src/errors.ts index f91b6a47..b1a2ff4c 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -29,7 +29,8 @@ class ErrorWebSocketClientDestroyed extends ErrorWebSocketClient { } class ErrorWebSocketClientInvalidHost extends ErrorWebSocketClient { - static description = 'WebSocket Client cannot be created with the specified host'; + static description = + 'WebSocket Client cannot be created with the specified host'; } // Connection diff --git a/src/events.ts b/src/events.ts index 618dac61..8ba3268c 100644 --- a/src/events.ts +++ b/src/events.ts @@ -90,6 +90,7 @@ export { WebSocketServerConnectionEvent, WebSocketServerStartEvent, WebSocketServerStopEvent, + WebSocketServerErrorEvent, WebSocketConnectionEvent, WebSocketConnectionStreamEvent, WebSocketConnectionStopEvent, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index e26af2d2..93f3e836 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -22,7 +22,6 @@ function promise(): PromiseDeconstructed { }; } - function toVarInt(array: Uint8Array): Parsed { let streamId: bigint; @@ -108,8 +107,16 @@ enum StreamType { enum StreamShutdown { Read = 0, - Write = 1 + Write = 1, } - -export { never, promise, toVarInt, fromVarInt, toStreamId, fromStreamId, StreamType, StreamShutdown }; +export { + never, + promise, + toVarInt, + fromVarInt, + toStreamId, + fromStreamId, + StreamType, + StreamShutdown, +}; diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index 5626bb75..b99d18e2 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,12 +1,10 @@ import type { WebSocketServerConnectionEvent } from '@/events'; -import WebSocketServer from '@/WebSocketServer'; import { WebSocket } from 'ws'; +import WebSocketServer from '@/WebSocketServer'; import * as testsUtils from './utils'; describe('test', () => { - beforeEach(() => { - - }); + beforeEach(() => {}); test('test', async () => { const tlsConfigServer = await testsUtils.generateConfig('RSA'); const server = new WebSocketServer({ @@ -14,7 +12,7 @@ describe('test', () => { ...tlsConfigServer, }, }); - server.start({ + await server.start({ port: 3000, }); server.addEventListener( @@ -24,8 +22,8 @@ describe('test', () => { const stream = await connection.streamNew('bidi'); const writer = stream.writable.getWriter(); await writer.ready; - writer.write(new Uint8Array([1, 2, 3])); - writer.write(new Uint8Array([1, 2, 3])); + await writer.write(new Uint8Array([1, 2, 3])); + await writer.write(new Uint8Array([1, 2, 3])); }, ); }); diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index e89a4ff7..2c336522 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -1,16 +1,14 @@ -import { StreamId } from "@/types"; -import WebSocketStream from "@/WebSocketStream"; -import WebSocketConnection from "@/WebSocketConnection"; -import * as events from "@/events"; -import { promise } from "@/utils"; -import { fc, testProp } from "@fast-check/jest"; +import type { StreamId } from '@/types'; +import WebSocketStream from '@/WebSocketStream'; +import WebSocketConnection from '@/WebSocketConnection'; +import * as events from '@/events'; +import { promise } from '@/utils'; import * as testUtils from './utils'; const DEFAULT_BUFFER_SIZE = 1024; jest.mock('@/WebSocketConnection', () => { - return jest.fn().mockImplementation(( - ) => { + return jest.fn().mockImplementation(() => { const instance = new EventTarget() as EventTarget & { connectedConnection: WebSocketConnection | undefined; connectTo: (connection: WebSocketConnection) => void; @@ -31,11 +29,13 @@ jest.mock('@/WebSocketConnection', () => { bufferSize: DEFAULT_BUFFER_SIZE, connection: instance.connectedConnection!, }); - instance.connectedConnection!.dispatchEvent(new events.WebSocketConnectionStreamEvent({ - detail: stream, - })); + instance.connectedConnection!.dispatchEvent( + new events.WebSocketConnectionStreamEvent({ + detail: stream, + }), + ); } - stream.streamRecv(data); + await stream.streamRecv(data); }; return instance; }); @@ -57,16 +57,20 @@ describe(WebSocketStream.name, () => { const stream1 = await WebSocketStream.createWebSocketStream({ streamId: 0n as StreamId, bufferSize: DEFAULT_BUFFER_SIZE, - connection: connection1 + connection: connection1, }); const createStream2Prom = promise(); - connection2.addEventListener("connectionStream", (e: events.WebSocketConnectionStreamEvent) => { - createStream2Prom.resolveP(e.detail); - }, { once: true }); + connection2.addEventListener( + 'connectionStream', + (e: events.WebSocketConnectionStreamEvent) => { + createStream2Prom.resolveP(e.detail); + }, + { once: true }, + ); const stream2 = await createStream2Prom.p; return [stream1, stream2]; } - // testProp( + // TestProp( // 'normal', // [fc.infiniteStream(fc.uint8Array({minLength: 1, maxLength: DEFAULT_BUFFER_SIZE}))], // async (iterable) => { @@ -92,21 +96,18 @@ describe(WebSocketStream.name, () => { // // await stream1.destroy(); // } // ); - test( - 'normal', - async () => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); + test('normal', async () => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); - const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE); - testUtils.randomBytes(buffer); + const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE); + await testUtils.randomBytes(buffer); - const stream1Readable = stream1.readable; - const stream2Writable = stream2.writable; - const writeProm = stream2Writable.getWriter().write(buffer); - await stream1Readable.getReader().read(); - await writeProm; + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + const writeProm = stream2Writable.getWriter().write(buffer); + await stream1Readable.getReader().read(); + await writeProm; - await stream1.destroy(); - } - ); + await stream1.destroy(); + }); }); diff --git a/tests/utils.ts b/tests/utils.ts index e9a71099..6b79503a 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -595,11 +595,10 @@ function toReadableStream(iterator: IterableIterator) { controller.enqueue(chunk); } controller.close(); - } + }, }); } - export { sleep, randomBytes, From 24cdfa9b5f00e2be25e241754a647417c92a1b3d Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 17:11:56 +1000 Subject: [PATCH 023/149] chore: deleted old WebSocketStream --- src/WebSocketStream.bak.ts | 361 ------------------------------------- 1 file changed, 361 deletions(-) delete mode 100644 src/WebSocketStream.bak.ts diff --git a/src/WebSocketStream.bak.ts b/src/WebSocketStream.bak.ts deleted file mode 100644 index ace34c60..00000000 --- a/src/WebSocketStream.bak.ts +++ /dev/null @@ -1,361 +0,0 @@ -import type { ReadableWritablePair } from 'stream/web'; -import type { - ReadableStreamController, - WritableStreamDefaultController, -} from 'stream/web'; -import type Logger from '@matrixai/logger'; -import type WebSocketConnection from './WebSocketConnection'; -import { WritableStream, ReadableStream } from 'stream/web'; -import * as ws from 'ws'; -import * as webSocketErrors from './errors'; -import * as utilsErrors from './utils/errors'; -import { promise } from './utils'; - -class WebSocketStream implements ReadableWritablePair { - public readable: ReadableStream; - public writable: WritableStream; - - protected _readableEnded = false; - protected _readableEndedProm = promise(); - protected _writableEnded = false; - protected _writableEndedProm = promise(); - protected _webSocketEnded = false; - protected _webSocketEndedProm = promise(); - protected _endedProm: Promise; - - protected readableController: - | ReadableStreamController - | undefined; - protected writableController: WritableStreamDefaultController | undefined; - - constructor( - protected connection: WebSocketConnection, - pingInterval: number, - pingTimeoutTime: number, - protected metadata: { - localHost: string; - localPort: number; - remoteHost: string; - remotePort: number; - }, - logger: Logger, - ) { - // Sanitise promises so they don't result in unhandled rejections - this._readableEndedProm.p.catch(() => {}); - this._writableEndedProm.p.catch(() => {}); - this._webSocketEndedProm.p.catch(() => {}); - // Creating the endedPromise - this._endedProm = Promise.allSettled([ - this._readableEndedProm.p, - this._writableEndedProm.p, - this._webSocketEndedProm.p, - ]).then((result) => { - if ( - result[0].status === 'rejected' || - result[1].status === 'rejected' || - result[2].status === 'rejected' - ) { - // Throw a compound error - throw AggregateError(result, 'stream failed'); - } - // Otherwise return nothing - }); - // Ignore errors if it's never used - this._endedProm.catch(() => {}); - - logger.info('WS opened'); - const readableLogger = logger.getChild('readable'); - const writableLogger = logger.getChild('writable'); - // Setting up the readable stream - this.readable = new ReadableStream( - { - start: (controller) => { - readableLogger.debug('Starting'); - this.readableController = controller; - const messageHandler = (data: ws.RawData, isBinary: boolean) => { - if (!isBinary || data instanceof Array) { - controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); - return; - } - const message = data as Buffer; - readableLogger.debug(`Received ${message.toString()}`); - if (message.length === 0) { - readableLogger.debug('Null message received'); - connection.socket.removeListener('message', messageHandler); - if (!this._readableEnded) { - readableLogger.debug('Closing'); - this.signalReadableEnd(); - controller.close(); - } - if (this._writableEnded) { - logger.debug('Closing socket'); - connection.socket.close(); - } - return; - } - if (this._readableEnded) { - return; - } - controller.enqueue(message); - if (controller.desiredSize == null) { - controller.error(new utilsErrors.ErrorUtilsUndefinedBehaviour()); - return; - } - if (controller.desiredSize < 0) { - readableLogger.debug('Applying readable backpressure'); - connection.socket.pause(); - } - }; - readableLogger.debug('Registering socket message handler'); - connection.socket.on('message', messageHandler); - connection.socket.once('close', (code, reason) => { - logger.info('Socket closed'); - connection.socket.removeListener('message', messageHandler); - if (!this._readableEnded) { - readableLogger.debug( - `Closed early, ${code}, ${reason.toString()}`, - ); - const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); - this.signalReadableEnd(e); - controller.error(e); - } - }); - ws.once('error', (e) => { - if (!this._readableEnded) { - readableLogger.error(e); - this.signalReadableEnd(e); - controller.error(e); - } - }); - }, - cancel: (reason) => { - readableLogger.debug('Cancelled'); - this.signalReadableEnd(reason); - if (this._writableEnded) { - readableLogger.debug('Closing socket'); - this.signalWritableEnd(reason); - ws.close(); - } - }, - pull: () => { - readableLogger.debug('Releasing backpressure'); - ws.resume(); - }, - }, - { highWaterMark: 1 }, - ); - this.writable = new WritableStream( - { - start: (controller) => { - this.writableController = controller; - writableLogger.info('Starting'); - ws.once('error', (e) => { - if (!this._writableEnded) { - writableLogger.error(e); - this.signalWritableEnd(e); - controller.error(e); - } - }); - ws.once('close', (code, reason) => { - if (!this._writableEnded) { - writableLogger.debug( - `Closed early, ${code}, ${reason.toString()}`, - ); - const e = new webSocketErrors.ErrorClientConnectionEndedEarly(); - this.signalWritableEnd(e); - controller.error(e); - } - }); - }, - close: async () => { - writableLogger.debug('Closing, sending null message'); - const sendProm = promise(); - ws.send(Buffer.from([]), (err) => { - if (err == null) sendProm.resolveP(); - else sendProm.rejectP(err); - }); - await sendProm.p; - this.signalWritableEnd(); - if (this._readableEnded) { - writableLogger.debug('Closing socket'); - ws.close(); - } - }, - abort: (reason) => { - writableLogger.debug('Aborted'); - this.signalWritableEnd(reason); - if (this._readableEnded) { - writableLogger.debug('Closing socket'); - ws.close(4000, `Aborting connection with ${reason.message}`); - } - }, - write: async (chunk, controller) => { - if (this._writableEnded) return; - writableLogger.debug(`Sending ${chunk?.toString()}`); - const wait = promise(); - ws.send(chunk, (e) => { - if (e != null && !this._writableEnded) { - // Opting to debug message here and not log an error, sending - // failure is common if we send before the close event. - writableLogger.debug('failed to send'); - const err = new webSocketErrors.ErrorClientConnectionEndedEarly( - undefined, - { - cause: e, - }, - ); - this.signalWritableEnd(err); - controller.error(err); - } - wait.resolveP(); - }); - await wait.p; - }, - }, - { highWaterMark: 1 }, - ); - - // Setting up heartbeat - const pingTimer = setInterval(() => { - ws.ping(); - }, pingInterval); - const pingTimeoutTimeTimer = setTimeout(() => { - logger.debug('Ping timed out'); - ws.close(4002, 'Timed out'); - }, pingTimeoutTime); - const pingHandler = () => { - logger.debug('Received ping'); - ws.pong(); - }; - const pongHandler = () => { - logger.debug('Received pong'); - pingTimeoutTimeTimer.refresh(); - }; - ws.on('ping', pingHandler); - ws.on('pong', pongHandler); - ws.once('close', (code, reason) => { - ws.off('ping', pingHandler); - ws.off('pong', pongHandler); - logger.debug('WebSocket closed'); - const err = - code !== 1000 - ? new webSocketErrors.ErrorClientConnectionEndedEarly( - `ended with code ${code}, ${reason.toString()}`, - ) - : undefined; - this.signalWebSocketEnd(err); - logger.debug('Cleaning up timers'); - // Clean up timers - clearTimeout(pingTimer); - clearTimeout(pingTimeoutTimeTimer); - }); - } - - get readableEnded() { - return this._readableEnded; - } - - /** - * Resolves when the readable has ended and rejects with any errors. - */ - get readableEndedProm() { - return this._readableEndedProm.p; - } - - get writableEnded() { - return this._writableEnded; - } - - /** - * Resolves when the writable has ended and rejects with any errors. - */ - get writableEndedProm() { - return this._writableEndedProm.p; - } - - get webSocketEnded() { - return this._webSocketEnded; - } - - /** - * Resolves when the webSocket has ended and rejects with any errors. - */ - get webSocketEndedProm() { - return this._webSocketEndedProm.p; - } - - get ended() { - return this._readableEnded && this._writableEnded; - } - - /** - * Resolves when the stream has fully closed - */ - get endedProm(): Promise { - return this._endedProm; - } - - get meta() { - // Spreading to avoid modifying the data - return { - ...this.metadata, - }; - } - - /** - * Forces the active stream to end early - */ - public cancel(reason?: any): void { - // Default error - const err = reason ?? new webSocketErrors.ErrorClientConnectionEndedEarly(); - // Close the streams with the given error, - if (!this._readableEnded) { - this.readableController?.error(err); - this.signalReadableEnd(err); - } - if (!this._writableEnded) { - this.writableController?.error(err); - this.signalWritableEnd(err); - } - // Then close the websocket - if (!this._webSocketEnded) { - this.ws.close(4000, 'Ending connection'); - this.signalWebSocketEnd(err); - } - } - - /** - * Signals the end of the ReadableStream. to be used with the extended class - * to track the streams state. - */ - protected signalReadableEnd(reason?: any) { - if (this._readableEnded) return; - this._readableEnded = true; - if (reason == null) this._readableEndedProm.resolveP(); - else this._readableEndedProm.rejectP(reason); - } - - /** - * Signals the end of the WritableStream. to be used with the extended class - * to track the streams state. - */ - protected signalWritableEnd(reason?: any) { - if (this._writableEnded) return; - this._writableEnded = true; - if (reason == null) this._writableEndedProm.resolveP(); - else this._writableEndedProm.rejectP(reason); - } - - /** - * Signals the end of the WebSocket. to be used with the extended class - * to track the streams state. - */ - protected signalWebSocketEnd(reason?: any) { - if (this._webSocketEnded) return; - this._webSocketEnded = true; - if (reason == null) this._webSocketEndedProm.resolveP(); - else this._webSocketEndedProm.rejectP(reason); - } -} - -export default WebSocketStream; From a09fde1100c681f04577087b1e73501f28d73b62 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 17:12:20 +1000 Subject: [PATCH 024/149] fix: https/ws server errors are propagated onto WebSocketServer --- src/WebSocketServer.ts | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 5e90dee3..cce16d55 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -20,10 +20,17 @@ import * as utils from './utils'; import WebSocketConnectionMap from './WebSocketConnectionMap'; /** + * You must provide an error handler `addEventListener('error')`. + * Otherwise, errors will just be ignored. + * * Events: - * - start - * - stop - * - connection + * - serverStop + * - serverError + * - serverConnection + * - connectionStream - when new stream is created from a connection + * - connectionError - connection error event + * - connectionDestroy - when connection is destroyed + * - streamDestroy - when stream is destroyed */ interface WebSocketServer extends startStop.StartStop {} @startStop.StartStop() @@ -293,7 +300,16 @@ class WebSocketServer extends EventTarget { * Used to propagate error conditions */ protected errorHandler = (e: Error) => { - this.logger.error(e); + this.dispatchEvent( + new webSocketEvents.WebSocketServerErrorEvent({ + detail: new errors.ErrorWebSocketServer( + 'An error occured on the underlying server', + { + cause: e, + }, + ), + }), + ); }; /** From 52d2a061ab98bb7fb492a576095fbe01279bc38b Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 17:20:22 +1000 Subject: [PATCH 025/149] feat: errors on WebSocket are now bubbled up --- src/WebSocketConnection.ts | 15 +++++++++++++++ src/errors.ts | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 78b6005f..32e12420 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -185,6 +185,19 @@ class WebSocketConnection extends EventTarget { this.setKeepAliveTimeoutTimer(); }; + protected errorHandler = (err: Error) => { + this.dispatchEvent( + new events.WebSocketConnectionErrorEvent({ + detail: new errors.ErrorWebSocketConnectionSocket( + 'An error occurred on the underlying WebSocket instance.', + { + cause: err, + }, + ), + }), + ); + }; + public static createWebSocketConnection( args: | { @@ -410,6 +423,7 @@ class WebSocketConnection extends EventTarget { this.socket.on('message', this.messageHandler); this.socket.on('ping', this.pingHandler); this.socket.on('pong', this.pongHandler); + this.socket.on('error', this.errorHandler); this.logger.info(`Started ${this.constructor.name}`); } @@ -503,6 +517,7 @@ class WebSocketConnection extends EventTarget { this.socket.off('message', this.messageHandler); this.socket.off('ping', this.pingHandler); this.socket.off('pong', this.pongHandler); + this.socket.off('error', this.errorHandler); this.keepAliveTimeOutTimer?.cancel(timerCleanupReasonSymbol); if (this.type === 'server') { diff --git a/src/errors.ts b/src/errors.ts index b1a2ff4c..9d89bca7 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -57,6 +57,10 @@ class ErrorWebSocketConnectionKeepAliveTimeOut< static description = 'WebSocket Connection reached idle timeout'; } +class ErrorWebSocketConnectionSocket extends ErrorWebSocketConnection { + static description = 'WebSocket Connection underlying websocket error'; +} + // Stream class ErrorWebSocketStream extends ErrorWebSocket { @@ -89,6 +93,7 @@ export { ErrorWebSocketConnectionNotRunning, ErrorWebSocketConnectionStartTimeOut, ErrorWebSocketConnectionKeepAliveTimeOut, + ErrorWebSocketConnectionSocket, ErrorWebSocketStream, ErrorWebSocketStreamReaderBufferOverload, ErrorWebSocketStreamDestroyed, From 8a58602a3b5169f814591751a85a7653610a4d26 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 18:06:31 +1000 Subject: [PATCH 026/149] fix: errors on parsing WebSocketStream messages are now bubbled up --- src/WebSocketStream.ts | 122 +++++++++++++++++++++++++++-------------- src/errors.ts | 10 ++++ 2 files changed, 90 insertions(+), 42 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 3b7ef904..53924f48 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -94,12 +94,18 @@ class WebSocketStream { start: async (controller) => { this.readableController = controller; - this.logger.debug('started'); }, pull: async (controller) => { - if (controller.desiredSize != null && controller.desiredSize > 0) { - await this.streamSend(StreamType.ACK, controller.desiredSize!); + // If a readable has ended, whether by the closing of the sender's WritableStream or by calling `.close`, do not bother to send back an ACK + if (this._readableEnded) { + return; } + // If desiredSize is less than or equal to 0, it means that the buffer is still full after a read + if (controller.desiredSize != null && controller.desiredSize <= 0) { + return; + } + // Send ACK on every read as there will be more usable space on the buffer. + await this.streamSend(StreamType.ACK, controller.desiredSize!); }, cancel: async (reason) => { this.logger.debug(`readable aborted with [${reason.message}]`); @@ -111,10 +117,14 @@ class WebSocketStream }), ); - const writableWrite = async ( + const writeHandler = async ( chunk: Uint8Array, controller: WritableStreamDefaultController, ) => { + // Do not bother to write or wait for ACK if the writable has ended + if (this._writableEnded) { + return; + } await this.writableDesiredSizeProm.p; this.logger.debug( `${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`, @@ -129,23 +139,17 @@ class WebSocketStream } else { data = chunk; } - const oldProm = this.writableDesiredSizeProm; - try { - if (this.writableDesiredSize === data.length) { - this.logger.debug(`this chunk will trigger receiver to send an ACK`); - // Reset the promise to wait for another ACK - this.writableDesiredSizeProm = promise(); - } - const bytesWritten = this.writableDesiredSize; - await this.streamSend(StreamType.DATA, data); - // Decrement the desired size and resolved the old promise as to not block application exit - this.writableDesiredSize = -data.length; - oldProm.resolveP(); - if (isChunkable) { - await writableWrite(chunk.subarray(bytesWritten), controller); - } - } catch { - this.writableDesiredSizeProm = oldProm; + if (this.writableDesiredSize === data.length) { + this.logger.debug(`this chunk will trigger receiver to send an ACK`); + // Reset the promise to wait for another ACK + this.writableDesiredSizeProm = promise(); + } + const bytesWritten = data.length; + await this.streamSend(StreamType.DATA, data); + // Decrement the desired size by the amount of bytes written + this.writableDesiredSize = -bytesWritten; + if (isChunkable) { + await writeHandler(chunk.subarray(bytesWritten), controller); } }; @@ -154,7 +158,7 @@ class WebSocketStream start: (controller) => { this.writableController = controller; }, - write: writableWrite, + write: writeHandler, close: async () => { await this.signalWritableEnd(); }, @@ -267,16 +271,38 @@ class WebSocketStream * @internal */ public async streamRecv(message: Uint8Array) { + if (message.length === 0) { + this.logger.debug(`received empty message, closing stream`); + await this.signalReadableEnd( + true, + new errors.ErrorWebSocketStreamReaderParse('empty message', { + cause: new RangeError(), + }), + ); + } const type = message[0] as StreamType; const data = message.subarray(1); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); if (type === StreamType.ACK) { - const bufferSize = dv.getUint32(0, false); - this.writableDesiredSize = bufferSize; - this.writableDesiredSizeProm.resolveP(); - this.logger.debug( - `received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`, - ); + try { + const bufferSize = dv.getUint32(0, false); + this.writableDesiredSize = bufferSize; + this.writableDesiredSizeProm.resolveP(); + this.logger.debug( + `received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`, + ); + } catch (e) { + this.logger.debug(`received malformed ACK, closing stream`); + await this.signalReadableEnd( + true, + new errors.ErrorWebSocketStreamReaderParse( + 'ACK message did not contain a valid buffer size', + { + cause: e, + }, + ), + ); + } } else if (type === StreamType.DATA) { if ( this.readableController.desiredSize != null && @@ -290,20 +316,32 @@ class WebSocketStream } this.readableController.enqueue(data); } else if (type === StreamType.ERROR || type === StreamType.CLOSE) { - const shutdown = dv.getUint8(0) as StreamShutdown; - let isError = false; - let reason: any; - if (type === StreamType.ERROR) { - isError = true; - const errorCode = toVarInt(data.subarray(1)).data; - reason = await this.codeToReason('recv', errorCode); - } - if (shutdown === StreamShutdown.Read) { - await this.signalReadableEnd(isError, reason); - } else if (shutdown === StreamShutdown.Write) { - await this.signalWritableEnd(isError, reason); - } else { - never(); + try { + const shutdown = dv.getUint8(0) as StreamShutdown; + let isError = false; + let reason: any; + if (type === StreamType.ERROR) { + isError = true; + const errorCode = toVarInt(data.subarray(1)).data; + reason = await this.codeToReason('recv', errorCode); + } + if (shutdown === StreamShutdown.Read) { + await this.signalReadableEnd(isError, reason); + } else if (shutdown === StreamShutdown.Write) { + await this.signalWritableEnd(isError, reason); + } else { + never('invalid shutdown type'); + } + } catch (e) { + await this.signalReadableEnd( + true, + new errors.ErrorWebSocketStreamReaderParse( + 'ERROR/CLOSE message did not contain a valid payload', + { + cause: e, + }, + ), + ); } } else { never(); diff --git a/src/errors.ts b/src/errors.ts index 9d89bca7..913b7326 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -67,6 +67,14 @@ class ErrorWebSocketStream extends ErrorWebSocket { static description = 'WebSocket Stream error'; } +class ErrorWebSocketStreamReader extends ErrorWebSocketStream { + static description = 'WebSocket Stream readable error'; +} + +class ErrorWebSocketStreamReaderParse extends ErrorWebSocketStreamReader { + static description = 'WebSocket Stream readable message parse failed'; +} + class ErrorWebSocketStreamReaderBufferOverload extends ErrorWebSocket { static description = 'WebSocket Stream readable buffer has overloaded'; } @@ -95,6 +103,8 @@ export { ErrorWebSocketConnectionKeepAliveTimeOut, ErrorWebSocketConnectionSocket, ErrorWebSocketStream, + ErrorWebSocketStreamReader, + ErrorWebSocketStreamReaderParse, ErrorWebSocketStreamReaderBufferOverload, ErrorWebSocketStreamDestroyed, ErrorWebSocketUndefinedBehaviour, From 633f6e5725053a84bd10122714af19442f1b882c Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 30 Aug 2023 18:40:18 +1000 Subject: [PATCH 027/149] fix: buffering writableDesiredSize was going into negative --- src/WebSocketStream.ts | 9 +++++---- tests/WebSocketStream.test.ts | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 53924f48..8072afb0 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -139,15 +139,16 @@ class WebSocketStream } else { data = chunk; } - if (this.writableDesiredSize === data.length) { + const bytesWritten = data.length; + if (this.writableDesiredSize === bytesWritten) { this.logger.debug(`this chunk will trigger receiver to send an ACK`); // Reset the promise to wait for another ACK this.writableDesiredSizeProm = promise(); } - const bytesWritten = data.length; await this.streamSend(StreamType.DATA, data); // Decrement the desired size by the amount of bytes written - this.writableDesiredSize = -bytesWritten; + this.writableDesiredSize -= bytesWritten; + if (isChunkable) { await writeHandler(chunk.subarray(bytesWritten), controller); } @@ -289,7 +290,7 @@ class WebSocketStream this.writableDesiredSize = bufferSize; this.writableDesiredSizeProm.resolveP(); this.logger.debug( - `received ACK, writerDesiredSize is now reset to ${bufferSize} bytes`, + `received ACK, writableDesiredSize is now reset to ${bufferSize} bytes`, ); } catch (e) { this.logger.debug(`received malformed ACK, closing stream`); diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 2c336522..4d74c679 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -22,6 +22,7 @@ jest.mock('@/WebSocketConnection', () => { }; instance.streamMap = new Map(); instance.streamSend = async (streamId: StreamId, data: Uint8Array) => { + console.log('streamSend', streamId, data); let stream = instance.connectedConnection!.streamMap.get(streamId); if (stream == null) { stream = await WebSocketStream.createWebSocketStream({ @@ -99,7 +100,7 @@ describe(WebSocketStream.name, () => { test('normal', async () => { const [stream1, stream2] = await createStreamPair(connection1, connection2); - const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE); + const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE + 1); await testUtils.randomBytes(buffer); const stream1Readable = stream1.readable; From 9052a47387ec1a211b6486e9a5e7d25cd2a9bbc3 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:43:12 +1000 Subject: [PATCH 028/149] fix: WebSocketConnectionMap is now using correct `resource-counter` default export --- src/WebSocketConnectionMap.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebSocketConnectionMap.ts b/src/WebSocketConnectionMap.ts index feeedd90..19e9768d 100644 --- a/src/WebSocketConnectionMap.ts +++ b/src/WebSocketConnectionMap.ts @@ -1,5 +1,5 @@ import type WebSocketConnection from './WebSocketConnection'; -import { Counter } from 'resource-counter'; +import Counter from 'resource-counter'; class WebSocketConnectionMap extends Map { counter: Counter; From d7610aab5ddb55c027496295df800d458db767ec Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:11:34 +1000 Subject: [PATCH 029/149] feat: added specifying force stop/destroy to connections and streams --- src/WebSocketConnection.ts | 18 ++++++++++++++---- src/WebSocketServer.ts | 1 + src/WebSocketStream.ts | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 32e12420..49aaa2b1 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -416,7 +416,7 @@ class WebSocketConnection extends EventTarget { this.socket.once('close', () => { this.resolveClosedP(); if (this[startStop.running] && this[startStop.status] !== 'stopping') { - void this.stop(); + void this.stop({ force: true }); } }); @@ -483,23 +483,32 @@ class WebSocketConnection extends EventTarget { }); await sendProm.p; } catch (err) { - await this.stop(); + await this.stop({ + force: true, + errorCode: 1006, + errorMessage: 'connection was unable to send data', + }); } } public async stop({ errorCode = 1000, errorMessage = '', + force = false, }: { errorCode?: number; errorMessage?: string; + force?: boolean; } = {}) { this.logger.info(`Stop ${this.constructor.name}`); // Cleaning up existing streams const streamsDestroyP: Array> = []; this.logger.debug('triggering stream destruction'); for (const stream of this.streamMap.values()) { - streamsDestroyP.push(stream.destroy()); + if (force) { + await stream.destroy(); + } + streamsDestroyP.push(stream.destroyedP); } this.logger.debug('waiting for streams to destroy'); await Promise.all(streamsDestroyP); @@ -538,7 +547,8 @@ class WebSocketConnection extends EventTarget { }), ); if (this[startStop.running] && this[startStop.status] !== 'stopping') { - void this.stop(); + // Background stopping, we don't want to block the timer resolving + void this.stop({ force: true }); } }; // If there was an existing timer, we cancel it and set a new one diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index cce16d55..81ff80bd 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -144,6 +144,7 @@ class WebSocketServer extends EventTarget { destroyProms.push( webSocketConnection.stop({ errorMessage: 'cleaning up connections', + force }), ); } diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 8072afb0..4631cd75 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -406,6 +406,8 @@ class WebSocketStream if (this._writableEnded) return; // Indicate that sending side is closed this._writableEnded = true; + // resolve backpressure blocking promise in case unresolved + this.writableDesiredSizeProm.resolveP(); // Shutdown the read side of the other stream if (isError) { const code = await this.reasonToCode('send', reason); From ac391db6a2b333a9379b9dc6dc8a7a9050dc509b Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:20:30 +1000 Subject: [PATCH 030/149] feat: aligned WebSocketClient to use WebSocketConnection --- src/WebSocketClient.ts | 289 ++++++++++++++-------------------- src/events.ts | 25 +++ tests/WebSocketClient.test.ts | 37 +++++ 3 files changed, 176 insertions(+), 175 deletions(-) create mode 100644 tests/WebSocketClient.test.ts diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 6e1aa6f9..4b129826 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,22 +1,79 @@ -import type { DetailedPeerCertificate, TLSSocket } from 'tls'; -import type { ContextTimed } from '@matrixai/contexts'; import type { Host, Port, VerifyCallback, WebSocketConfig } from './types'; import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; import { Validator } from 'ip-num'; -import { Timer } from '@matrixai/timer'; -import Counter from 'resource-counter'; -import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; -import { promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; import WebSocketConnectionMap from './WebSocketConnectionMap'; import { clientDefault } from './config'; +import * as events from './events'; +import * as utils from './utils'; interface WebSocketClient extends createDestroy.CreateDestroy {} @createDestroy.CreateDestroy() class WebSocketClient extends EventTarget { + protected logger: Logger; + + protected _connection: WebSocketConnection; + public readonly connectionMap: WebSocketConnectionMap = + new WebSocketConnectionMap(); + + protected address: string; + + protected handleWebSocketConnectionEvents = async ( + event: events.WebSocketConnectionEvent, + ) => { + if (event instanceof events.WebSocketConnectionErrorEvent) { + this.dispatchEvent( + new events.WebSocketConnectionErrorEvent({ + detail: new errors.ErrorWebSocketClient('Connection error', { + cause: event.detail, + }), + }), + ); + try { + // Force destroy means don't destroy gracefully + await this.destroy({ + force: true, + }); + } catch (e) { + this.dispatchEvent( + new events.WebSocketClientErrorEvent({ + detail: e.detail, + }), + ); + } + } else if (event instanceof events.WebSocketConnectionStopEvent) { + try { + // Force destroy means don't destroy gracefully + await this.destroy({ + force: true, + }); + } catch (e) { + this.dispatchEvent( + new events.WebSocketClientErrorEvent({ + detail: e.detail, + }), + ); + } + } else if (event instanceof events.WebSocketConnectionStreamEvent) { + this.dispatchEvent( + new events.WebSocketConnectionStreamEvent({ detail: event.detail }), + ); + } else if (event instanceof events.WebSocketStreamDestroyEvent) { + this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); + } else { + utils.never(); + } + }; + + constructor({ address, logger }: { address: string; logger: Logger }) { + super(); + this.address = address; + this.logger = logger; + } + /** * @param obj * @param obj.host - Target host address to connect to @@ -73,11 +130,11 @@ class WebSocketClient extends EventTarget { }); const webSocket = new WebSocket(address, { - rejectUnauthorized: verifyCallback != null, + rejectUnauthorized: verifyCallback == null, }); const connectionId = client.connectionMap.allocateId(); - const connection = WebSocketConnection.createWebSocketConnection({ + const connection = await WebSocketConnection.createWebSocketConnection({ type: 'client', connectionId, remoteInfo: { @@ -88,185 +145,67 @@ class WebSocketClient extends EventTarget { socket: webSocket, verifyCallback, client: client, + }, { + timer: wsConfig.connectTimeoutTime, }); + connection.addEventListener( + 'connectionStream', + client.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'connectionStop', + client.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'connectionError', + client.handleWebSocketConnectionEvents, + ); + connection.addEventListener( + 'streamDestroy', + client.handleWebSocketConnectionEvents, + ); + client._connection = connection; + logger.info(`Created ${this.name}`); return client; } - protected address: string; - protected logger: Logger; - - public readonly connectionMap: WebSocketConnectionMap = - new WebSocketConnectionMap(); - - constructor({ address, logger }: { address: string; logger: Logger }) { - super(); - this.address = address; - this.logger = logger; - } - - public async destroy(force: boolean = false) { - this.logger.info(`Destroying ${this.constructor.name}`); - if (force) { - for (const activeConnection of this.connectionMap.values()) { - activeConnection.stop({ force }); - } - } - for (const activeConnection of this.connectionMap.values()) { - // Ignore errors here, we only care that it finishes - await activeConnection.endedProm.catch(() => {}); - } - this.logger.info(`Destroyed ${this.constructor.name}`); - } - @createDestroy.ready(new errors.ErrorWebSocketClientDestroyed()) - public async stopConnections() { - for (const activeConnection of this.activeConnections) { - activeConnection.cancel(new errors.ErrorClientEndingConnections()); - } - for (const activeConnection of this.activeConnections) { - // Ignore errors here, we only care that it finished - await activeConnection.endedProm.catch(() => {}); - } + public get connection() { + return this._connection; } - @createDestroy.ready(new errors.ErrorWebSocketClientDestroyed()) - public async startConnection( - ctx: Partial = {}, - ): Promise { - // Setting up abort/cancellation logic - const abortRaceProm = promise(); - // Ignore unhandled rejection - abortRaceProm.p.catch(() => {}); - const timer = - ctx.timer ?? - new Timer({ - delay: this.connectionTimeoutTime, - }); - void timer.then( - () => { - abortRaceProm.rejectP(new errors.ErrorClientConnectionTimedOut()); - }, - () => {}, // Ignore cancellation errors - ); - const { signal } = ctx; - let abortHandler: () => void | undefined; - if (signal != null) { - abortHandler = () => { - abortRaceProm.rejectP(signal.reason); - }; - if (signal.aborted) abortHandler(); - else signal.addEventListener('abort', abortHandler); - } - const cleanUp = () => { - // Cancel timer if it was internally created - if (ctx.timer == null) timer.cancel(); - signal?.removeEventListener('abort', abortHandler); - }; - const address = `wss://${this.host}:${this.port}`; - this.logger.info(`Connecting to ${address}`); - const connectProm = promise(); - - // Let ws handle authentication if no custom verify callback is provided. - const ws = new WebSocket(address, { - rejectUnauthorized: this.verifyCallback != null, - }); - // Handle connection failure - const openErrorHandler = (e) => { - connectProm.rejectP( - new errors.ErrorClientConnectionFailed(undefined, { - cause: e, - }), + public async destroy({ + force = false, + }: { + force?: boolean; + } = {}) { + this.logger.info(`Destroy ${this.constructor.name} on ${this.address}`); + for (const connection of this.connectionMap.values()) { + this._connection.removeEventListener( + 'connectionStream', + this.handleWebSocketConnectionEvents, ); - }; - ws.once('error', openErrorHandler); - // Authenticate server's certificate (this will be automatically done) - ws.once('upgrade', async (request) => { - const tlsSocket = request.socket as TLSSocket; - const peerCert = tlsSocket.getPeerCertificate(true); - try { - if (this.verifyCallback != null) { - await this.verifyCallback(peerCert); - } - authenticateProm.resolveP({ - localHost: request.connection.localAddress ?? '', - localPort: request.connection.localPort ?? 0, - remoteHost: request.connection.remoteAddress ?? '', - remotePort: request.connection.remotePort ?? 0, - peerCert, - }); - } catch (e) { - authenticateProm.rejectP(e); - } - }); - ws.once('open', () => { - this.logger.info('starting connection'); - connectProm.resolveP(); - }); - const earlyCloseProm = promise(); - ws.once('close', () => { - earlyCloseProm.resolveP(); - }); - // There are 3 resolve conditions here. - // 1. Connection established and authenticated - // 2. connection error or authentication failure - // 3. connection timed out - try { - await Promise.race([ - abortRaceProm.p, - await Promise.all([authenticateProm.p, connectProm.p]), - ]); - } catch (e) { - // Clean up - // unregister handlers - ws.removeAllListeners('error'); - ws.removeAllListeners('upgrade'); - ws.removeAllListeners('open'); - // Close the ws if it's open at this stage - ws.terminate(); - // Ensure the connection is removed from the active connection set before - // returning. - await earlyCloseProm.p; - throw e; - } finally { - cleanUp(); - // Cleaning up connection error - ws.removeEventListener('error', openErrorHandler); - } - - // Constructing the `ReadableWritablePair`, the lifecycle is handed off to - // the webSocketStream at this point. - const webSocketStreamClient = WebSocketConnection.createWebSocketConnection( - ws, - this.pingIntervalTime, - this.pingTimeoutTimeTime, - { - ...(await authenticateProm.p), - }, - this.logger.getChild(WebSocketStream.name), - ); - const abortStream = () => { - webSocketStreamClient.cancel( - new errors.ErrorClientStreamAborted(undefined, { - cause: signal?.reason, - }), + this._connection.removeEventListener( + 'connectionStop', + this.handleWebSocketConnectionEvents, ); - }; - // Setting up activeStream map lifecycle - this.activeConnections.add(webSocketStreamClient); - void webSocketStreamClient.endedProm - // Ignore errors, we only care that it finished - .catch(() => {}) - .finally(() => { - this.activeConnections.delete(webSocketStreamClient); - signal?.removeEventListener('abort', abortStream); + this._connection.removeEventListener( + 'connectionError', + this.handleWebSocketConnectionEvents, + ); + this._connection.removeEventListener( + 'streamDestroy', + this.handleWebSocketConnectionEvents, + ); + await connection.stop({ + errorMessage: 'cleaning up connections', + force }); - // Abort connection on signal - if (signal?.aborted === true) abortStream(); - else signal?.addEventListener('abort', abortStream); - return webSocketStreamClient; + } + this.dispatchEvent(new events.WebSocketClientDestroyEvent()); + this.logger.info(`Destroyed ${this.constructor.name}`); } } -// This is the internal implementation of the client's stream pair. export default WebSocketClient; diff --git a/src/events.ts b/src/events.ts index 8ba3268c..f27c9548 100644 --- a/src/events.ts +++ b/src/events.ts @@ -1,6 +1,28 @@ import type WebSocketStream from './WebSocketStream'; import type WebSocketConnection from './WebSocketConnection'; +// Client Events + +abstract class WebSocketClientEvent extends Event {} + +class WebSocketClientDestroyEvent extends Event { + constructor(options?: EventInit) { + super('clientDestroy', options); + } +} + +class WebSocketClientErrorEvent extends Event { + public detail: Error; + constructor( + options: EventInit & { + detail: Error; + }, + ) { + super('clientError', options); + this.detail = options.detail; + } +} + // Server events abstract class WebSocketServerEvent extends Event {} @@ -86,6 +108,9 @@ class WebSocketStreamDestroyEvent extends WebSocketStreamEvent { } export { + WebSocketClientEvent, + WebSocketClientErrorEvent, + WebSocketClientDestroyEvent, WebSocketServerEvent, WebSocketServerConnectionEvent, WebSocketServerStartEvent, diff --git a/tests/WebSocketClient.test.ts b/tests/WebSocketClient.test.ts new file mode 100644 index 00000000..edef4270 --- /dev/null +++ b/tests/WebSocketClient.test.ts @@ -0,0 +1,37 @@ +import WebSocketClient from "@/WebSocketClient"; +import WebSocketServer from "@/WebSocketServer"; +import Logger, { formatting, LogLevel, StreamHandler } from "@matrixai/logger"; +import * as testsUtils from "./utils"; + +describe(WebSocketClient.name, () => { + let tlsConfigServer: testsUtils.TLSConfigs; + + beforeAll(async () => { + tlsConfigServer = await testsUtils.generateConfig("RSA"); + }); + + const logger = new Logger('websocket test', LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), + ]); + + test('test', async () => { + const server = new WebSocketServer({ + config: tlsConfigServer, + logger + }); + await server.start(); + + const client = await WebSocketClient.createWebSocketClient({ + host: server.getHost(), + port: server.getPort(), + logger, + verifyCallback: async (cert) => {}, + }); + + const stream1 = await client.connection.streamNew('bidi'); + stream1.writable.getWriter().write(new Uint8Array([1, 2, 3])); + + }); +}); From c1eed2489000371ea4316f918ba9fb8694bee48b Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:46:41 +1000 Subject: [PATCH 031/149] fix: writableDesiredSize should be decremented before stream messages are sent to prevent race condition --- src/WebSocketStream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 4631cd75..8708e8c1 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -145,9 +145,9 @@ class WebSocketStream // Reset the promise to wait for another ACK this.writableDesiredSizeProm = promise(); } - await this.streamSend(StreamType.DATA, data); // Decrement the desired size by the amount of bytes written this.writableDesiredSize -= bytesWritten; + await this.streamSend(StreamType.DATA, data); if (isChunkable) { await writeHandler(chunk.subarray(bytesWritten), controller); From 73f7ae635b41b79512a3814a0b8fb69a20845b30 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:03:04 +1000 Subject: [PATCH 032/149] fix: bubble up WebSocketStreamDestroyEvents on the server --- src/WebSocketServer.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 81ff80bd..d690bda4 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -66,7 +66,12 @@ class WebSocketServer extends EventTarget { detail: event.detail, }), ); - } else { + } else if (event instanceof webSocketEvents.WebSocketStreamDestroyEvent) { + this.dispatchEvent( + new webSocketEvents.WebSocketStreamDestroyEvent() + ); + } + else { utils.never(); } }; From a84fa4c9d530aa6ed4567db76470c2cf1e6d635d Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:53:16 +1000 Subject: [PATCH 033/149] fix: close and error frames will be ignored on closed streams --- src/WebSocketConnection.ts | 6 +++++- src/WebSocketStream.ts | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 49aaa2b1..ed0eec60 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -22,7 +22,7 @@ import { Timer } from '@matrixai/timer'; import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; -import { fromStreamId, promise, toStreamId } from './utils'; +import { fromStreamId, promise, StreamType, toStreamId } from './utils'; import * as events from './events'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -156,6 +156,10 @@ class WebSocketConnection extends EventTarget { let stream = this.streamMap.get(streamId); if (stream == null) { + const messageType = message.at(0); + if (messageType === StreamType.CLOSE || messageType === StreamType.ERROR) { + return; + } stream = await WebSocketStream.createWebSocketStream({ connection: this, streamId, diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 8708e8c1..6786759f 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -191,6 +191,10 @@ class WebSocketStream this.writableDesiredSizeProm.resolveP(); this.cancel(); // Removing stream from the connection's stream map + // TODO: the other side currently will send back an ERROR/CLOSE frame from us sending an ERROR/CLOSE frame from this.close(). + // However, out stream gets deleted before we receive that message on the connection. + // So the connection will infinitely create streams with the same streamId when it receives the ERROR/CLOSE frame. + // I'm dealing with this by just filtering out ERROR/CLOSE frames in the connection's onMessage handler, but there might be a better way to do this. this.connection.streamMap.delete(this.streamId); this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); this.logger.info(`Destroyed ${this.constructor.name}`); From bce0e7deaa3498b89c11b916be1f1cf0fa9705fa Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:09:08 +1000 Subject: [PATCH 034/149] fix: readable controller properly closes on WebSocketStream --- src/WebSocketStream.ts | 22 +++++++++++++++++----- src/errors.ts | 10 ++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 6786759f..1fce6a69 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -189,7 +189,7 @@ class WebSocketStream this.logger.info(`Destroy ${this.constructor.name}`); // Force close any open streams this.writableDesiredSizeProm.resolveP(); - this.cancel(); + this.cancel(new errors.ErrorWebSocketStreamClose()); // Removing stream from the connection's stream map // TODO: the other side currently will send back an ERROR/CLOSE frame from us sending an ERROR/CLOSE frame from this.close(). // However, out stream gets deleted before we receive that message on the connection. @@ -309,6 +309,9 @@ class WebSocketStream ); } } else if (type === StreamType.DATA) { + if (this._readableEnded) { + return; + } if ( this.readableController.desiredSize != null && data.length > this.readableController.desiredSize @@ -331,8 +334,15 @@ class WebSocketStream reason = await this.codeToReason('recv', errorCode); } if (shutdown === StreamShutdown.Read) { + if (this._readableEnded) { + return; + } await this.signalReadableEnd(isError, reason); + this.readableController.close(); } else if (shutdown === StreamShutdown.Write) { + if (this._writableEnded) { + return; + } await this.signalWritableEnd(isError, reason); } else { never('invalid shutdown type'); @@ -348,7 +358,8 @@ class WebSocketStream ), ); } - } else { + } + else { never(); } } @@ -357,7 +368,8 @@ class WebSocketStream * Forces the active stream to end early */ public cancel(reason?: any): void { - const isError = reason != null; + const isError = reason != null && !(reason instanceof errors.ErrorWebSocketStreamClose); + reason = reason ?? new errors.ErrorWebSocketStreamCancel(); // Close the streams with the given error, if (!this._readableEnded) { this.readableController.error(reason); @@ -386,10 +398,10 @@ class WebSocketStream if (isError) { const code = await this.reasonToCode('send', reason); await this.streamSend(StreamType.ERROR, StreamShutdown.Write, code); + this.readableController.error(reason); } else { await this.streamSend(StreamType.CLOSE, StreamShutdown.Write); } - this.readableController.error(reason); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); if (this[status] !== 'destroying') void this.destroy(); @@ -416,10 +428,10 @@ class WebSocketStream if (isError) { const code = await this.reasonToCode('send', reason); await this.streamSend(StreamType.ERROR, StreamShutdown.Read, code); + this.writableController.error(reason); } else { await this.streamSend(StreamType.CLOSE, StreamShutdown.Read); } - this.writableController.error(reason); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); if (this[status] !== 'destroying') void this.destroy(); diff --git a/src/errors.ts b/src/errors.ts index 913b7326..e757315b 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -83,6 +83,14 @@ class ErrorWebSocketStreamDestroyed extends ErrorWebSocketStream { static description = 'WebSocket Stream is destroyed'; } +class ErrorWebSocketStreamClose extends ErrorWebSocketStream { + static description = 'WebSocket Stream force close'; +} + +class ErrorWebSocketStreamCancel extends ErrorWebSocketStream { + static description = 'WebSocket Stream was cancelled without a provided reason'; +} + // Misc class ErrorWebSocketUndefinedBehaviour extends ErrorWebSocket { @@ -107,5 +115,7 @@ export { ErrorWebSocketStreamReaderParse, ErrorWebSocketStreamReaderBufferOverload, ErrorWebSocketStreamDestroyed, + ErrorWebSocketStreamClose, + ErrorWebSocketStreamCancel, ErrorWebSocketUndefinedBehaviour, }; From 59d7e5c63879f479c2e2ca7775c8e565465b292b Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:30:49 +1000 Subject: [PATCH 035/149] feat: WebSocketStream tests for varying sized writes --- tests/WebSocketClient.test.ts | 26 +++-- tests/WebSocketStream.test.ts | 209 ++++++++++++++++++++++++++-------- tests/utils.ts | 12 ++ 3 files changed, 193 insertions(+), 54 deletions(-) diff --git a/tests/WebSocketClient.test.ts b/tests/WebSocketClient.test.ts index edef4270..df425898 100644 --- a/tests/WebSocketClient.test.ts +++ b/tests/WebSocketClient.test.ts @@ -1,20 +1,26 @@ +import { serverDefault } from "@/config"; +import { WebSocketConnectionStreamEvent } from "@/events"; import WebSocketClient from "@/WebSocketClient"; import WebSocketServer from "@/WebSocketServer"; import Logger, { formatting, LogLevel, StreamHandler } from "@matrixai/logger"; import * as testsUtils from "./utils"; -describe(WebSocketClient.name, () => { - let tlsConfigServer: testsUtils.TLSConfigs; - - beforeAll(async () => { - tlsConfigServer = await testsUtils.generateConfig("RSA"); - }); +// process.on('unhandledRejection', (reason) => { +// console.log(reason); // log the reason including the stack trace +// throw reason; +// }); +describe(WebSocketClient.name, () => { const logger = new Logger('websocket test', LogLevel.WARN, [ new StreamHandler( formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, ), ]); + let tlsConfigServer: testsUtils.TLSConfigs; + + beforeAll(async () => { + tlsConfigServer = await testsUtils.generateConfig("RSA"); + }); test('test', async () => { const server = new WebSocketServer({ @@ -23,6 +29,10 @@ describe(WebSocketClient.name, () => { }); await server.start(); + server.addEventListener("connectionStream", async (event: WebSocketConnectionStreamEvent) => { + // await event.detail.readable.getReader().read(); + }); + const client = await WebSocketClient.createWebSocketClient({ host: server.getHost(), port: server.getPort(), @@ -31,7 +41,7 @@ describe(WebSocketClient.name, () => { }); const stream1 = await client.connection.streamNew('bidi'); - stream1.writable.getWriter().write(new Uint8Array([1, 2, 3])); - + await stream1.writable.getWriter().write(new Uint8Array(serverDefault.streamBufferSize)); + await stream1.destroy(); }); }); diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 4d74c679..83f2e51c 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -2,10 +2,23 @@ import type { StreamId } from '@/types'; import WebSocketStream from '@/WebSocketStream'; import WebSocketConnection from '@/WebSocketConnection'; import * as events from '@/events'; -import { promise } from '@/utils'; +import { promise, StreamType } from '@/utils'; import * as testUtils from './utils'; +import Logger, { formatting, LogLevel, StreamHandler } from "@matrixai/logger"; +import { fc, testProp } from '@fast-check/jest'; +import * as config from '@/config'; -const DEFAULT_BUFFER_SIZE = 1024; +const logger1 = new Logger('stream 1', LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), +]); + +const logger2 = new Logger('stream 2', LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), +]); jest.mock('@/WebSocketConnection', () => { return jest.fn().mockImplementation(() => { @@ -22,13 +35,16 @@ jest.mock('@/WebSocketConnection', () => { }; instance.streamMap = new Map(); instance.streamSend = async (streamId: StreamId, data: Uint8Array) => { - console.log('streamSend', streamId, data); let stream = instance.connectedConnection!.streamMap.get(streamId); if (stream == null) { + if (data.at(0) === StreamType.CLOSE || data.at(0) === StreamType.ERROR) { + return; + } stream = await WebSocketStream.createWebSocketStream({ streamId, - bufferSize: DEFAULT_BUFFER_SIZE, + bufferSize: config.clientDefault.streamBufferSize, connection: instance.connectedConnection!, + logger: logger2 }); instance.connectedConnection!.dispatchEvent( new events.WebSocketConnectionStreamEvent({ @@ -47,8 +63,11 @@ const connectionMock = jest.mocked(WebSocketConnection, true); describe(WebSocketStream.name, () => { let connection1: WebSocketConnection; let connection2: WebSocketConnection; + let streamIdCounter = 0n; + beforeEach(async () => { connectionMock.mockClear(); + streamIdCounter = 0n; connection1 = new (WebSocketConnection as any)(); connection2 = new (WebSocketConnection as any)(); (connection1 as any).connectTo(connection2); @@ -56,9 +75,10 @@ describe(WebSocketStream.name, () => { async function createStreamPair(connection1, connection2) { const stream1 = await WebSocketStream.createWebSocketStream({ - streamId: 0n as StreamId, - bufferSize: DEFAULT_BUFFER_SIZE, + streamId: streamIdCounter as StreamId, + bufferSize: config.clientDefault.streamBufferSize, connection: connection1, + logger: logger1 }); const createStream2Prom = promise(); connection2.addEventListener( @@ -69,46 +89,143 @@ describe(WebSocketStream.name, () => { { once: true }, ); const stream2 = await createStream2Prom.p; + streamIdCounter++; return [stream1, stream2]; } - // TestProp( - // 'normal', - // [fc.infiniteStream(fc.uint8Array({minLength: 1, maxLength: DEFAULT_BUFFER_SIZE}))], - // async (iterable) => { - - // const writingTest = async () => { - // const stream2Writable = stream2.writable; - // const stream = testUtils.toReadableStream(iterable); - // await stream.pipeTo(stream2Writable); - // } - - // const readingTest = async () => { - // const stream1Readable = stream1.readable; - // await stream1Readable.pipeTo([]); - // } - - // await Promise.all([writingTest(), readingTest()]); - - // // const stream2Writable = stream2.writable; - // // const buffer = new Uint8Array(2); - // // const writeProm = stream2Writable.getWriter().write(buffer); - // // await stream1Readable.getReader().read(); - // // await writeProm; - // // await stream1.destroy(); - // } - // ); - test('normal', async () => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); - - const buffer = new Uint8Array(DEFAULT_BUFFER_SIZE + 1); - await testUtils.randomBytes(buffer); - - const stream1Readable = stream1.readable; - const stream2Writable = stream2.writable; - const writeProm = stream2Writable.getWriter().write(buffer); - await stream1Readable.getReader().read(); - await writeProm; - - await stream1.destroy(); - }); + testProp( + 'single write within buffer size', + [fc.uint8Array({maxLength: config.clientDefault.streamBufferSize})], + async (data) => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + + const writer = stream2Writable.getWriter(); + const reader = stream1Readable.getReader(); + + const writeProm = (async () => { + await writer.write(data); + await writer.close(); + })(); + + const readChunks: Array = []; + const readProm = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + })(); + + await Promise.all([writeProm, readProm]); + + expect(readChunks).toEqual([data]); + + await stream1.destroy(); + } + ); + testProp( + 'single write outside buffer size', + [fc.uint8Array()], + async (data) => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + + const writer = stream2Writable.getWriter(); + const reader = stream1Readable.getReader(); + + const writeProm = (async () => { + await writer.write(data); + await writer.close(); + })(); + + const readChunks: Array = []; + const readProm = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + })(); + + await Promise.all([writeProm, readProm]); + + expect(testUtils.concatUInt8Array(...readChunks)).toEqual(data); + + await stream1.destroy(); + } + ); + testProp( + 'multiple writes within buffer size', + [fc.array(fc.uint8Array())], + async (data) => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + + const writer = stream2Writable.getWriter(); + const reader = stream1Readable.getReader(); + + const writeProm = (async () => { + for (const chunk of data) { + await writer.write(chunk); + } + await writer.close(); + })(); + + const readChunks: Array = []; + const readProm = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + })(); + + await Promise.all([writeProm, readProm]); + + expect(testUtils.concatUInt8Array(...readChunks)).toEqual(testUtils.concatUInt8Array(...data)); + + await stream1.destroy(); + } + ); + testProp( + 'multiple writes outside buffer size', + [fc.array(fc.uint8Array())], + async (data) => { + const [stream1, stream2] = await createStreamPair(connection1, connection2); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + + const writer = stream2Writable.getWriter(); + const reader = stream1Readable.getReader(); + + const writeProm = (async () => { + for (const chunk of data) { + await writer.write(chunk); + } + await writer.close(); + })(); + + const readChunks: Array = []; + const readProm = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + })(); + + await Promise.all([writeProm, readProm]); + + expect(testUtils.concatUInt8Array(...readChunks)).toEqual(testUtils.concatUInt8Array(...data)); + + await stream1.destroy(); + } + ); }); diff --git a/tests/utils.ts b/tests/utils.ts index 6b79503a..605b1a00 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -599,6 +599,17 @@ function toReadableStream(iterator: IterableIterator) { }); } +function concatUInt8Array(...arrays: Array) { + const totalLength = arrays.reduce((acc, val) => acc + val.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; +} + export { sleep, randomBytes, @@ -615,6 +626,7 @@ export { verifyHMAC, generateConfig, toReadableStream, + concatUInt8Array }; export type { KeyTypes, TLSConfigs }; From 051b2cd6e998da896e94fd73638c92d7a908ce4c Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:31:25 +1000 Subject: [PATCH 036/149] lintfix --- src/WebSocketClient.ts | 31 +++++++++++--------- src/WebSocketConnection.ts | 5 +++- src/WebSocketServer.ts | 9 ++---- src/WebSocketStream.ts | 8 +++--- src/errors.ts | 3 +- tests/WebSocketClient.test.ts | 31 +++++++++++--------- tests/WebSocketStream.test.ts | 53 ++++++++++++++++++++++++----------- tests/utils.ts | 2 +- 8 files changed, 85 insertions(+), 57 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 4b129826..3132c5ff 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -134,20 +134,23 @@ class WebSocketClient extends EventTarget { }); const connectionId = client.connectionMap.allocateId(); - const connection = await WebSocketConnection.createWebSocketConnection({ - type: 'client', - connectionId, - remoteInfo: { - host: host_, - port: port_, + const connection = await WebSocketConnection.createWebSocketConnection( + { + type: 'client', + connectionId, + remoteInfo: { + host: host_, + port: port_, + }, + config: wsConfig, + socket: webSocket, + verifyCallback, + client: client, }, - config: wsConfig, - socket: webSocket, - verifyCallback, - client: client, - }, { - timer: wsConfig.connectTimeoutTime, - }); + { + timer: wsConfig.connectTimeoutTime, + }, + ); connection.addEventListener( 'connectionStream', client.handleWebSocketConnectionEvents, @@ -200,7 +203,7 @@ class WebSocketClient extends EventTarget { ); await connection.stop({ errorMessage: 'cleaning up connections', - force + force, }); } this.dispatchEvent(new events.WebSocketClientDestroyEvent()); diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index ed0eec60..48314aaf 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -157,7 +157,10 @@ class WebSocketConnection extends EventTarget { let stream = this.streamMap.get(streamId); if (stream == null) { const messageType = message.at(0); - if (messageType === StreamType.CLOSE || messageType === StreamType.ERROR) { + if ( + messageType === StreamType.CLOSE || + messageType === StreamType.ERROR + ) { return; } stream = await WebSocketStream.createWebSocketStream({ diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index d690bda4..c2298377 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -67,11 +67,8 @@ class WebSocketServer extends EventTarget { }), ); } else if (event instanceof webSocketEvents.WebSocketStreamDestroyEvent) { - this.dispatchEvent( - new webSocketEvents.WebSocketStreamDestroyEvent() - ); - } - else { + this.dispatchEvent(new webSocketEvents.WebSocketStreamDestroyEvent()); + } else { utils.never(); } }; @@ -149,7 +146,7 @@ class WebSocketServer extends EventTarget { destroyProms.push( webSocketConnection.stop({ errorMessage: 'cleaning up connections', - force + force, }), ); } diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 1fce6a69..e62f9a32 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -358,8 +358,7 @@ class WebSocketStream ), ); } - } - else { + } else { never(); } } @@ -368,7 +367,8 @@ class WebSocketStream * Forces the active stream to end early */ public cancel(reason?: any): void { - const isError = reason != null && !(reason instanceof errors.ErrorWebSocketStreamClose); + const isError = + reason != null && !(reason instanceof errors.ErrorWebSocketStreamClose); reason = reason ?? new errors.ErrorWebSocketStreamCancel(); // Close the streams with the given error, if (!this._readableEnded) { @@ -422,7 +422,7 @@ class WebSocketStream if (this._writableEnded) return; // Indicate that sending side is closed this._writableEnded = true; - // resolve backpressure blocking promise in case unresolved + // Resolve backpressure blocking promise in case unresolved this.writableDesiredSizeProm.resolveP(); // Shutdown the read side of the other stream if (isError) { diff --git a/src/errors.ts b/src/errors.ts index e757315b..89324444 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -88,7 +88,8 @@ class ErrorWebSocketStreamClose extends ErrorWebSocketStream { } class ErrorWebSocketStreamCancel extends ErrorWebSocketStream { - static description = 'WebSocket Stream was cancelled without a provided reason'; + static description = + 'WebSocket Stream was cancelled without a provided reason'; } // Misc diff --git a/tests/WebSocketClient.test.ts b/tests/WebSocketClient.test.ts index df425898..39fb70ff 100644 --- a/tests/WebSocketClient.test.ts +++ b/tests/WebSocketClient.test.ts @@ -1,11 +1,11 @@ -import { serverDefault } from "@/config"; -import { WebSocketConnectionStreamEvent } from "@/events"; -import WebSocketClient from "@/WebSocketClient"; -import WebSocketServer from "@/WebSocketServer"; -import Logger, { formatting, LogLevel, StreamHandler } from "@matrixai/logger"; -import * as testsUtils from "./utils"; +import type { WebSocketConnectionStreamEvent } from '@/events'; +import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; +import { serverDefault } from '@/config'; +import WebSocketClient from '@/WebSocketClient'; +import WebSocketServer from '@/WebSocketServer'; +import * as testsUtils from './utils'; -// process.on('unhandledRejection', (reason) => { +// Process.on('unhandledRejection', (reason) => { // console.log(reason); // log the reason including the stack trace // throw reason; // }); @@ -19,19 +19,22 @@ describe(WebSocketClient.name, () => { let tlsConfigServer: testsUtils.TLSConfigs; beforeAll(async () => { - tlsConfigServer = await testsUtils.generateConfig("RSA"); + tlsConfigServer = await testsUtils.generateConfig('RSA'); }); test('test', async () => { const server = new WebSocketServer({ config: tlsConfigServer, - logger + logger, }); await server.start(); - server.addEventListener("connectionStream", async (event: WebSocketConnectionStreamEvent) => { - // await event.detail.readable.getReader().read(); - }); + server.addEventListener( + 'connectionStream', + async (event: WebSocketConnectionStreamEvent) => { + // Await event.detail.readable.getReader().read(); + }, + ); const client = await WebSocketClient.createWebSocketClient({ host: server.getHost(), @@ -41,7 +44,9 @@ describe(WebSocketClient.name, () => { }); const stream1 = await client.connection.streamNew('bidi'); - await stream1.writable.getWriter().write(new Uint8Array(serverDefault.streamBufferSize)); + await stream1.writable + .getWriter() + .write(new Uint8Array(serverDefault.streamBufferSize)); await stream1.destroy(); }); }); diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 83f2e51c..c2be7b75 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -1,12 +1,12 @@ import type { StreamId } from '@/types'; +import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; +import { fc, testProp } from '@fast-check/jest'; import WebSocketStream from '@/WebSocketStream'; import WebSocketConnection from '@/WebSocketConnection'; import * as events from '@/events'; import { promise, StreamType } from '@/utils'; -import * as testUtils from './utils'; -import Logger, { formatting, LogLevel, StreamHandler } from "@matrixai/logger"; -import { fc, testProp } from '@fast-check/jest'; import * as config from '@/config'; +import * as testUtils from './utils'; const logger1 = new Logger('stream 1', LogLevel.WARN, [ new StreamHandler( @@ -37,14 +37,17 @@ jest.mock('@/WebSocketConnection', () => { instance.streamSend = async (streamId: StreamId, data: Uint8Array) => { let stream = instance.connectedConnection!.streamMap.get(streamId); if (stream == null) { - if (data.at(0) === StreamType.CLOSE || data.at(0) === StreamType.ERROR) { + if ( + data.at(0) === StreamType.CLOSE || + data.at(0) === StreamType.ERROR + ) { return; } stream = await WebSocketStream.createWebSocketStream({ streamId, bufferSize: config.clientDefault.streamBufferSize, connection: instance.connectedConnection!, - logger: logger2 + logger: logger2, }); instance.connectedConnection!.dispatchEvent( new events.WebSocketConnectionStreamEvent({ @@ -78,7 +81,7 @@ describe(WebSocketStream.name, () => { streamId: streamIdCounter as StreamId, bufferSize: config.clientDefault.streamBufferSize, connection: connection1, - logger: logger1 + logger: logger1, }); const createStream2Prom = promise(); connection2.addEventListener( @@ -94,9 +97,12 @@ describe(WebSocketStream.name, () => { } testProp( 'single write within buffer size', - [fc.uint8Array({maxLength: config.clientDefault.streamBufferSize})], + [fc.uint8Array({ maxLength: config.clientDefault.streamBufferSize })], async (data) => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); + const [stream1, stream2] = await createStreamPair( + connection1, + connection2, + ); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -123,13 +129,16 @@ describe(WebSocketStream.name, () => { expect(readChunks).toEqual([data]); await stream1.destroy(); - } + }, ); testProp( 'single write outside buffer size', [fc.uint8Array()], async (data) => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); + const [stream1, stream2] = await createStreamPair( + connection1, + connection2, + ); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -156,13 +165,16 @@ describe(WebSocketStream.name, () => { expect(testUtils.concatUInt8Array(...readChunks)).toEqual(data); await stream1.destroy(); - } + }, ); testProp( 'multiple writes within buffer size', [fc.array(fc.uint8Array())], async (data) => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); + const [stream1, stream2] = await createStreamPair( + connection1, + connection2, + ); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -188,16 +200,21 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual(testUtils.concatUInt8Array(...data)); + expect(testUtils.concatUInt8Array(...readChunks)).toEqual( + testUtils.concatUInt8Array(...data), + ); await stream1.destroy(); - } + }, ); testProp( 'multiple writes outside buffer size', [fc.array(fc.uint8Array())], async (data) => { - const [stream1, stream2] = await createStreamPair(connection1, connection2); + const [stream1, stream2] = await createStreamPair( + connection1, + connection2, + ); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -223,9 +240,11 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual(testUtils.concatUInt8Array(...data)); + expect(testUtils.concatUInt8Array(...readChunks)).toEqual( + testUtils.concatUInt8Array(...data), + ); await stream1.destroy(); - } + }, ); }); diff --git a/tests/utils.ts b/tests/utils.ts index 605b1a00..74063ee2 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -626,7 +626,7 @@ export { verifyHMAC, generateConfig, toReadableStream, - concatUInt8Array + concatUInt8Array, }; export type { KeyTypes, TLSConfigs }; From a7e41f2556b976a55bf3b6c3d49e21715bced9d0 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:38:52 +1000 Subject: [PATCH 037/149] chore: upgraded to node 20 --- .eslintrc | 1 + pkgs.nix | 2 +- shell.nix | 2 +- tests/utils.ts | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.eslintrc b/.eslintrc index 44a8d5ac..786b7e5a 100644 --- a/.eslintrc +++ b/.eslintrc @@ -39,6 +39,7 @@ "message": "Use `globalThis` instead" } ], + "prefer-rest-params": 0, "require-yield": 0, "eqeqeq": ["error", "smart"], "spaced-comment": [ diff --git a/pkgs.nix b/pkgs.nix index bb501409..2997ae2e 100644 --- a/pkgs.nix +++ b/pkgs.nix @@ -1,4 +1,4 @@ import ( - let rev = "f294325aed382b66c7a188482101b0f336d1d7db"; in + let rev = "ea5234e7073d5f44728c499192544a84244bf35a"; in builtins.fetchTarball "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz" ) diff --git a/shell.nix b/shell.nix index 5839479a..4cfef47a 100644 --- a/shell.nix +++ b/shell.nix @@ -3,7 +3,7 @@ with pkgs; mkShell { nativeBuildInputs = [ - nodejs + nodejs_20 shellcheck gitAndTools.gh ]; diff --git a/tests/utils.ts b/tests/utils.ts index 74063ee2..ec391eac 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -11,9 +11,9 @@ import { never } from '@/utils'; const webcrypto = new Crypto(); /** - * Monkey patches the global crypto object polyfill + * Monkey patches the global crypto object polyfill. This doesn't work on node 20, as globalThis has been made read-only. */ -globalThis.crypto = webcrypto; +// globalThis.crypto = webcrypto; x509.cryptoProvider.set(webcrypto); From e669551daaadee17bdd1ef58c0156ffbe8864686 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:39:12 +1000 Subject: [PATCH 038/149] fix: `single write within buffer size` now uses correct buffer size --- tests/WebSocketStream.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index c2be7b75..cd1f60dc 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -169,7 +169,7 @@ describe(WebSocketStream.name, () => { ); testProp( 'multiple writes within buffer size', - [fc.array(fc.uint8Array())], + [fc.array(fc.uint8Array({maxLength: config.clientDefault.streamBufferSize}))], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, From 29dbd2dc287cb9884b7c3d336fb7af3a4907f3ca Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 14:10:37 +1000 Subject: [PATCH 039/149] feat: migrated events to use js-events system WIP: migrating to js-events --- package-lock.json | 17 +++- package.json | 3 +- src/WebSocketClient.ts | 67 ++++++--------- src/WebSocketConnection.ts | 32 +++++--- src/WebSocketServer.ts | 69 +++++----------- src/WebSocketStream.ts | 13 +-- src/events.ts | 148 ++++++++++------------------------ tests/WebSocketClient.test.ts | 6 +- tests/WebSocketServer.test.ts | 7 +- tests/WebSocketStream.test.ts | 17 ++-- 10 files changed, 144 insertions(+), 235 deletions(-) diff --git a/package-lock.json b/package-lock.json index ee53680a..0d3d0521 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,9 @@ "@matrixai/async-cancellable": "^1.1.1", "@matrixai/async-init": "^1.8.4", "@matrixai/async-locks": "^4.0.0", - "@matrixai/contexts": "^1.1.0", + "@matrixai/contexts": "^1.2.0", "@matrixai/errors": "^1.1.7", + "@matrixai/events": "^3.0.0", "@matrixai/logger": "^3.1.0", "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", @@ -1681,9 +1682,9 @@ } }, "node_modules/@matrixai/contexts": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@matrixai/contexts/-/contexts-1.1.0.tgz", - "integrity": "sha512-sB4UrT8T6OICBujNxTOss8O+dAHnbfndBqZG0fO1PSZUgaZlXDg3cSz9ButbV4JLEz25UvPgh4ChvwTP31DUcQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@matrixai/contexts/-/contexts-1.2.0.tgz", + "integrity": "sha512-MR/B02Kf4UoliP9b/gMMKsvWV6QM4JSPKTIqrhQP2tbOl3FwLI+AIhL3vgYEj1Xw+PP8bY5cr8ontJ8x6AJyMg==", "dependencies": { "@matrixai/async-cancellable": "^1.1.1", "@matrixai/async-locks": "^4.0.0", @@ -1700,6 +1701,14 @@ "ts-custom-error": "3.2.2" } }, + "node_modules/@matrixai/events": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@matrixai/events/-/events-3.0.0.tgz", + "integrity": "sha512-qg1pXV3wcGi3Gzy8vkQM/hiTyeVPrR3gN2N7gB/2JkX2L2fGmPf1xrJCizK7KgtUZOvjcKwjN+uZzspu3aGDaw==", + "engines": { + "node": ">=19.0.0" + } + }, "node_modules/@matrixai/logger": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@matrixai/logger/-/logger-3.1.0.tgz", diff --git a/package.json b/package.json index e9edba5c..1cb7084e 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,9 @@ "@matrixai/async-cancellable": "^1.1.1", "@matrixai/async-init": "^1.8.4", "@matrixai/async-locks": "^4.0.0", - "@matrixai/contexts": "^1.1.0", + "@matrixai/contexts": "^1.2.0", "@matrixai/errors": "^1.1.7", + "@matrixai/events": "^3.0.0", "@matrixai/logger": "^3.1.0", "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 3132c5ff..33ac65aa 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -3,6 +3,7 @@ import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; import { Validator } from 'ip-num'; +import { EventAll } from '@matrixai/events'; import * as errors from './errors'; import WebSocketConnection from './WebSocketConnection'; import WebSocketConnectionMap from './WebSocketConnectionMap'; @@ -21,16 +22,18 @@ class WebSocketClient extends EventTarget { protected address: string; - protected handleWebSocketConnectionEvents = async ( - event: events.WebSocketConnectionEvent, + protected handleEventWebSocketConnection = async ( + event_: EventAll | Event, ) => { - if (event instanceof events.WebSocketConnectionErrorEvent) { + let event: Event; + if (event_ instanceof EventAll) { + event = event_.detail; + } else { + event = event_; + } + if (event instanceof events.EventWebSocketConnectionError) { this.dispatchEvent( - new events.WebSocketConnectionErrorEvent({ - detail: new errors.ErrorWebSocketClient('Connection error', { - cause: event.detail, - }), - }), + (event as events.EventWebSocketConnectionError).clone(), ); try { // Force destroy means don't destroy gracefully @@ -39,12 +42,12 @@ class WebSocketClient extends EventTarget { }); } catch (e) { this.dispatchEvent( - new events.WebSocketClientErrorEvent({ + new events.EventWebSocketClientError({ detail: e.detail, }), ); } - } else if (event instanceof events.WebSocketConnectionStopEvent) { + } else if (event instanceof events.EventWebSocketConnectionStop) { try { // Force destroy means don't destroy gracefully await this.destroy({ @@ -52,17 +55,17 @@ class WebSocketClient extends EventTarget { }); } catch (e) { this.dispatchEvent( - new events.WebSocketClientErrorEvent({ + new events.EventWebSocketClientError({ detail: e.detail, }), ); } - } else if (event instanceof events.WebSocketConnectionStreamEvent) { + } else if (event instanceof events.EventWebSocketConnectionStream) { this.dispatchEvent( - new events.WebSocketConnectionStreamEvent({ detail: event.detail }), + (event as events.EventWebSocketConnectionStream).clone(), ); - } else if (event instanceof events.WebSocketStreamDestroyEvent) { - this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); + } else if (event instanceof events.EventWebSocketStreamDestroy) { + this.dispatchEvent((event as events.EventWebSocketStreamDestroy).clone()); } else { utils.never(); } @@ -152,20 +155,8 @@ class WebSocketClient extends EventTarget { }, ); connection.addEventListener( - 'connectionStream', - client.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'connectionStop', - client.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'connectionError', - client.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'streamDestroy', - client.handleWebSocketConnectionEvents, + EventAll.name, + client.handleEventWebSocketConnection, ); client._connection = connection; @@ -186,27 +177,15 @@ class WebSocketClient extends EventTarget { this.logger.info(`Destroy ${this.constructor.name} on ${this.address}`); for (const connection of this.connectionMap.values()) { this._connection.removeEventListener( - 'connectionStream', - this.handleWebSocketConnectionEvents, - ); - this._connection.removeEventListener( - 'connectionStop', - this.handleWebSocketConnectionEvents, - ); - this._connection.removeEventListener( - 'connectionError', - this.handleWebSocketConnectionEvents, - ); - this._connection.removeEventListener( - 'streamDestroy', - this.handleWebSocketConnectionEvents, + EventAll.name, + this.handleEventWebSocketConnection, ); await connection.stop({ errorMessage: 'cleaning up connections', force, }); } - this.dispatchEvent(new events.WebSocketClientDestroyEvent()); + this.dispatchEvent(new events.EventWebSocketClientDestroy()); this.logger.info(`Destroyed ${this.constructor.name}`); } } diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 48314aaf..f792d05f 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -20,6 +20,7 @@ import Logger from '@matrixai/logger'; import * as ws from 'ws'; import { Timer } from '@matrixai/timer'; import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; +import { Evented } from '@matrixai/events'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { fromStreamId, promise, StreamType, toStreamId } from './utils'; @@ -40,7 +41,9 @@ const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); * - streamDestroy */ interface WebSocketConnection extends startStop.StartStop {} +interface WebSocketConnection extends Evented {} @startStop.StartStop() +@Evented() class WebSocketConnection extends EventTarget { /** * This determines when it is a client or server connection. @@ -122,8 +125,10 @@ class WebSocketConnection extends EventTarget { /** * Bubble up stream destroy event */ - protected handleWebSocketStreamDestroyEvent = () => { - this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); + protected handleEventWebSocketStreamDestroy = ( + event: events.EventWebSocketStreamDestroy, + ) => { + this.dispatchEvent(event.clone()); }; /** @@ -142,8 +147,8 @@ class WebSocketConnection extends EventTarget { protected messageHandler = async (data: ws.RawData, isBinary: boolean) => { if (!isBinary || data instanceof Array) { this.dispatchEvent( - new events.WebSocketConnectionErrorEvent({ - detail: new errors.ErrorWebSocketUndefinedBehaviour(), + new events.EventWebSocketConnectionError({ + detail: new errors.ErrorWebSocketUndefinedBehaviour() as Error, }), ); return; @@ -170,12 +175,12 @@ class WebSocketConnection extends EventTarget { logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); stream.addEventListener( - 'streamDestroy', - this.handleWebSocketStreamDestroyEvent, + events.EventWebSocketStreamDestroy.name, + this.handleEventWebSocketStreamDestroy, { once: true }, ); this.dispatchEvent( - new events.WebSocketConnectionStreamEvent({ + new events.EventWebSocketConnectionStream({ detail: stream, }), ); @@ -194,7 +199,7 @@ class WebSocketConnection extends EventTarget { protected errorHandler = (err: Error) => { this.dispatchEvent( - new events.WebSocketConnectionErrorEvent({ + new events.EventWebSocketConnectionError({ detail: new errors.ErrorWebSocketConnectionSocket( 'An error occurred on the underlying WebSocket instance.', { @@ -455,8 +460,8 @@ class WebSocketConnection extends EventTarget { logger: this.logger.getChild(`${WebSocketStream.name} ${streamId!}`), }); stream.addEventListener( - 'streamDestroy', - this.handleWebSocketStreamDestroyEvent, + events.EventWebSocketStreamDestroy.name, + this.handleEventWebSocketStreamDestroy, { once: true }, ); // Ok the stream is opened and working @@ -540,7 +545,7 @@ class WebSocketConnection extends EventTarget { this.parentInstance.connectionMap.delete(this.connectionId); } - this.dispatchEvent(new events.WebSocketConnectionStopEvent()); + this.dispatchEvent(new events.EventWebSocketConnectionStop()); this.logger.info(`Stopped ${this.constructor.name}`); } @@ -549,8 +554,9 @@ class WebSocketConnection extends EventTarget { const timeout = this.config.keepAliveTimeoutTime; const keepAliveTimeOutHandler = () => { this.dispatchEvent( - new events.WebSocketConnectionErrorEvent({ - detail: new errors.ErrorWebSocketConnectionKeepAliveTimeOut(), + new events.EventWebSocketConnectionError({ + detail: + new errors.ErrorWebSocketConnectionKeepAliveTimeOut() as Error, }), ); if (this[startStop.running] && this[startStop.status] !== 'stopping') { diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index c2298377..0efa964d 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -11,12 +11,12 @@ import https from 'https'; import { startStop, status } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; +import { EventAll, EventDefault } from '@matrixai/events'; import * as errors from './errors'; -import * as webSocketEvents from './events'; +import * as events from './events'; import { never, promise } from './utils'; import WebSocketConnection from './WebSocketConnection'; import { serverDefault } from './config'; -import * as utils from './utils'; import WebSocketConnectionMap from './WebSocketConnectionMap'; /** @@ -48,28 +48,14 @@ class WebSocketServer extends EventTarget { protected _host: string; protected handleWebSocketConnectionEvents = ( - event: webSocketEvents.WebSocketConnectionEvent, + event: + | events.EventWebSocketConnection + | EventAll, ) => { - if (event instanceof webSocketEvents.WebSocketConnectionErrorEvent) { - this.dispatchEvent( - new webSocketEvents.WebSocketConnectionErrorEvent({ - detail: event.detail, - }), - ); - } else if (event instanceof webSocketEvents.WebSocketConnectionStopEvent) { - this.dispatchEvent(new webSocketEvents.WebSocketConnectionStopEvent()); - } else if ( - event instanceof webSocketEvents.WebSocketConnectionStreamEvent - ) { - this.dispatchEvent( - new webSocketEvents.WebSocketConnectionStreamEvent({ - detail: event.detail, - }), - ); - } else if (event instanceof webSocketEvents.WebSocketStreamDestroyEvent) { - this.dispatchEvent(new webSocketEvents.WebSocketStreamDestroyEvent()); + if (event instanceof EventAll) { + this.dispatchEvent(event.detail.clone()); } else { - utils.never(); + this.dispatchEvent(event.clone()); } }; @@ -135,7 +121,7 @@ class WebSocketServer extends EventTarget { this._port = address.port; this.logger.debug(`Listening on port ${this._port}`); this._host = address.address ?? '127.0.0.1'; - this.dispatchEvent(new webSocketEvents.WebSocketServerStartEvent()); + this.dispatchEvent(new events.EventWebSocketServerStart()); this.logger.info(`Started ${this.constructor.name}`); } @@ -180,7 +166,7 @@ class WebSocketServer extends EventTarget { this.server.off('error', this.errorHandler); this.server.on('request', this.requestHandler); - this.dispatchEvent(new webSocketEvents.WebSocketServerStopEvent()); + this.dispatchEvent(new events.EventWebSocketServerStop()); this.logger.info(`Stopped ${this.constructor.name}`); } @@ -249,30 +235,10 @@ class WebSocketServer extends EventTarget { // Handling connection events connection.addEventListener( - 'connectionError', - this.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'connectionStream', - this.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'streamDestroy', - this.handleWebSocketConnectionEvents, - ); - connection.addEventListener( - 'connectionStop', - (event) => { - connection.removeEventListener( - 'connectionError', - this.handleWebSocketConnectionEvents, - ); + events.EventWebSocketConnectionStop.name, + (event: events.EventWebSocketConnectionStop) => { connection.removeEventListener( - 'connectionStream', - this.handleWebSocketConnectionEvents, - ); - connection.removeEventListener( - 'streamDestroy', + EventDefault.name, this.handleWebSocketConnectionEvents, ); this.handleWebSocketConnectionEvents(event); @@ -280,8 +246,13 @@ class WebSocketServer extends EventTarget { { once: true }, ); + connection.addEventListener( + EventDefault.name, + this.handleWebSocketConnectionEvents, + ); + this.dispatchEvent( - new webSocketEvents.WebSocketServerConnectionEvent({ + new events.EventWebSocketServerConnection({ detail: connection, }), ); @@ -304,7 +275,7 @@ class WebSocketServer extends EventTarget { */ protected errorHandler = (e: Error) => { this.dispatchEvent( - new webSocketEvents.WebSocketServerErrorEvent({ + new events.EventWebSocketServerError({ detail: new errors.ErrorWebSocketServer( 'An error occured on the underlying server', { diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index e62f9a32..137e1260 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -2,6 +2,7 @@ import type { StreamCodeToReason, StreamId, StreamReasonToCode } from './types'; import type WebSocketConnection from './WebSocketConnection'; import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; +import { Evented } from '@matrixai/events'; import { fromVarInt, never, @@ -14,11 +15,10 @@ import * as errors from './errors'; import * as events from './events'; interface WebSocketStream extends CreateDestroy {} +interface WebSocketStream extends Evented {} @CreateDestroy() -class WebSocketStream - extends EventTarget - implements ReadableWritablePair -{ +@Evented() +class WebSocketStream implements ReadableWritablePair { public streamId: StreamId; public readable: ReadableStream; public writable: WritableStream; @@ -83,7 +83,6 @@ class WebSocketStream codeToReason: StreamCodeToReason; logger: Logger; }) { - super(); this.logger = logger; this.streamId = streamId; this.connection = connection; @@ -196,7 +195,9 @@ class WebSocketStream // So the connection will infinitely create streams with the same streamId when it receives the ERROR/CLOSE frame. // I'm dealing with this by just filtering out ERROR/CLOSE frames in the connection's onMessage handler, but there might be a better way to do this. this.connection.streamMap.delete(this.streamId); - this.dispatchEvent(new events.WebSocketStreamDestroyEvent()); + this.dispatchEvent( + new events.EventWebSocketStreamDestroy({ bubbles: true }), + ); this.logger.info(`Destroyed ${this.constructor.name}`); } diff --git a/src/events.ts b/src/events.ts index f27c9548..78aac4d0 100644 --- a/src/events.ts +++ b/src/events.ts @@ -1,125 +1,59 @@ import type WebSocketStream from './WebSocketStream'; import type WebSocketConnection from './WebSocketConnection'; +import { AbstractEvent } from '@matrixai/events'; + +abstract class EventWebSocket extends AbstractEvent {} // Client Events -abstract class WebSocketClientEvent extends Event {} - -class WebSocketClientDestroyEvent extends Event { - constructor(options?: EventInit) { - super('clientDestroy', options); - } -} - -class WebSocketClientErrorEvent extends Event { - public detail: Error; - constructor( - options: EventInit & { - detail: Error; - }, - ) { - super('clientError', options); - this.detail = options.detail; - } -} +abstract class EventWebSocketClient extends EventWebSocket {} + +class EventWebSocketClientDestroy extends EventWebSocketClient {} + +class EventWebSocketClientError extends EventWebSocketClient {} // Server events -abstract class WebSocketServerEvent extends Event {} - -class WebSocketServerConnectionEvent extends Event { - public detail: WebSocketConnection; - constructor( - options: EventInit & { - detail: WebSocketConnection; - }, - ) { - super('serverConnection', options); - this.detail = options.detail; - } -} - -class WebSocketServerStartEvent extends Event { - constructor(options?: EventInit) { - super('serverStart', options); - } -} - -class WebSocketServerStopEvent extends Event { - constructor(options?: EventInit) { - super('serverStop', options); - } -} - -class WebSocketServerErrorEvent extends Event { - public detail: Error; - constructor( - options: EventInit & { - detail: Error; - }, - ) { - super('serverError', options); - this.detail = options.detail; - } -} +abstract class EventWebSocketServer extends EventWebSocket {} + +class EventWebSocketServerConnection extends EventWebSocketServer {} + +class EventWebSocketServerStart extends EventWebSocketServer {} + +class EventWebSocketServerStop extends EventWebSocketServer {} + +class EventWebSocketServerError extends EventWebSocketServer {} // Connection events -abstract class WebSocketConnectionEvent extends Event {} - -class WebSocketConnectionStreamEvent extends WebSocketConnectionEvent { - public detail: WebSocketStream; - constructor( - options: EventInit & { - detail: WebSocketStream; - }, - ) { - super('connectionStream', options); - this.detail = options.detail; - } -} - -class WebSocketConnectionStopEvent extends WebSocketConnectionEvent { - constructor(options?: EventInit) { - super('connectionStop', options); - } -} - -class WebSocketConnectionErrorEvent extends WebSocketConnectionEvent { - public detail: Error; - constructor( - options: EventInit & { - detail: Error; - }, - ) { - super('connectionError', options); - this.detail = options.detail; - } -} +abstract class EventWebSocketConnection extends EventWebSocket {} + +class EventWebSocketConnectionStream extends EventWebSocketConnection {} + +class EventWebSocketConnectionStop extends EventWebSocketConnection {} + +class EventWebSocketConnectionError extends EventWebSocketConnection {} // Stream events -abstract class WebSocketStreamEvent extends Event {} +abstract class EventWebSocketStream extends EventWebSocket {} -class WebSocketStreamDestroyEvent extends WebSocketStreamEvent { - constructor(options?: EventInit) { - super('streamDestroy', options); - } -} +class EventWebSocketStreamDestroy extends EventWebSocketStream {} export { - WebSocketClientEvent, - WebSocketClientErrorEvent, - WebSocketClientDestroyEvent, - WebSocketServerEvent, - WebSocketServerConnectionEvent, - WebSocketServerStartEvent, - WebSocketServerStopEvent, - WebSocketServerErrorEvent, - WebSocketConnectionEvent, - WebSocketConnectionStreamEvent, - WebSocketConnectionStopEvent, - WebSocketConnectionErrorEvent, - WebSocketStreamEvent, - WebSocketStreamDestroyEvent, + EventWebSocket, + EventWebSocketClient, + EventWebSocketClientError, + EventWebSocketClientDestroy, + EventWebSocketServer, + EventWebSocketServerConnection, + EventWebSocketServerStart, + EventWebSocketServerStop, + EventWebSocketServerError, + EventWebSocketConnection, + EventWebSocketConnectionStream, + EventWebSocketConnectionStop, + EventWebSocketConnectionError, + EventWebSocketStream, + EventWebSocketStreamDestroy, }; diff --git a/tests/WebSocketClient.test.ts b/tests/WebSocketClient.test.ts index 39fb70ff..acba4ad8 100644 --- a/tests/WebSocketClient.test.ts +++ b/tests/WebSocketClient.test.ts @@ -1,5 +1,5 @@ -import type { WebSocketConnectionStreamEvent } from '@/events'; import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; +import { EventWebSocketConnectionStream } from '@/events'; import { serverDefault } from '@/config'; import WebSocketClient from '@/WebSocketClient'; import WebSocketServer from '@/WebSocketServer'; @@ -30,8 +30,8 @@ describe(WebSocketClient.name, () => { await server.start(); server.addEventListener( - 'connectionStream', - async (event: WebSocketConnectionStreamEvent) => { + EventWebSocketConnectionStream.name, + async (event: EventWebSocketConnectionStream) => { // Await event.detail.readable.getReader().read(); }, ); diff --git a/tests/WebSocketServer.test.ts b/tests/WebSocketServer.test.ts index b99d18e2..7df0e650 100644 --- a/tests/WebSocketServer.test.ts +++ b/tests/WebSocketServer.test.ts @@ -1,6 +1,7 @@ -import type { WebSocketServerConnectionEvent } from '@/events'; import { WebSocket } from 'ws'; +import { EventWebSocketServerConnection } from '@/events'; import WebSocketServer from '@/WebSocketServer'; +import { promise } from '@/utils'; import * as testsUtils from './utils'; describe('test', () => { @@ -16,8 +17,8 @@ describe('test', () => { port: 3000, }); server.addEventListener( - 'serverConnection', - async (event: WebSocketServerConnectionEvent) => { + EventWebSocketServerConnection.name, + async (event: EventWebSocketServerConnection) => { const connection = event.detail; const stream = await connection.streamNew('bidi'); const writer = stream.writable.getWriter(); diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index cd1f60dc..eff1fffc 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -50,7 +50,7 @@ jest.mock('@/WebSocketConnection', () => { logger: logger2, }); instance.connectedConnection!.dispatchEvent( - new events.WebSocketConnectionStreamEvent({ + new events.EventWebSocketConnectionStream({ detail: stream, }), ); @@ -76,7 +76,10 @@ describe(WebSocketStream.name, () => { (connection1 as any).connectTo(connection2); }); - async function createStreamPair(connection1, connection2) { + async function createStreamPair( + connection1: WebSocketConnection, + connection2: WebSocketConnection, + ) { const stream1 = await WebSocketStream.createWebSocketStream({ streamId: streamIdCounter as StreamId, bufferSize: config.clientDefault.streamBufferSize, @@ -85,8 +88,8 @@ describe(WebSocketStream.name, () => { }); const createStream2Prom = promise(); connection2.addEventListener( - 'connectionStream', - (e: events.WebSocketConnectionStreamEvent) => { + events.EventWebSocketConnectionStream.name, + (e: events.EventWebSocketConnectionStream) => { createStream2Prom.resolveP(e.detail); }, { once: true }, @@ -169,7 +172,11 @@ describe(WebSocketStream.name, () => { ); testProp( 'multiple writes within buffer size', - [fc.array(fc.uint8Array({maxLength: config.clientDefault.streamBufferSize}))], + [ + fc.array( + fc.uint8Array({ maxLength: config.clientDefault.streamBufferSize }), + ), + ], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, From f4d399fccfdaad2e5bc43615a5c57007bed2d5c5 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 14:38:47 +1000 Subject: [PATCH 040/149] feat: simultaneous writing tests for WebSocketStream --- tests/WebSocketStream.test.ts | 127 +++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 10 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index eff1fffc..29e21ff3 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -8,6 +8,9 @@ import { promise, StreamType } from '@/utils'; import * as config from '@/config'; import * as testUtils from './utils'; +// Smaller buffer size for the sake of testing +const STREAM_BUFFER_SIZE = 64; + const logger1 = new Logger('stream 1', LogLevel.WARN, [ new StreamHandler( formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, @@ -45,7 +48,7 @@ jest.mock('@/WebSocketConnection', () => { } stream = await WebSocketStream.createWebSocketStream({ streamId, - bufferSize: config.clientDefault.streamBufferSize, + bufferSize: STREAM_BUFFER_SIZE, connection: instance.connectedConnection!, logger: logger2, }); @@ -82,7 +85,7 @@ describe(WebSocketStream.name, () => { ) { const stream1 = await WebSocketStream.createWebSocketStream({ streamId: streamIdCounter as StreamId, - bufferSize: config.clientDefault.streamBufferSize, + bufferSize: STREAM_BUFFER_SIZE, connection: connection1, logger: logger1, }); @@ -100,7 +103,7 @@ describe(WebSocketStream.name, () => { } testProp( 'single write within buffer size', - [fc.uint8Array({ maxLength: config.clientDefault.streamBufferSize })], + [fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE })], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, @@ -136,7 +139,7 @@ describe(WebSocketStream.name, () => { ); testProp( 'single write outside buffer size', - [fc.uint8Array()], + [fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 })], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, @@ -172,11 +175,7 @@ describe(WebSocketStream.name, () => { ); testProp( 'multiple writes within buffer size', - [ - fc.array( - fc.uint8Array({ maxLength: config.clientDefault.streamBufferSize }), - ), - ], + [fc.array(fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }))], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, @@ -216,7 +215,54 @@ describe(WebSocketStream.name, () => { ); testProp( 'multiple writes outside buffer size', - [fc.array(fc.uint8Array())], + [fc.array(fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }))], + async (data) => { + const [stream1, stream2] = await createStreamPair( + connection1, + connection2, + ); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + + const writer = stream2Writable.getWriter(); + const reader = stream1Readable.getReader(); + + const writeProm = (async () => { + for (const chunk of data) { + await writer.write(chunk); + } + await writer.close(); + })(); + + const readChunks: Array = []; + const readProm = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + })(); + + await Promise.all([writeProm, readProm]); + + expect(testUtils.concatUInt8Array(...readChunks)).toEqual( + testUtils.concatUInt8Array(...data), + ); + + await stream1.destroy(); + }, + ); + testProp( + 'multiple writes within and outside buffer size', + [ + fc.array( + fc.oneof( + fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }), + fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }), + ), + ), + ], async (data) => { const [stream1, stream2] = await createStreamPair( connection1, @@ -254,4 +300,65 @@ describe(WebSocketStream.name, () => { await stream1.destroy(); }, ); + testProp( + 'simultaneous writes', + [ + fc.array( + fc.oneof( + fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }), + fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }), + ), + ), + fc.array( + fc.oneof( + fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }), + fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }), + ), + ), + ], + async (...data) => { + const streams = await createStreamPair( + connection1, + connection2, + ); + + const readProms: Array>> = []; + const writeProms: Array> = []; + + for (const [i, stream] of streams.entries()) { + const reader = stream.readable.getReader(); + const writer = stream.writable.getWriter(); + const writeProm = (async () => { + for (const chunk of data[i]) { + await writer.write(chunk); + } + await writer.close(); + })(); + const readProm = (async () => { + const readChunks: Array = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + readChunks.push(value); + } + return readChunks; + })(); + readProms.push(readProm); + writeProms.push(writeProm); + } + await Promise.all(writeProms); + const readResults = await Promise.all(readProms); + + data.reverse(); + for (const [i, readResult] of readResults.entries()) { + expect(testUtils.concatUInt8Array(...readResult)).toEqual( + testUtils.concatUInt8Array(...data[i]), + ); + } + + for (const stream of streams) { + await stream.destroy(); + } + }, + ); }); From 8b4649586c4171599a23bd12f3ac235d373e262c Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:18:26 +1000 Subject: [PATCH 041/149] chore: renamed `StreamType` to `StreamMessageType` --- src/WebSocketConnection.ts | 6 ++--- src/WebSocketServer.ts | 3 --- src/WebSocketStream.ts | 42 +++++++++++++++++------------------ src/utils/utils.ts | 4 ++-- tests/WebSocketStream.test.ts | 6 ++--- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index f792d05f..d14d40a7 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -23,7 +23,7 @@ import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; import { Evented } from '@matrixai/events'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; -import { fromStreamId, promise, StreamType, toStreamId } from './utils'; +import { fromStreamId, promise, StreamMessageType, toStreamId } from './utils'; import * as events from './events'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -163,8 +163,8 @@ class WebSocketConnection extends EventTarget { if (stream == null) { const messageType = message.at(0); if ( - messageType === StreamType.CLOSE || - messageType === StreamType.ERROR + messageType === StreamMessageType.CLOSE || + messageType === StreamMessageType.ERROR ) { return; } diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index 0efa964d..b8911f48 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -24,9 +24,6 @@ import WebSocketConnectionMap from './WebSocketConnectionMap'; * Otherwise, errors will just be ignored. * * Events: - * - serverStop - * - serverError - * - serverConnection * - connectionStream - when new stream is created from a connection * - connectionError - connection error event * - connectionDestroy - when connection is destroyed diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 137e1260..4f888a87 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -7,7 +7,7 @@ import { fromVarInt, never, promise, - StreamType, + StreamMessageType, StreamShutdown, toVarInt, } from './utils'; @@ -104,7 +104,7 @@ class WebSocketStream implements ReadableWritablePair { return; } // Send ACK on every read as there will be more usable space on the buffer. - await this.streamSend(StreamType.ACK, controller.desiredSize!); + await this.streamSend(StreamMessageType.ACK, controller.desiredSize!); }, cancel: async (reason) => { this.logger.debug(`readable aborted with [${reason.message}]`); @@ -146,7 +146,7 @@ class WebSocketStream implements ReadableWritablePair { } // Decrement the desired size by the amount of bytes written this.writableDesiredSize -= bytesWritten; - await this.streamSend(StreamType.DATA, data); + await this.streamSend(StreamMessageType.DATA, data); if (isChunkable) { await writeHandler(chunk.subarray(bytesWritten), controller); @@ -207,7 +207,7 @@ class WebSocketStream implements ReadableWritablePair { * @param payloadSize - The number of bytes that the receiver can accept. */ protected async streamSend( - type: StreamType.ACK, + type: StreamMessageType.ACK, payloadSize: number, ): Promise; /** @@ -216,7 +216,7 @@ class WebSocketStream implements ReadableWritablePair { * @param data - The payload to send. */ protected async streamSend( - type: StreamType.DATA, + type: StreamMessageType.DATA, data: Uint8Array, ): Promise; /** @@ -225,7 +225,7 @@ class WebSocketStream implements ReadableWritablePair { * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. */ protected async streamSend( - type: StreamType.ERROR, + type: StreamMessageType.ERROR, shutdown: StreamShutdown, code: bigint, ): Promise; @@ -235,28 +235,28 @@ class WebSocketStream implements ReadableWritablePair { * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. */ protected async streamSend( - type: StreamType.CLOSE, + type: StreamMessageType.CLOSE, shutdown: StreamShutdown, ): Promise; protected async streamSend( - type: StreamType, + type: StreamMessageType, data_?: Uint8Array | number, code?: bigint, ): Promise { let data: Uint8Array | undefined; - if (type === StreamType.ACK && typeof data_ === 'number') { + if (type === StreamMessageType.ACK && typeof data_ === 'number') { data = new Uint8Array(4); const dv = new DataView(data.buffer); dv.setUint32(0, data_, false); - } else if (type === StreamType.DATA) { + } else if (type === StreamMessageType.DATA) { data = data_ as Uint8Array; - } else if (type === StreamType.ERROR) { + } else if (type === StreamMessageType.ERROR) { const errorCode = fromVarInt(code!); data = new Uint8Array(1 + errorCode.length); const dv = new DataView(data.buffer); dv.setUint8(0, data_ as StreamShutdown); data.set(errorCode, 1); - } else if (type === StreamType.CLOSE) { + } else if (type === StreamMessageType.CLOSE) { data = new Uint8Array([data_ as StreamShutdown]); } else { never(); @@ -286,10 +286,10 @@ class WebSocketStream implements ReadableWritablePair { }), ); } - const type = message[0] as StreamType; + const type = message[0] as StreamMessageType; const data = message.subarray(1); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); - if (type === StreamType.ACK) { + if (type === StreamMessageType.ACK) { try { const bufferSize = dv.getUint32(0, false); this.writableDesiredSize = bufferSize; @@ -309,7 +309,7 @@ class WebSocketStream implements ReadableWritablePair { ), ); } - } else if (type === StreamType.DATA) { + } else if (type === StreamMessageType.DATA) { if (this._readableEnded) { return; } @@ -324,12 +324,12 @@ class WebSocketStream implements ReadableWritablePair { return; } this.readableController.enqueue(data); - } else if (type === StreamType.ERROR || type === StreamType.CLOSE) { + } else if (type === StreamMessageType.ERROR || type === StreamMessageType.CLOSE) { try { const shutdown = dv.getUint8(0) as StreamShutdown; let isError = false; let reason: any; - if (type === StreamType.ERROR) { + if (type === StreamMessageType.ERROR) { isError = true; const errorCode = toVarInt(data.subarray(1)).data; reason = await this.codeToReason('recv', errorCode); @@ -398,10 +398,10 @@ class WebSocketStream implements ReadableWritablePair { // Shutdown the write side of the other stream if (isError) { const code = await this.reasonToCode('send', reason); - await this.streamSend(StreamType.ERROR, StreamShutdown.Write, code); + await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Write, code); this.readableController.error(reason); } else { - await this.streamSend(StreamType.CLOSE, StreamShutdown.Write); + await this.streamSend(StreamMessageType.CLOSE, StreamShutdown.Write); } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); @@ -428,10 +428,10 @@ class WebSocketStream implements ReadableWritablePair { // Shutdown the read side of the other stream if (isError) { const code = await this.reasonToCode('send', reason); - await this.streamSend(StreamType.ERROR, StreamShutdown.Read, code); + await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Read, code); this.writableController.error(reason); } else { - await this.streamSend(StreamType.CLOSE, StreamShutdown.Read); + await this.streamSend(StreamMessageType.CLOSE, StreamShutdown.Read); } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 93f3e836..74559f30 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -98,7 +98,7 @@ function fromVarInt(varInt: bigint): Uint8Array { const fromStreamId = fromVarInt as (streamId: StreamId) => Uint8Array; const toStreamId = toVarInt as (array: Uint8Array) => Parsed; -enum StreamType { +enum StreamMessageType { DATA = 0, ACK = 1, ERROR = 2, @@ -117,6 +117,6 @@ export { fromVarInt, toStreamId, fromStreamId, - StreamType, + StreamMessageType, StreamShutdown, }; diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 29e21ff3..5795fcb8 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -4,7 +4,7 @@ import { fc, testProp } from '@fast-check/jest'; import WebSocketStream from '@/WebSocketStream'; import WebSocketConnection from '@/WebSocketConnection'; import * as events from '@/events'; -import { promise, StreamType } from '@/utils'; +import { promise, StreamMessageType } from '@/utils'; import * as config from '@/config'; import * as testUtils from './utils'; @@ -41,8 +41,8 @@ jest.mock('@/WebSocketConnection', () => { let stream = instance.connectedConnection!.streamMap.get(streamId); if (stream == null) { if ( - data.at(0) === StreamType.CLOSE || - data.at(0) === StreamType.ERROR + data.at(0) === StreamMessageType.CLOSE || + data.at(0) === StreamMessageType.ERROR ) { return; } From baf00ca3c1ceef7cb653244e27c5d791905b9810 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:36:11 +1000 Subject: [PATCH 042/149] fix: renamed ErrorWebSocketStream errors --- src/WebSocketStream.ts | 45 +++++++++++++++++++++++++++++++++--------- src/errors.ts | 8 ++++---- src/utils/utils.ts | 7 +++++++ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 4f888a87..6058e67f 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -7,6 +7,7 @@ import { fromVarInt, never, promise, + StreamErrorCode, StreamMessageType, StreamShutdown, toVarInt, @@ -281,7 +282,7 @@ class WebSocketStream implements ReadableWritablePair { this.logger.debug(`received empty message, closing stream`); await this.signalReadableEnd( true, - new errors.ErrorWebSocketStreamReaderParse('empty message', { + new errors.ErrorWebSocketStreamReadableParse('empty message', { cause: new RangeError(), }), ); @@ -301,7 +302,7 @@ class WebSocketStream implements ReadableWritablePair { this.logger.debug(`received malformed ACK, closing stream`); await this.signalReadableEnd( true, - new errors.ErrorWebSocketStreamReaderParse( + new errors.ErrorWebSocketStreamReadableParse( 'ACK message did not contain a valid buffer size', { cause: e, @@ -319,7 +320,7 @@ class WebSocketStream implements ReadableWritablePair { ) { await this.signalReadableEnd( true, - new errors.ErrorWebSocketStreamReaderBufferOverload(), + new errors.ErrorWebSocketStreamReadableBufferOverload(), ); return; } @@ -327,12 +328,20 @@ class WebSocketStream implements ReadableWritablePair { } else if (type === StreamMessageType.ERROR || type === StreamMessageType.CLOSE) { try { const shutdown = dv.getUint8(0) as StreamShutdown; - let isError = false; + let isError = type === StreamMessageType.ERROR; let reason: any; if (type === StreamMessageType.ERROR) { - isError = true; const errorCode = toVarInt(data.subarray(1)).data; - reason = await this.codeToReason('recv', errorCode); + switch (errorCode) { + case BigInt(StreamErrorCode.ErrorReadableStreamParse): + reason = new errors.ErrorWebSocketStreamReadableParse('receiver was unable to parse a sent message'); + break; + case BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow): + reason = new errors.ErrorWebSocketStreamReadableBufferOverload('receiver was unable to accept a sent message'); + break; + default: + reason = await this.codeToReason('recv', errorCode); + } } if (shutdown === StreamShutdown.Read) { if (this._readableEnded) { @@ -351,7 +360,7 @@ class WebSocketStream implements ReadableWritablePair { } catch (e) { await this.signalReadableEnd( true, - new errors.ErrorWebSocketStreamReaderParse( + new errors.ErrorWebSocketStreamReadableParse( 'ERROR/CLOSE message did not contain a valid payload', { cause: e, @@ -397,7 +406,16 @@ class WebSocketStream implements ReadableWritablePair { this._readableEnded = true; // Shutdown the write side of the other stream if (isError) { - const code = await this.reasonToCode('send', reason); + let code: bigint; + if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { + code = BigInt(StreamErrorCode.ErrorReadableStreamParse); + } + else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { + code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow); + } + else { + code = await this.reasonToCode('send', reason); + } await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Write, code); this.readableController.error(reason); } else { @@ -427,7 +445,16 @@ class WebSocketStream implements ReadableWritablePair { this.writableDesiredSizeProm.resolveP(); // Shutdown the read side of the other stream if (isError) { - const code = await this.reasonToCode('send', reason); + let code: bigint; + if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { + code = BigInt(StreamErrorCode.ErrorReadableStreamParse); + } + else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { + code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow); + } + else { + code = await this.reasonToCode('send', reason); + } await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Read, code); this.writableController.error(reason); } else { diff --git a/src/errors.ts b/src/errors.ts index 89324444..511b57ce 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -71,11 +71,11 @@ class ErrorWebSocketStreamReader extends ErrorWebSocketStream { static description = 'WebSocket Stream readable error'; } -class ErrorWebSocketStreamReaderParse extends ErrorWebSocketStreamReader { +class ErrorWebSocketStreamReadableParse extends ErrorWebSocketStreamReader { static description = 'WebSocket Stream readable message parse failed'; } -class ErrorWebSocketStreamReaderBufferOverload extends ErrorWebSocket { +class ErrorWebSocketStreamReadableBufferOverload extends ErrorWebSocket { static description = 'WebSocket Stream readable buffer has overloaded'; } @@ -113,8 +113,8 @@ export { ErrorWebSocketConnectionSocket, ErrorWebSocketStream, ErrorWebSocketStreamReader, - ErrorWebSocketStreamReaderParse, - ErrorWebSocketStreamReaderBufferOverload, + ErrorWebSocketStreamReadableParse, + ErrorWebSocketStreamReadableBufferOverload, ErrorWebSocketStreamDestroyed, ErrorWebSocketStreamClose, ErrorWebSocketStreamCancel, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 74559f30..afe51a7b 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -110,6 +110,12 @@ enum StreamShutdown { Write = 1, } +enum StreamErrorCode { + Unknown = 0, + ErrorReadableStreamParse = 1, + ErrorReadableStreamBufferOverflow = 2, +} + export { never, promise, @@ -119,4 +125,5 @@ export { fromStreamId, StreamMessageType, StreamShutdown, + StreamErrorCode }; From 44ddbd61d1c30053c0d0584a3c59d393d08f37dc Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:47:03 +1000 Subject: [PATCH 043/149] fix: made a VarInt type that SteamId is --- src/WebSocketStream.ts | 22 +++++++++++----------- src/types.ts | 10 ++++++++-- src/utils/utils.ts | 8 ++++---- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 6058e67f..cf4b83fe 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,4 +1,4 @@ -import type { StreamCodeToReason, StreamId, StreamReasonToCode } from './types'; +import type { StreamCodeToReason, StreamId, StreamReasonToCode, VarInt } from './types'; import type WebSocketConnection from './WebSocketConnection'; import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; @@ -228,7 +228,7 @@ class WebSocketStream implements ReadableWritablePair { protected async streamSend( type: StreamMessageType.ERROR, shutdown: StreamShutdown, - code: bigint, + code: VarInt, ): Promise; /** * Send a CLOSE frame with a payload on the stream. @@ -242,7 +242,7 @@ class WebSocketStream implements ReadableWritablePair { protected async streamSend( type: StreamMessageType, data_?: Uint8Array | number, - code?: bigint, + code?: VarInt, ): Promise { let data: Uint8Array | undefined; if (type === StreamMessageType.ACK && typeof data_ === 'number') { @@ -406,15 +406,15 @@ class WebSocketStream implements ReadableWritablePair { this._readableEnded = true; // Shutdown the write side of the other stream if (isError) { - let code: bigint; + let code: VarInt; if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { - code = BigInt(StreamErrorCode.ErrorReadableStreamParse); + code = BigInt(StreamErrorCode.ErrorReadableStreamParse) as VarInt; } else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { - code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow); + code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow) as VarInt; } else { - code = await this.reasonToCode('send', reason); + code = await this.reasonToCode('send', reason) as VarInt; } await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Write, code); this.readableController.error(reason); @@ -445,15 +445,15 @@ class WebSocketStream implements ReadableWritablePair { this.writableDesiredSizeProm.resolveP(); // Shutdown the read side of the other stream if (isError) { - let code: bigint; + let code: VarInt; if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { - code = BigInt(StreamErrorCode.ErrorReadableStreamParse); + code = BigInt(StreamErrorCode.ErrorReadableStreamParse) as VarInt; } else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { - code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow); + code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow) as VarInt; } else { - code = await this.reasonToCode('send', reason); + code = await this.reasonToCode('send', reason) as VarInt; } await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Read, code); this.writableController.error(reason); diff --git a/src/types.ts b/src/types.ts index 2e621f21..94c56352 100644 --- a/src/types.ts +++ b/src/types.ts @@ -27,9 +27,14 @@ type PromiseDeconstructed = { type ConnectionId = Opaque<'ConnectionId', number>; /** - * StreamId is a 62 bit unsigned integer + * VarInt is a 62 bit unsigned integer */ -type StreamId = Opaque<'StreamId', bigint>; +type VarInt = Opaque<'VarInt', bigint>; + +/** + * StreamId is a VarInt + */ +type StreamId = VarInt; /** * Host is always an IP address @@ -103,6 +108,7 @@ export type { Callback, PromiseDeconstructed, ConnectionId, + VarInt, StreamId, Host, Hostname, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index afe51a7b..6bc79d16 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,5 +1,5 @@ import type { PromiseDeconstructed } from './types'; -import type { Parsed, StreamId } from '@/types'; +import type { Parsed, StreamId, VarInt } from '@/types'; import * as errors from '../errors'; function never(message?: string): never { @@ -22,7 +22,7 @@ function promise(): PromiseDeconstructed { }; } -function toVarInt(array: Uint8Array): Parsed { +function toVarInt(array: Uint8Array): Parsed { let streamId: bigint; // Get header and prefix @@ -57,12 +57,12 @@ function toVarInt(array: Uint8Array): Parsed { break; } return { - data: streamId!, + data: streamId! as VarInt, remainder: array.subarray(readBytes), }; } -function fromVarInt(varInt: bigint): Uint8Array { +function fromVarInt(varInt: VarInt): Uint8Array { let array: Uint8Array; let dv: DataView; let prefixMask = 0; From 3740a78babdd9b56387725f708a2cc0a2fed703d Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:28:34 +1000 Subject: [PATCH 044/149] chore: upgraded js-async-init to 1.9.1 for implicit lifecycle events --- package-lock.json | 25 +++--- package.json | 4 +- src/WebSocketClient.ts | 74 +++++++++++------ src/WebSocketConnection.ts | 166 ++++++++++++++----------------------- src/WebSocketServer.ts | 70 +++++++++------- src/WebSocketStream.ts | 16 ++-- src/events.ts | 22 +++++ 7 files changed, 202 insertions(+), 175 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0d3d0521..f9f4ccbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,11 @@ "license": "Apache-2.0", "dependencies": { "@matrixai/async-cancellable": "^1.1.1", - "@matrixai/async-init": "^1.8.4", + "@matrixai/async-init": "^1.9.1", "@matrixai/async-locks": "^4.0.0", "@matrixai/contexts": "^1.2.0", "@matrixai/errors": "^1.1.7", - "@matrixai/events": "^3.0.0", + "@matrixai/events": "^3.0.2", "@matrixai/logger": "^3.1.0", "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", @@ -1662,12 +1662,13 @@ "integrity": "sha512-f0yxu7dHwvffZ++7aCm2WIcCJn18uLcOTdCCwEA3R3KVHYE3TG/JNoTWD9/mqBkAV1AI5vBfJzg27WnF9rOUXQ==" }, "node_modules/@matrixai/async-init": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/@matrixai/async-init/-/async-init-1.8.4.tgz", - "integrity": "sha512-33cGC7kHTs9KKwMHJA5d5XURWhx3QUq7lLxPEXLoVfWdTHixcWNvtfshAOso0hbRfx1P3ZSgsb+ZHaIASHhWfg==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@matrixai/async-init/-/async-init-1.9.1.tgz", + "integrity": "sha512-/nktToDllAKPZUhTWrAu1ux8r0rfV6kqGj+GEeU4zwTamdqaVblp4RhfyA3omLeIB3LosHaHsAs3tKKtRG9Rww==", "dependencies": { "@matrixai/async-locks": "^4.0.0", - "@matrixai/errors": "^1.1.7" + "@matrixai/errors": "^1.2.0", + "@matrixai/events": "^3.0.2" } }, "node_modules/@matrixai/async-locks": { @@ -1694,17 +1695,17 @@ } }, "node_modules/@matrixai/errors": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/@matrixai/errors/-/errors-1.1.7.tgz", - "integrity": "sha512-WD6MrlfgtNSTfXt60lbMgwasS5T7bdRgH4eYSOxV+KWngqlkEij9EoDt5LwdvcMD1yuC33DxPTnH4Xu2XV3nMw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@matrixai/errors/-/errors-1.2.0.tgz", + "integrity": "sha512-eZHPHFla5GFmi0O0yGgbtkca+ZjwpDbMz+60NC3y+DzQq6BMoe4gHmPjDalAHTxyxv0+Q+AWJTuV8Ows+IqBfQ==", "dependencies": { "ts-custom-error": "3.2.2" } }, "node_modules/@matrixai/events": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@matrixai/events/-/events-3.0.0.tgz", - "integrity": "sha512-qg1pXV3wcGi3Gzy8vkQM/hiTyeVPrR3gN2N7gB/2JkX2L2fGmPf1xrJCizK7KgtUZOvjcKwjN+uZzspu3aGDaw==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@matrixai/events/-/events-3.0.2.tgz", + "integrity": "sha512-U2gb0IQadncd3ROijE8PPOWGJibKyeY87YKIZiCoWNzof3EIOUck3kC2DtLzBSpLBjbqpRC6K/dk48dnRzG9Qg==", "engines": { "node": ">=19.0.0" } diff --git a/package.json b/package.json index 1cb7084e..bfe06184 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,11 @@ }, "dependencies": { "@matrixai/async-cancellable": "^1.1.1", - "@matrixai/async-init": "^1.8.4", + "@matrixai/async-init": "^1.9.1", "@matrixai/async-locks": "^4.0.0", "@matrixai/contexts": "^1.2.0", "@matrixai/errors": "^1.1.7", - "@matrixai/events": "^3.0.0", + "@matrixai/events": "^3.0.2", "@matrixai/logger": "^3.1.0", "@matrixai/resources": "^1.1.5", "@matrixai/timer": "^1.1.1", diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 33ac65aa..90df449c 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -3,7 +3,7 @@ import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; import { Validator } from 'ip-num'; -import { EventAll } from '@matrixai/events'; +import { AbstractEvent, EventAll, EventDefault } from '@matrixai/events'; import * as errors from './errors'; import WebSocketConnection from './WebSocketConnection'; import WebSocketConnectionMap from './WebSocketConnectionMap'; @@ -12,7 +12,27 @@ import * as events from './events'; import * as utils from './utils'; interface WebSocketClient extends createDestroy.CreateDestroy {} -@createDestroy.CreateDestroy() +/** + * You must provide an error handler `addEventListener('error')`. + * Otherwise, errors will just be ignored. + * + * Events: + * - {@link events.EventWebSocketClientDestroy} + * - {@link events.EventWebSocketClientDestroyed} + * - {@link events.EventWebSocketClientError} + * - {@link events.EventWebSocketConnectionStream} + * - {@link events.EventWebSocketConnectionStart} + * - {@link events.EventWebSocketConnectionStarted} + * - {@link events.EventWebSocketConnectionStop} + * - {@link events.EventWebSocketConnectionStopped} + * - {@link events.EventWebSocketConnectionError} - can occur due to a timeout too + * - {@link events.EventWebSocketStreamDestroy} + * - {@link events.EventWebSocketStreamDestroyed} + */ +@createDestroy.CreateDestroy({ + eventDestroy: events.EventWebSocketClientDestroy, + eventDestroyed: events.EventWebSocketClientDestroyed, +}) class WebSocketClient extends EventTarget { protected logger: Logger; @@ -23,18 +43,18 @@ class WebSocketClient extends EventTarget { protected address: string; protected handleEventWebSocketConnection = async ( - event_: EventAll | Event, + event_: EventAll | AbstractEvent, ) => { - let event: Event; + let event: AbstractEvent; if (event_ instanceof EventAll) { event = event_.detail; } else { event = event_; } - if (event instanceof events.EventWebSocketConnectionError) { - this.dispatchEvent( - (event as events.EventWebSocketConnectionError).clone(), - ); + + this.dispatchEvent(event.clone()); + + if (event instanceof events.EventWebSocketConnectionStopped) { try { // Force destroy means don't destroy gracefully await this.destroy({ @@ -47,7 +67,11 @@ class WebSocketClient extends EventTarget { }), ); } - } else if (event instanceof events.EventWebSocketConnectionStop) { + } + else if (event instanceof events.EventWebSocketConnectionError) { + this.dispatchEvent( + (event as events.EventWebSocketConnectionError).clone(), + ); try { // Force destroy means don't destroy gracefully await this.destroy({ @@ -60,14 +84,6 @@ class WebSocketClient extends EventTarget { }), ); } - } else if (event instanceof events.EventWebSocketConnectionStream) { - this.dispatchEvent( - (event as events.EventWebSocketConnectionStream).clone(), - ); - } else if (event instanceof events.EventWebSocketStreamDestroy) { - this.dispatchEvent((event as events.EventWebSocketStreamDestroy).clone()); - } else { - utils.never(); } }; @@ -137,7 +153,7 @@ class WebSocketClient extends EventTarget { }); const connectionId = client.connectionMap.allocateId(); - const connection = await WebSocketConnection.createWebSocketConnection( + const connection = new WebSocketConnection( { type: 'client', connectionId, @@ -149,13 +165,24 @@ class WebSocketClient extends EventTarget { socket: webSocket, verifyCallback, client: client, - }, - { - timer: wsConfig.connectTimeoutTime, - }, + } ); + await connection.start( { + timer: wsConfig.connectTimeoutTime, + }); + connection.addEventListener( + events.EventWebSocketConnectionStopped.name, + (event: events.EventWebSocketConnectionStopped) => { + connection.removeEventListener( + EventAll.name, + client.handleEventWebSocketConnection, + ); + client.handleEventWebSocketConnection(event); + }, + { once: true } + ) connection.addEventListener( - EventAll.name, + EventDefault.name, client.handleEventWebSocketConnection, ); client._connection = connection; @@ -185,7 +212,6 @@ class WebSocketClient extends EventTarget { force, }); } - this.dispatchEvent(new events.EventWebSocketClientDestroy()); this.logger.info(`Destroyed ${this.constructor.name}`); } } diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index d14d40a7..75f74526 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -20,7 +20,7 @@ import Logger from '@matrixai/logger'; import * as ws from 'ws'; import { Timer } from '@matrixai/timer'; import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; -import { Evented } from '@matrixai/events'; +import { EventAll, EventDefault, Evented } from '@matrixai/events'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; import { fromStreamId, promise, StreamMessageType, toStreamId } from './utils'; @@ -28,23 +28,30 @@ import * as events from './events'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); +interface WebSocketConnection extends startStop.StartStop {} /** * Think of this as equivalent to `net.Socket`. * This is one-to-one with the ws.WebSocket. * Errors here are emitted to the connection only. * Not to the server. * - * Events (events are executed post-facto): - * - connectionStream - * - connectionStop - * - connectionError - can occur due to a timeout too - * - streamDestroy + * Events: + * - {@link events.EventWebSocketConnectionStream} + * - {@link events.EventWebSocketConnectionStart} + * - {@link events.EventWebSocketConnectionStarted} + * - {@link events.EventWebSocketConnectionStop} + * - {@link events.EventWebSocketConnectionStopped} + * - {@link events.EventWebSocketConnectionError} - can occur due to a timeout too + * - {@link events.EventWebSocketStreamDestroy} + * - {@link events.EventWebSocketStreamDestroyed} */ -interface WebSocketConnection extends startStop.StartStop {} -interface WebSocketConnection extends Evented {} -@startStop.StartStop() -@Evented() -class WebSocketConnection extends EventTarget { +@startStop.StartStop({ + eventStart: events.EventWebSocketConnectionStart, + eventStarted: events.EventWebSocketConnectionStarted, + eventStop: events.EventWebSocketConnectionStop, + eventStopped: events.EventWebSocketConnectionStopped, +}) +class WebSocketConnection { /** * This determines when it is a client or server connection. */ @@ -125,10 +132,15 @@ class WebSocketConnection extends EventTarget { /** * Bubble up stream destroy event */ - protected handleEventWebSocketStreamDestroy = ( - event: events.EventWebSocketStreamDestroy, + protected handleEventWebSocketStream = ( + event: EventAll | EventDefault | events.EventWebSocketStream, ) => { - this.dispatchEvent(event.clone()); + if (event instanceof EventAll || event instanceof EventDefault) { + this.dispatchEvent(event.detail.clone()); + } + else { + this.dispatchEvent(event.clone()); + } }; /** @@ -176,7 +188,12 @@ class WebSocketConnection extends EventTarget { }); stream.addEventListener( events.EventWebSocketStreamDestroy.name, - this.handleEventWebSocketStreamDestroy, + this.handleEventWebSocketStream, + { once: true }, + ); + stream.addEventListener( + events.EventWebSocketStreamDestroyed.name, + this.handleEventWebSocketStream, { once: true }, ); this.dispatchEvent( @@ -210,89 +227,6 @@ class WebSocketConnection extends EventTarget { ); }; - public static createWebSocketConnection( - args: - | { - type: 'client'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: undefined; - client?: WebSocketClient; - reasonToCode?: StreamReasonToCode; - codeToReason?: StreamCodeToReason; - verifyCallback?: VerifyCallback; - logger?: Logger; - } - | { - type: 'server'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: WebSocketServer; - client?: undefined; - reasonToCode?: StreamReasonToCode; - codeToReason?: StreamCodeToReason; - verifyCallback?: undefined; - logger?: Logger; - }, - ctx?: Partial, - ): PromiseCancellable; - @timedCancellable(true, Infinity, errors.ErrorWebSocketConnectionStartTimeOut) - public static async createWebSocketConnection( - args: - | { - type: 'client'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: undefined; - client?: WebSocketClient; - reasonToCode?: StreamReasonToCode; - codeToReason?: StreamCodeToReason; - verifyCallback?: VerifyCallback; - logger?: Logger; - } - | { - type: 'server'; - connectionId: number; - remoteInfo: RemoteInfo; - config: WebSocketConfig; - socket: ws.WebSocket; - server?: WebSocketServer; - client?: undefined; - reasonToCode?: StreamReasonToCode; - codeToReason?: StreamCodeToReason; - verifyCallback?: undefined; - logger?: Logger; - }, - @context ctx: ContextTimed, - ): Promise { - // Setting up abort/cancellation logic - const abortProm = promise(); - const abortHandler = () => { - abortProm.rejectP(ctx.signal.reason); - }; - ctx.signal.addEventListener('abort', abortHandler); - const connection = new this(args); - try { - await Promise.race([connection.start(), abortProm.p]); - } catch (e) { - await connection.stop(); - throw e; - } finally { - ctx.signal.removeEventListener('abort', abortHandler); - } - if (connection.config.keepAliveIntervalTime != null) { - connection.startKeepAliveIntervalTimer( - connection.config.keepAliveIntervalTime, - ); - } - return connection; - } public constructor({ type, connectionId, @@ -332,7 +266,6 @@ class WebSocketConnection extends EventTarget { verifyCallback?: undefined; logger?: Logger; }) { - super(); this.logger = logger ?? new Logger(`${this.constructor.name}`); this.connectionId = connectionId; this.socket = socket; @@ -353,8 +286,21 @@ class WebSocketConnection extends EventTarget { this.resolveClosedP = resolveClosedP; this.rejectClosedP = rejectClosedP; } - public async start(): Promise { + public start( + ctx?: Partial, + ): PromiseCancellable; + @timedCancellable(true, Infinity, errors.ErrorWebSocketConnectionStartTimeOut) + public async start( + @context ctx: ContextTimed, + ): Promise { this.logger.info(`Start ${this.constructor.name}`); + ctx.signal.throwIfAborted(); + const { p: abortP, rejectP: rejectAbortP }= promise(); + const abortHandler = () => { + rejectAbortP(ctx.signal.reason); + }; + ctx.signal.addEventListener('abort', abortHandler); + const promises: Array> = []; const connectProm = promise(); @@ -408,7 +354,10 @@ class WebSocketConnection extends EventTarget { // Wait for open try { - await Promise.all(promises); + await Promise.race([ + Promise.all(promises), + abortP, + ]); } catch (e) { this.socket.removeAllListeners('error'); this.socket.removeAllListeners('upgrade'); @@ -417,6 +366,7 @@ class WebSocketConnection extends EventTarget { this.socket.terminate(); throw e; } finally { + ctx.signal.removeEventListener('abort', abortHandler); this.socket.removeAllListeners('upgrade'); this.socket.off('open', openHandler); this.socket.off('error', openErrorHandler); @@ -437,6 +387,12 @@ class WebSocketConnection extends EventTarget { this.socket.on('pong', this.pongHandler); this.socket.on('error', this.errorHandler); + if (this.config.keepAliveIntervalTime != null) { + this.startKeepAliveIntervalTimer( + this.config.keepAliveIntervalTime, + ); + } + this.logger.info(`Started ${this.constructor.name}`); } @@ -461,7 +417,12 @@ class WebSocketConnection extends EventTarget { }); stream.addEventListener( events.EventWebSocketStreamDestroy.name, - this.handleEventWebSocketStreamDestroy, + this.handleEventWebSocketStream, + { once: true }, + ); + stream.addEventListener( + events.EventWebSocketStreamDestroyed.name, + this.handleEventWebSocketStream, { once: true }, ); // Ok the stream is opened and working @@ -545,7 +506,6 @@ class WebSocketConnection extends EventTarget { this.parentInstance.connectionMap.delete(this.connectionId); } - this.dispatchEvent(new events.EventWebSocketConnectionStop()); this.logger.info(`Stopped ${this.constructor.name}`); } diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index b8911f48..e9b6c8e9 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -8,10 +8,10 @@ import type { WebSocketConfig, } from './types'; import https from 'https'; -import { startStop, status } from '@matrixai/async-init'; +import { StartStop, status, ready } from '@matrixai/async-init/dist/StartStop'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; -import { EventAll, EventDefault } from '@matrixai/events'; +import { AbstractEvent, EventAll, EventDefault } from '@matrixai/events'; import * as errors from './errors'; import * as events from './events'; import { never, promise } from './utils'; @@ -19,18 +19,33 @@ import WebSocketConnection from './WebSocketConnection'; import { serverDefault } from './config'; import WebSocketConnectionMap from './WebSocketConnectionMap'; +interface WebSocketServer extends StartStop {} /** * You must provide an error handler `addEventListener('error')`. * Otherwise, errors will just be ignored. * * Events: - * - connectionStream - when new stream is created from a connection - * - connectionError - connection error event - * - connectionDestroy - when connection is destroyed - * - streamDestroy - when stream is destroyed + * - {@link events.EventWebSocketServerStart}, + * - {@link events.EventWebSocketServerStarted}, + * - {@link events.EventWebSocketServerStop}, + * - {@link events.EventWebSocketServerStopped}, + * - {@link events.EventWebSocketServerConnection} + * - {@link events.EventWebSocketServerError} + * - {@link events.EventWebSocketConnectionStream} + * - {@link events.EventWebSocketConnectionStart} + * - {@link events.EventWebSocketConnectionStarted} + * - {@link events.EventWebSocketConnectionStop} + * - {@link events.EventWebSocketConnectionStopped} + * - {@link events.EventWebSocketConnectionError} - can occur due to a timeout too + * - {@link events.EventWebSocketStreamDestroy} + * - {@link events.EventWebSocketStreamDestroyed} */ -interface WebSocketServer extends startStop.StartStop {} -@startStop.StartStop() +@StartStop({ + eventStart: events.EventWebSocketServerStart, + eventStarted: events.EventWebSocketServerStarted, + eventStop: events.EventWebSocketServerStop, + eventStopped: events.EventWebSocketServerStopped, +}) class WebSocketServer extends EventTarget { protected logger: Logger; protected config: WebSocketConfig; @@ -44,12 +59,13 @@ class WebSocketServer extends EventTarget { protected _port: number; protected _host: string; - protected handleWebSocketConnectionEvents = ( + protected handleWebSocketConnection = ( event: - | events.EventWebSocketConnection - | EventAll, + | AbstractEvent + | EventDefault + | EventAll, ) => { - if (event instanceof EventAll) { + if (event instanceof EventAll || event instanceof EventDefault) { this.dispatchEvent(event.detail.clone()); } else { this.dispatchEvent(event.clone()); @@ -118,7 +134,6 @@ class WebSocketServer extends EventTarget { this._port = address.port; this.logger.debug(`Listening on port ${this._port}`); this._host = address.address ?? '127.0.0.1'; - this.dispatchEvent(new events.EventWebSocketServerStart()); this.logger.info(`Started ${this.constructor.name}`); } @@ -162,22 +177,20 @@ class WebSocketServer extends EventTarget { this.webSocketServer.off('error', this.errorHandler); this.server.off('error', this.errorHandler); this.server.on('request', this.requestHandler); - - this.dispatchEvent(new events.EventWebSocketServerStop()); this.logger.info(`Stopped ${this.constructor.name}`); } - @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) + @ready(new errors.ErrorWebSocketServerNotRunning()) public getPort(): number { return this._port; } - @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) + @ready(new errors.ErrorWebSocketServerNotRunning()) public getHost(): string { return this._host; } - @startStop.ready(new errors.ErrorWebSocketServerNotRunning()) + @ready(new errors.ErrorWebSocketServerNotRunning()) public updateConfig( config: Partial & { key?: string; @@ -208,7 +221,7 @@ class WebSocketServer extends EventTarget { ) => { const httpSocket = request.connection; const connectionId = this.connectionMap.allocateId(); - const connection = await WebSocketConnection.createWebSocketConnection( + const connection = new WebSocketConnection( { type: 'server', connectionId: connectionId, @@ -225,27 +238,28 @@ class WebSocketServer extends EventTarget { ), server: this, }, - { - timer: this.config.connectTimeoutTime, - }, ); + await connection.start({ + timer: this.config.connectTimeoutTime, + }); + // Handling connection events connection.addEventListener( - events.EventWebSocketConnectionStop.name, - (event: events.EventWebSocketConnectionStop) => { + events.EventWebSocketConnectionStopped.name, + (event: events.EventWebSocketConnectionStopped) => { connection.removeEventListener( - EventDefault.name, - this.handleWebSocketConnectionEvents, + EventAll.name, + this.handleWebSocketConnection, ); - this.handleWebSocketConnectionEvents(event); + this.handleWebSocketConnection(event); }, { once: true }, ); connection.addEventListener( EventDefault.name, - this.handleWebSocketConnectionEvents, + this.handleWebSocketConnection, ); this.dispatchEvent( diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index cf4b83fe..652c02f8 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -15,10 +15,17 @@ import { import * as errors from './errors'; import * as events from './events'; + interface WebSocketStream extends CreateDestroy {} -interface WebSocketStream extends Evented {} -@CreateDestroy() -@Evented() +/** + * Events: + * - {@link events.EventWebSocketStreamDestroy} + * - {@link events.EventWebSocketStreamDestroyed} + */ +@CreateDestroy({ + eventDestroy: events.EventWebSocketStreamDestroy, + eventDestroyed: events.EventWebSocketStreamDestroyed, +}) class WebSocketStream implements ReadableWritablePair { public streamId: StreamId; public readable: ReadableStream; @@ -196,9 +203,6 @@ class WebSocketStream implements ReadableWritablePair { // So the connection will infinitely create streams with the same streamId when it receives the ERROR/CLOSE frame. // I'm dealing with this by just filtering out ERROR/CLOSE frames in the connection's onMessage handler, but there might be a better way to do this. this.connection.streamMap.delete(this.streamId); - this.dispatchEvent( - new events.EventWebSocketStreamDestroy({ bubbles: true }), - ); this.logger.info(`Destroyed ${this.constructor.name}`); } diff --git a/src/events.ts b/src/events.ts index 78aac4d0..a29fa399 100644 --- a/src/events.ts +++ b/src/events.ts @@ -10,6 +10,8 @@ abstract class EventWebSocketClient extends EventWebSocket {} class EventWebSocketClientDestroy extends EventWebSocketClient {} +class EventWebSocketClientDestroyed extends EventWebSocketClient {} + class EventWebSocketClientError extends EventWebSocketClient {} // Server events @@ -20,8 +22,12 @@ class EventWebSocketServerConnection extends EventWebSocketServer {} // Connection events @@ -30,8 +36,14 @@ abstract class EventWebSocketConnection extends EventWebSocket {} class EventWebSocketConnectionStream extends EventWebSocketConnection {} +class EventWebSocketConnectionStart extends EventWebSocketConnection {} + +class EventWebSocketConnectionStarted extends EventWebSocketConnection {} + class EventWebSocketConnectionStop extends EventWebSocketConnection {} +class EventWebSocketConnectionStopped extends EventWebSocketConnection {} + class EventWebSocketConnectionError extends EventWebSocketConnection {} // Stream events @@ -40,20 +52,30 @@ abstract class EventWebSocketStream extends EventWebSocket {} class EventWebSocketStreamDestroy extends EventWebSocketStream {} +class EventWebSocketStreamDestroyed extends EventWebSocketStream {} + + export { EventWebSocket, EventWebSocketClient, EventWebSocketClientError, EventWebSocketClientDestroy, + EventWebSocketClientDestroyed, EventWebSocketServer, EventWebSocketServerConnection, EventWebSocketServerStart, + EventWebSocketServerStarted, EventWebSocketServerStop, + EventWebSocketServerStopped, EventWebSocketServerError, EventWebSocketConnection, EventWebSocketConnectionStream, + EventWebSocketConnectionStart, + EventWebSocketConnectionStarted, EventWebSocketConnectionStop, + EventWebSocketConnectionStopped, EventWebSocketConnectionError, EventWebSocketStream, EventWebSocketStreamDestroy, + EventWebSocketStreamDestroyed }; From 9addf86ac7314a554fd2a3fe618eb3cb8920db21 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:32:18 +1000 Subject: [PATCH 045/149] lintfix --- src/WebSocketClient.ts | 38 ++++++++++++++++++-------------------- src/WebSocketConnection.ts | 27 ++++++++++----------------- src/WebSocketServer.ts | 35 +++++++++++++++++------------------ src/errors.ts | 4 +++- src/events.ts | 3 +-- src/utils/utils.ts | 2 +- 6 files changed, 50 insertions(+), 59 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 90df449c..b39cff90 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -1,9 +1,10 @@ import type { Host, Port, VerifyCallback, WebSocketConfig } from './types'; +import type { AbstractEvent } from '@matrixai/events'; import { createDestroy } from '@matrixai/async-init'; import Logger from '@matrixai/logger'; import WebSocket from 'ws'; import { Validator } from 'ip-num'; -import { AbstractEvent, EventAll, EventDefault } from '@matrixai/events'; +import { EventAll, EventDefault } from '@matrixai/events'; import * as errors from './errors'; import WebSocketConnection from './WebSocketConnection'; import WebSocketConnectionMap from './WebSocketConnectionMap'; @@ -67,8 +68,7 @@ class WebSocketClient extends EventTarget { }), ); } - } - else if (event instanceof events.EventWebSocketConnectionError) { + } else if (event instanceof events.EventWebSocketConnectionError) { this.dispatchEvent( (event as events.EventWebSocketConnectionError).clone(), ); @@ -153,21 +153,19 @@ class WebSocketClient extends EventTarget { }); const connectionId = client.connectionMap.allocateId(); - const connection = new WebSocketConnection( - { - type: 'client', - connectionId, - remoteInfo: { - host: host_, - port: port_, - }, - config: wsConfig, - socket: webSocket, - verifyCallback, - client: client, - } - ); - await connection.start( { + const connection = new WebSocketConnection({ + type: 'client', + connectionId, + remoteInfo: { + host: host_, + port: port_, + }, + config: wsConfig, + socket: webSocket, + verifyCallback, + client: client, + }); + await connection.start({ timer: wsConfig.connectTimeoutTime, }); connection.addEventListener( @@ -179,8 +177,8 @@ class WebSocketClient extends EventTarget { ); client.handleEventWebSocketConnection(event); }, - { once: true } - ) + { once: true }, + ); connection.addEventListener( EventDefault.name, client.handleEventWebSocketConnection, diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 75f74526..56ab8c30 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -133,12 +133,14 @@ class WebSocketConnection { * Bubble up stream destroy event */ protected handleEventWebSocketStream = ( - event: EventAll | EventDefault | events.EventWebSocketStream, + event: + | EventAll + | EventDefault + | events.EventWebSocketStream, ) => { if (event instanceof EventAll || event instanceof EventDefault) { this.dispatchEvent(event.detail.clone()); - } - else { + } else { this.dispatchEvent(event.clone()); } }; @@ -286,16 +288,12 @@ class WebSocketConnection { this.resolveClosedP = resolveClosedP; this.rejectClosedP = rejectClosedP; } - public start( - ctx?: Partial, - ): PromiseCancellable; + public start(ctx?: Partial): PromiseCancellable; @timedCancellable(true, Infinity, errors.ErrorWebSocketConnectionStartTimeOut) - public async start( - @context ctx: ContextTimed, - ): Promise { + public async start(@context ctx: ContextTimed): Promise { this.logger.info(`Start ${this.constructor.name}`); ctx.signal.throwIfAborted(); - const { p: abortP, rejectP: rejectAbortP }= promise(); + const { p: abortP, rejectP: rejectAbortP } = promise(); const abortHandler = () => { rejectAbortP(ctx.signal.reason); }; @@ -354,10 +352,7 @@ class WebSocketConnection { // Wait for open try { - await Promise.race([ - Promise.all(promises), - abortP, - ]); + await Promise.race([Promise.all(promises), abortP]); } catch (e) { this.socket.removeAllListeners('error'); this.socket.removeAllListeners('upgrade'); @@ -388,9 +383,7 @@ class WebSocketConnection { this.socket.on('error', this.errorHandler); if (this.config.keepAliveIntervalTime != null) { - this.startKeepAliveIntervalTimer( - this.config.keepAliveIntervalTime, - ); + this.startKeepAliveIntervalTimer(this.config.keepAliveIntervalTime); } this.logger.info(`Started ${this.constructor.name}`); diff --git a/src/WebSocketServer.ts b/src/WebSocketServer.ts index e9b6c8e9..431f4ab1 100644 --- a/src/WebSocketServer.ts +++ b/src/WebSocketServer.ts @@ -7,11 +7,12 @@ import type { StreamReasonToCode, WebSocketConfig, } from './types'; +import type { AbstractEvent } from '@matrixai/events'; import https from 'https'; import { StartStop, status, ready } from '@matrixai/async-init/dist/StartStop'; import Logger from '@matrixai/logger'; import * as ws from 'ws'; -import { AbstractEvent, EventAll, EventDefault } from '@matrixai/events'; +import { EventAll, EventDefault } from '@matrixai/events'; import * as errors from './errors'; import * as events from './events'; import { never, promise } from './utils'; @@ -221,24 +222,22 @@ class WebSocketServer extends EventTarget { ) => { const httpSocket = request.connection; const connectionId = this.connectionMap.allocateId(); - const connection = new WebSocketConnection( - { - type: 'server', - connectionId: connectionId, - remoteInfo: { - host: (httpSocket.remoteAddress ?? '') as Host, - port: (httpSocket.remotePort ?? 0) as Port, - }, - socket: webSocket, - config: this.config, - reasonToCode: this.reasonToCode, - codeToReason: this.codeToReason, - logger: this.logger.getChild( - `${WebSocketConnection.name} ${connectionId}`, - ), - server: this, + const connection = new WebSocketConnection({ + type: 'server', + connectionId: connectionId, + remoteInfo: { + host: (httpSocket.remoteAddress ?? '') as Host, + port: (httpSocket.remotePort ?? 0) as Port, }, - ); + socket: webSocket, + config: this.config, + reasonToCode: this.reasonToCode, + codeToReason: this.codeToReason, + logger: this.logger.getChild( + `${WebSocketConnection.name} ${connectionId}`, + ), + server: this, + }); await connection.start({ timer: this.config.connectTimeoutTime, diff --git a/src/errors.ts b/src/errors.ts index 511b57ce..ea8a25d2 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -71,7 +71,9 @@ class ErrorWebSocketStreamReader extends ErrorWebSocketStream { static description = 'WebSocket Stream readable error'; } -class ErrorWebSocketStreamReadableParse extends ErrorWebSocketStreamReader { +class ErrorWebSocketStreamReadableParse< + T, +> extends ErrorWebSocketStreamReader { static description = 'WebSocket Stream readable message parse failed'; } diff --git a/src/events.ts b/src/events.ts index a29fa399..4bebad50 100644 --- a/src/events.ts +++ b/src/events.ts @@ -54,7 +54,6 @@ class EventWebSocketStreamDestroy extends EventWebSocketStream {} class EventWebSocketStreamDestroyed extends EventWebSocketStream {} - export { EventWebSocket, EventWebSocketClient, @@ -77,5 +76,5 @@ export { EventWebSocketConnectionError, EventWebSocketStream, EventWebSocketStreamDestroy, - EventWebSocketStreamDestroyed + EventWebSocketStreamDestroyed, }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 6bc79d16..60e9d6c2 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -125,5 +125,5 @@ export { fromStreamId, StreamMessageType, StreamShutdown, - StreamErrorCode + StreamErrorCode, }; From e447669a307cd970684d8d5886a6cedd3c194415 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:46:34 +1000 Subject: [PATCH 046/149] fix: WebSocketStream lifecycle methods are now all awaited --- src/WebSocketStream.ts | 67 +++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 652c02f8..ad94e52e 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,4 +1,9 @@ -import type { StreamCodeToReason, StreamId, StreamReasonToCode, VarInt } from './types'; +import type { + StreamCodeToReason, + StreamId, + StreamReasonToCode, + VarInt, +} from './types'; import type WebSocketConnection from './WebSocketConnection'; import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; @@ -15,7 +20,6 @@ import { import * as errors from './errors'; import * as events from './events'; - interface WebSocketStream extends CreateDestroy {} /** * Events: @@ -329,19 +333,26 @@ class WebSocketStream implements ReadableWritablePair { return; } this.readableController.enqueue(data); - } else if (type === StreamMessageType.ERROR || type === StreamMessageType.CLOSE) { + } else if ( + type === StreamMessageType.ERROR || + type === StreamMessageType.CLOSE + ) { try { const shutdown = dv.getUint8(0) as StreamShutdown; - let isError = type === StreamMessageType.ERROR; + const isError = type === StreamMessageType.ERROR; let reason: any; if (type === StreamMessageType.ERROR) { const errorCode = toVarInt(data.subarray(1)).data; switch (errorCode) { case BigInt(StreamErrorCode.ErrorReadableStreamParse): - reason = new errors.ErrorWebSocketStreamReadableParse('receiver was unable to parse a sent message'); + reason = new errors.ErrorWebSocketStreamReadableParse( + 'receiver was unable to parse a sent message', + ); break; case BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow): - reason = new errors.ErrorWebSocketStreamReadableBufferOverload('receiver was unable to accept a sent message'); + reason = new errors.ErrorWebSocketStreamReadableBufferOverload( + 'receiver was unable to accept a sent message', + ); break; default: reason = await this.codeToReason('recv', errorCode); @@ -380,18 +391,18 @@ class WebSocketStream implements ReadableWritablePair { /** * Forces the active stream to end early */ - public cancel(reason?: any): void { + public async cancel(reason?: any) { const isError = reason != null && !(reason instanceof errors.ErrorWebSocketStreamClose); reason = reason ?? new errors.ErrorWebSocketStreamCancel(); // Close the streams with the given error, if (!this._readableEnded) { this.readableController.error(reason); - void this.signalReadableEnd(isError, reason); + await this.signalReadableEnd(isError, reason); } if (!this._writableEnded) { this.writableController.error(reason); - void this.signalWritableEnd(isError, reason); + await this.signalWritableEnd(isError, reason); } } @@ -413,21 +424,27 @@ class WebSocketStream implements ReadableWritablePair { let code: VarInt; if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { code = BigInt(StreamErrorCode.ErrorReadableStreamParse) as VarInt; + } else if ( + reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload + ) { + code = BigInt( + StreamErrorCode.ErrorReadableStreamBufferOverflow, + ) as VarInt; + } else { + code = (await this.reasonToCode('send', reason)) as VarInt; } - else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { - code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow) as VarInt; - } - else { - code = await this.reasonToCode('send', reason) as VarInt; - } - await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Write, code); + await this.streamSend( + StreamMessageType.ERROR, + StreamShutdown.Write, + code, + ); this.readableController.error(reason); } else { await this.streamSend(StreamMessageType.CLOSE, StreamShutdown.Write); } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); - if (this[status] !== 'destroying') void this.destroy(); + if (this[status] !== 'destroying') await this.destroy(); } this.logger.debug(`readable ended`); } @@ -452,12 +469,14 @@ class WebSocketStream implements ReadableWritablePair { let code: VarInt; if (reason instanceof errors.ErrorWebSocketStreamReadableParse) { code = BigInt(StreamErrorCode.ErrorReadableStreamParse) as VarInt; - } - else if (reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload) { - code = BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow) as VarInt; - } - else { - code = await this.reasonToCode('send', reason) as VarInt; + } else if ( + reason instanceof errors.ErrorWebSocketStreamReadableBufferOverload + ) { + code = BigInt( + StreamErrorCode.ErrorReadableStreamBufferOverflow, + ) as VarInt; + } else { + code = (await this.reasonToCode('send', reason)) as VarInt; } await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Read, code); this.writableController.error(reason); @@ -466,7 +485,7 @@ class WebSocketStream implements ReadableWritablePair { } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); - if (this[status] !== 'destroying') void this.destroy(); + if (this[status] !== 'destroying') await this.destroy(); } this.logger.debug(`writable ended`); } From 3cc29fb469a21859cf2a6f33a011e3d52d4bfcaf Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:46:48 +1000 Subject: [PATCH 047/149] feat: WebSocketStream lifecycle tests --- tests/WebSocketStream.test.ts | 75 ++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 5795fcb8..1a3a1d6f 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -6,6 +6,7 @@ import WebSocketConnection from '@/WebSocketConnection'; import * as events from '@/events'; import { promise, StreamMessageType } from '@/utils'; import * as config from '@/config'; +import * as utils from '@/utils'; import * as testUtils from './utils'; // Smaller buffer size for the sake of testing @@ -101,8 +102,60 @@ describe(WebSocketStream.name, () => { streamIdCounter++; return [stream1, stream2]; } + test('should create stream', async () => { + const streams = await createStreamPair(connection1, connection2); + expect(streams.length).toEqual(2); + for (const stream of streams) { + await stream.destroy(); + } + }); + test('destroying stream should clean up on both ends while streams are used', async () => { + const streamsNum = 10; + + let streamCreatedCount = 0; + let streamEndedCount = 0; + const streamCreationProm = utils.promise(); + const streamEndedProm = utils.promise(); + + const streams = new Array(); + + connection2.addEventListener( + events.EventWebSocketConnectionStream.name, + (event: events.EventWebSocketConnectionStream) => { + const stream = event.detail; + streamCreatedCount += 1; + if (streamCreatedCount >= streamsNum) streamCreationProm.resolveP(); + stream.addEventListener( + events.EventWebSocketStreamDestroyed.name, + () => { + streamEndedCount += 1; + if (streamEndedCount >= streamsNum) streamEndedProm.resolveP(); + } + ); + } + ); + + for (let i = 0; i < streamsNum; i++) { + const stream = await WebSocketStream.createWebSocketStream({ + streamId: streamIdCounter as StreamId, + bufferSize: STREAM_BUFFER_SIZE, + connection: connection1, + logger: logger1, + }); + streamIdCounter++; + streams.push(stream); + } + await streamCreationProm.p; + await Promise.allSettled(streams.map((stream) => stream.destroy())); + await streamEndedProm.p; + expect(streamCreatedCount).toEqual(streamsNum); + expect(streamEndedCount).toEqual(streamsNum); + for (const stream of streams) { + await stream.destroy(); + } + }); testProp( - 'single write within buffer size', + 'should send data over stream - single write within buffer size', [fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE })], async (data) => { const [stream1, stream2] = await createStreamPair( @@ -135,10 +188,11 @@ describe(WebSocketStream.name, () => { expect(readChunks).toEqual([data]); await stream1.destroy(); + await stream2.destroy(); }, ); testProp( - 'single write outside buffer size', + 'should send data over stream - single write outside buffer size', [fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 })], async (data) => { const [stream1, stream2] = await createStreamPair( @@ -171,10 +225,11 @@ describe(WebSocketStream.name, () => { expect(testUtils.concatUInt8Array(...readChunks)).toEqual(data); await stream1.destroy(); + await stream2.destroy(); }, ); testProp( - 'multiple writes within buffer size', + 'should send data over stream - multiple writes within buffer size', [fc.array(fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }))], async (data) => { const [stream1, stream2] = await createStreamPair( @@ -211,10 +266,11 @@ describe(WebSocketStream.name, () => { ); await stream1.destroy(); + await stream2.destroy(); }, ); testProp( - 'multiple writes outside buffer size', + 'should send data over stream - multiple writes outside buffer size', [fc.array(fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }))], async (data) => { const [stream1, stream2] = await createStreamPair( @@ -251,10 +307,11 @@ describe(WebSocketStream.name, () => { ); await stream1.destroy(); + await stream2.destroy(); }, ); testProp( - 'multiple writes within and outside buffer size', + 'should send data over stream - multiple writes within and outside buffer size', [ fc.array( fc.oneof( @@ -298,10 +355,11 @@ describe(WebSocketStream.name, () => { ); await stream1.destroy(); + await stream2.destroy(); }, ); testProp( - 'simultaneous writes', + 'should send data over stream - simultaneous multiple writes within and outside buffer size', [ fc.array( fc.oneof( @@ -317,10 +375,7 @@ describe(WebSocketStream.name, () => { ), ], async (...data) => { - const streams = await createStreamPair( - connection1, - connection2, - ); + const streams = await createStreamPair(connection1, connection2); const readProms: Array>> = []; const writeProms: Array> = []; From 5a9a641db62cab8d8929ad0584dc383be2c34494 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:56:20 +1000 Subject: [PATCH 048/149] feat: moved parsing/generation of stream/connection messages to separate domain --- src/WebSocketConnection.ts | 36 ++-- src/WebSocketStream.ts | 268 ++++++++++--------------- src/message/errors.ts | 15 ++ src/{utils => message}/index.ts | 1 + src/message/types.ts | 62 ++++++ src/message/utils.ts | 341 ++++++++++++++++++++++++++++++++ src/types.ts | 12 -- src/utils.ts | 24 +++ src/utils/types.ts | 10 - src/utils/utils.ts | 129 ------------ tests/WebSocketStream.test.ts | 34 ++-- tests/message/utils.test.ts | 111 +++++++++++ tests/message/utils.ts | 87 ++++++++ tests/utils.ts | 12 -- 14 files changed, 785 insertions(+), 357 deletions(-) create mode 100644 src/message/errors.ts rename src/{utils => message}/index.ts (65%) create mode 100644 src/message/types.ts create mode 100644 src/message/utils.ts create mode 100644 src/utils.ts delete mode 100644 src/utils/types.ts delete mode 100644 src/utils/utils.ts create mode 100644 tests/message/utils.test.ts create mode 100644 tests/message/utils.ts diff --git a/src/WebSocketConnection.ts b/src/WebSocketConnection.ts index 56ab8c30..6fc1cce5 100644 --- a/src/WebSocketConnection.ts +++ b/src/WebSocketConnection.ts @@ -4,7 +4,6 @@ import type { Host, RemoteInfo, StreamCodeToReason, - StreamId, StreamReasonToCode, VerifyCallback, WebSocketConfig, @@ -13,6 +12,7 @@ import type WebSocketClient from './WebSocketClient'; import type WebSocketServer from './WebSocketServer'; import type { DetailedPeerCertificate, TLSSocket } from 'tls'; import type WebSocketConnectionMap from './WebSocketConnectionMap'; +import type { StreamId } from './message'; import { startStop } from '@matrixai/async-init'; import { Lock } from '@matrixai/async-locks'; import { context, timedCancellable } from '@matrixai/contexts/dist/decorators'; @@ -20,11 +20,12 @@ import Logger from '@matrixai/logger'; import * as ws from 'ws'; import { Timer } from '@matrixai/timer'; import { ready } from '@matrixai/async-init/dist/CreateDestroyStartStop'; -import { EventAll, EventDefault, Evented } from '@matrixai/events'; +import { EventAll, EventDefault } from '@matrixai/events'; import WebSocketStream from './WebSocketStream'; import * as errors from './errors'; -import { fromStreamId, promise, StreamMessageType, toStreamId } from './utils'; import * as events from './events'; +import { generateStreamId, parseStreamId, StreamMessageType } from './message'; +import { promise } from './utils'; const timerCleanupReasonSymbol = Symbol('timerCleanupReasonSymbol'); @@ -167,18 +168,31 @@ class WebSocketConnection { ); return; } - let message: Uint8Array = + let remainder: Uint8Array = data instanceof ArrayBuffer ? new Uint8Array(data) : data; - const { data: streamId, remainder } = toStreamId(message); - message = remainder; + let streamId; + try { + const { data: parsedStreamId, remainder: postStreamIdRemainder } = + parseStreamId(remainder); + streamId = parsedStreamId; + remainder = postStreamIdRemainder; + } catch (e) { + // TODO: domain specific error + this.dispatchEvent( + new events.EventWebSocketConnectionError('parsing StreamId failed', { + detail: e, + }), + ); + return; + } let stream = this.streamMap.get(streamId); if (stream == null) { - const messageType = message.at(0); + const messageType = remainder.at(0); if ( - messageType === StreamMessageType.CLOSE || - messageType === StreamMessageType.ERROR + messageType === StreamMessageType.Close || + messageType === StreamMessageType.Error ) { return; } @@ -205,7 +219,7 @@ class WebSocketConnection { ); } - await stream!.streamRecv(message); + await stream!.streamRecv(remainder); }; protected pingHandler = () => { @@ -436,7 +450,7 @@ class WebSocketConnection { * @internal */ public async streamSend(streamId: StreamId, data: Uint8Array) { - const encodedStreamId = fromStreamId(streamId); + const encodedStreamId = generateStreamId(streamId); const array = new Uint8Array(encodedStreamId.length + data.length); array.set(encodedStreamId, 0); array.set(data, encodedStreamId.length); diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index ad94e52e..f6e129a5 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,24 +1,18 @@ -import type { - StreamCodeToReason, - StreamId, - StreamReasonToCode, - VarInt, -} from './types'; +import type { StreamCodeToReason, StreamReasonToCode } from './types'; import type WebSocketConnection from './WebSocketConnection'; +import type { StreamId, StreamMessage, VarInt } from './message'; import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; -import { Evented } from '@matrixai/events'; +import { promise } from './utils'; +import * as errors from './errors'; +import * as events from './events'; import { - fromVarInt, - never, - promise, + generateStreamMessage, + parseStreamMessage, StreamErrorCode, StreamMessageType, StreamShutdown, - toVarInt, -} from './utils'; -import * as errors from './errors'; -import * as events from './events'; +} from './message'; interface WebSocketStream extends CreateDestroy {} /** @@ -116,7 +110,10 @@ class WebSocketStream implements ReadableWritablePair { return; } // Send ACK on every read as there will be more usable space on the buffer. - await this.streamSend(StreamMessageType.ACK, controller.desiredSize!); + await this.streamSend({ + type: StreamMessageType.Ack, + payload: controller.desiredSize!, + }); }, cancel: async (reason) => { this.logger.debug(`readable aborted with [${reason.message}]`); @@ -158,7 +155,10 @@ class WebSocketStream implements ReadableWritablePair { } // Decrement the desired size by the amount of bytes written this.writableDesiredSize -= bytesWritten; - await this.streamSend(StreamMessageType.DATA, data); + await this.streamSend({ + type: StreamMessageType.Data, + payload: data, + }); if (isChunkable) { await writeHandler(chunk.subarray(bytesWritten), controller); @@ -200,7 +200,7 @@ class WebSocketStream implements ReadableWritablePair { this.logger.info(`Destroy ${this.constructor.name}`); // Force close any open streams this.writableDesiredSizeProm.resolveP(); - this.cancel(new errors.ErrorWebSocketStreamClose()); + await this.cancel(new errors.ErrorWebSocketStreamClose()); // Removing stream from the connection's stream map // TODO: the other side currently will send back an ERROR/CLOSE frame from us sending an ERROR/CLOSE frame from this.close(). // However, out stream gets deleted before we receive that message on the connection. @@ -210,72 +210,8 @@ class WebSocketStream implements ReadableWritablePair { this.logger.info(`Destroyed ${this.constructor.name}`); } - /** - * Send an ACK frame with a payloadSize. - * @param code - ACK - * @param payloadSize - The number of bytes that the receiver can accept. - */ - protected async streamSend( - type: StreamMessageType.ACK, - payloadSize: number, - ): Promise; - /** - * Send a DATA frame with a payload on the stream. - * @param code - DATA - * @param data - The payload to send. - */ - protected async streamSend( - type: StreamMessageType.DATA, - data: Uint8Array, - ): Promise; - /** - * Send an ERROR frame with a payload on the stream. - * @param code - CLOSE - * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. - */ - protected async streamSend( - type: StreamMessageType.ERROR, - shutdown: StreamShutdown, - code: VarInt, - ): Promise; - /** - * Send a CLOSE frame with a payload on the stream. - * @param code - CLOSE - * @param shutdown - Signifies whether the ReadableStream or the WritableStream has been shutdown. - */ - protected async streamSend( - type: StreamMessageType.CLOSE, - shutdown: StreamShutdown, - ): Promise; - protected async streamSend( - type: StreamMessageType, - data_?: Uint8Array | number, - code?: VarInt, - ): Promise { - let data: Uint8Array | undefined; - if (type === StreamMessageType.ACK && typeof data_ === 'number') { - data = new Uint8Array(4); - const dv = new DataView(data.buffer); - dv.setUint32(0, data_, false); - } else if (type === StreamMessageType.DATA) { - data = data_ as Uint8Array; - } else if (type === StreamMessageType.ERROR) { - const errorCode = fromVarInt(code!); - data = new Uint8Array(1 + errorCode.length); - const dv = new DataView(data.buffer); - dv.setUint8(0, data_ as StreamShutdown); - data.set(errorCode, 1); - } else if (type === StreamMessageType.CLOSE) { - data = new Uint8Array([data_ as StreamShutdown]); - } else { - never(); - } - const arrayLength = 1 + (data?.length ?? 0); - const array = new Uint8Array(arrayLength); - array.set([type], 0); - if (data != null) { - array.set(data, 1); - } + protected async streamSend(message: StreamMessage): Promise { + const array = generateStreamMessage(message); await this.connection.streamSend(this.streamId, array); } @@ -294,37 +230,32 @@ class WebSocketStream implements ReadableWritablePair { cause: new RangeError(), }), ); + return; } - const type = message[0] as StreamMessageType; - const data = message.subarray(1); - const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); - if (type === StreamMessageType.ACK) { - try { - const bufferSize = dv.getUint32(0, false); - this.writableDesiredSize = bufferSize; - this.writableDesiredSizeProm.resolveP(); - this.logger.debug( - `received ACK, writableDesiredSize is now reset to ${bufferSize} bytes`, - ); - } catch (e) { - this.logger.debug(`received malformed ACK, closing stream`); - await this.signalReadableEnd( - true, - new errors.ErrorWebSocketStreamReadableParse( - 'ACK message did not contain a valid buffer size', - { - cause: e, - }, - ), - ); - } - } else if (type === StreamMessageType.DATA) { + + let parsedMessage: StreamMessage; + try { + parsedMessage = parseStreamMessage(message); + } catch (err) { + await this.signalReadableEnd( + true, + new errors.ErrorWebSocketStreamReadableParse(err.message, { + cause: err, + }), + ); + return; + } + + if (parsedMessage.type === StreamMessageType.Ack) { + this.writableDesiredSize = parsedMessage.payload; + this.writableDesiredSizeProm.resolveP(); + } else if (parsedMessage.type === StreamMessageType.Data) { if (this._readableEnded) { return; } if ( this.readableController.desiredSize != null && - data.length > this.readableController.desiredSize + parsedMessage.payload.length > this.readableController.desiredSize ) { await this.signalReadableEnd( true, @@ -332,59 +263,50 @@ class WebSocketStream implements ReadableWritablePair { ); return; } - this.readableController.enqueue(data); - } else if ( - type === StreamMessageType.ERROR || - type === StreamMessageType.CLOSE - ) { - try { - const shutdown = dv.getUint8(0) as StreamShutdown; - const isError = type === StreamMessageType.ERROR; - let reason: any; - if (type === StreamMessageType.ERROR) { - const errorCode = toVarInt(data.subarray(1)).data; - switch (errorCode) { - case BigInt(StreamErrorCode.ErrorReadableStreamParse): - reason = new errors.ErrorWebSocketStreamReadableParse( - 'receiver was unable to parse a sent message', - ); - break; - case BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow): - reason = new errors.ErrorWebSocketStreamReadableBufferOverload( - 'receiver was unable to accept a sent message', - ); - break; - default: - reason = await this.codeToReason('recv', errorCode); - } + this.readableController.enqueue(parsedMessage.payload); + } else if (parsedMessage.type === StreamMessageType.Error) { + const { shutdown, code } = parsedMessage.payload; + let reason: any; + switch (code) { + case BigInt(StreamErrorCode.ErrorReadableStreamParse): + reason = new errors.ErrorWebSocketStreamReadableParse( + 'receiver was unable to parse a sent message', + ); + break; + case BigInt(StreamErrorCode.ErrorReadableStreamBufferOverflow): + reason = new errors.ErrorWebSocketStreamReadableBufferOverload( + 'receiver was unable to accept a sent message', + ); + break; + default: + reason = await this.codeToReason('recv', code); + } + if (shutdown === StreamShutdown.Read) { + if (this._readableEnded) { + return; } - if (shutdown === StreamShutdown.Read) { - if (this._readableEnded) { - return; - } - await this.signalReadableEnd(isError, reason); - this.readableController.close(); - } else if (shutdown === StreamShutdown.Write) { - if (this._writableEnded) { - return; - } - await this.signalWritableEnd(isError, reason); - } else { - never('invalid shutdown type'); + await this.signalReadableEnd(true, reason); + this.readableController.close(); + } else if (shutdown === StreamShutdown.Write) { + if (this._writableEnded) { + return; } - } catch (e) { - await this.signalReadableEnd( - true, - new errors.ErrorWebSocketStreamReadableParse( - 'ERROR/CLOSE message did not contain a valid payload', - { - cause: e, - }, - ), - ); + await this.signalWritableEnd(true, reason); + } + } else if (parsedMessage.type === StreamMessageType.Close) { + const shutdown = parsedMessage.payload; + if (shutdown === StreamShutdown.Read) { + if (this._readableEnded) { + return; + } + await this.signalReadableEnd(false); + this.readableController.close(); + } else if (shutdown === StreamShutdown.Write) { + if (this._writableEnded) { + return; + } + await this.signalWritableEnd(false); } - } else { - never(); } } @@ -433,14 +355,19 @@ class WebSocketStream implements ReadableWritablePair { } else { code = (await this.reasonToCode('send', reason)) as VarInt; } - await this.streamSend( - StreamMessageType.ERROR, - StreamShutdown.Write, - code, - ); + await this.streamSend({ + type: StreamMessageType.Error, + payload: { + shutdown: StreamShutdown.Read, + code, + }, + }); this.readableController.error(reason); } else { - await this.streamSend(StreamMessageType.CLOSE, StreamShutdown.Write); + await this.streamSend({ + type: StreamMessageType.Close, + payload: StreamShutdown.Write, + }); } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); @@ -478,10 +405,19 @@ class WebSocketStream implements ReadableWritablePair { } else { code = (await this.reasonToCode('send', reason)) as VarInt; } - await this.streamSend(StreamMessageType.ERROR, StreamShutdown.Read, code); + await this.streamSend({ + type: StreamMessageType.Error, + payload: { + shutdown: StreamShutdown.Read, + code, + }, + }); this.writableController.error(reason); } else { - await this.streamSend(StreamMessageType.CLOSE, StreamShutdown.Read); + await this.streamSend({ + type: StreamMessageType.Close, + payload: StreamShutdown.Read, + }); } if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); diff --git a/src/message/errors.ts b/src/message/errors.ts new file mode 100644 index 00000000..b179a16c --- /dev/null +++ b/src/message/errors.ts @@ -0,0 +1,15 @@ +import { AbstractError } from '@matrixai/errors'; + +class ErrorStreamMessage extends AbstractError { + static description = 'Stream Message error'; +} + +class ErrorStreamParse extends ErrorStreamMessage { + static description = 'Stream Message parse error'; +} + +class ErrorStreamGenerate extends ErrorStreamMessage { + static description = 'Stream Message generation error'; +} + +export { ErrorStreamMessage, ErrorStreamParse, ErrorStreamGenerate }; diff --git a/src/utils/index.ts b/src/message/index.ts similarity index 65% rename from src/utils/index.ts rename to src/message/index.ts index f0c4eaa5..1694af65 100644 --- a/src/utils/index.ts +++ b/src/message/index.ts @@ -1,2 +1,3 @@ export * from './utils'; export * from './types'; +export * from './errors'; diff --git a/src/message/types.ts b/src/message/types.ts new file mode 100644 index 00000000..f680fffe --- /dev/null +++ b/src/message/types.ts @@ -0,0 +1,62 @@ +import type { Opaque } from '@/types'; +import type { StreamMessageType, StreamShutdown } from './utils'; + +interface Parsed { + data: T; + remainder: Uint8Array; +} + +/** + * VarInt is a 62 bit unsigned integer + */ +type VarInt = Opaque<'VarInt', bigint>; + +/** + * StreamId is a VarInt + */ +type StreamId = VarInt; + +type ConnectionMessage = { + streamId: StreamId; +} & StreamMessage; + +type StreamMessage = + | StreamMessageAck + | StreamMessageData + | StreamMessageClose + | StreamMessageError; + +type StreamMessageBase = { + type: StreamMessageType; + payload: PayloadType; +}; + +type StreamMessageAck = StreamMessageBase; + +type StreamMessageData = StreamMessageBase; + +type StreamMessageClose = StreamMessageBase< + StreamMessageType.Close, + StreamShutdown +>; + +type StreamMessageError = StreamMessageBase< + StreamMessageType.Error, + { + shutdown: StreamShutdown; + code: VarInt; + } +>; + +export type { + Parsed, + VarInt, + StreamId, + ConnectionMessage, + StreamMessage, + StreamMessageBase, + StreamMessageAck, + StreamMessageData, + StreamMessageClose, + StreamMessageError, +}; diff --git a/src/message/utils.ts b/src/message/utils.ts new file mode 100644 index 00000000..d4c2ae0f --- /dev/null +++ b/src/message/utils.ts @@ -0,0 +1,341 @@ +import type { + ConnectionMessage, + Parsed, + StreamId, + StreamMessage, + VarInt, +} from './types'; +import { never } from '@/utils'; +import * as errors from './errors'; + +// Enums + +const enum StreamMessageType { + Data = 0, + Ack = 1, + Error = 2, + Close = 3, +} + +const enum StreamShutdown { + Read = 0, + Write = 1, +} + +const enum StreamErrorCode { + Unknown = 0, + ErrorReadableStreamParse = 1, + ErrorReadableStreamBufferOverflow = 2, +} + +// Misc + +function concatUInt8Array(...arrays: Array) { + const totalLength = arrays.reduce((acc, val) => acc + val.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; +} + +// VarInt + +function parseVarInt(array: Uint8Array): Parsed { + let streamId: bigint; + + // Get header and prefix + const header = array.at(0); + if (header == null) { + throw new errors.ErrorStreamParse('VarInt header is too short'); + } + const prefix = header >> 6; + + // Copy bytearray and remove prefix + const arrayCopy = new Uint8Array(array.length); + arrayCopy.set(array); + arrayCopy[0] &= 0b00111111; + + const dv = new DataView(arrayCopy.buffer, arrayCopy.byteOffset); + + let readBytes = 0; + + try { + switch (prefix) { + case 0b00: + readBytes = 1; + streamId = BigInt(dv.getUint8(0)); + break; + case 0b01: + readBytes = 2; + streamId = BigInt(dv.getUint16(0, false)); + break; + case 0b10: + readBytes = 4; + streamId = BigInt(dv.getUint32(0, false)); + break; + case 0b11: + readBytes = 8; + streamId = dv.getBigUint64(0, false); + break; + } + } catch (e) { + throw new errors.ErrorStreamParse('VarInt is too short'); + } + return { + data: streamId! as VarInt, + remainder: array.subarray(readBytes), + }; +} + +function generateVarInt(varInt: VarInt): Uint8Array { + let array: Uint8Array; + let dv: DataView; + let prefixMask = 0; + + if (varInt < 0x40) { + array = new Uint8Array(1); + dv = new DataView(array.buffer); + dv.setUint8(0, Number(varInt)); + } else if (varInt < 0x4000) { + array = new Uint8Array(2); + dv = new DataView(array.buffer); + dv.setUint16(0, Number(varInt)); + prefixMask = 0b01_000000; + } else if (varInt < 0x40000000) { + array = new Uint8Array(4); + dv = new DataView(array.buffer); + dv.setUint32(0, Number(varInt)); + prefixMask = 0b10_000000; + } else if (varInt < 0x4000000000000000n) { + array = new Uint8Array(8); + dv = new DataView(array.buffer); + dv.setBigUint64(0, varInt); + prefixMask = 0b11_000000; + } else { + throw new errors.ErrorStreamGenerate('VarInt too large'); + } + + let header = dv.getUint8(0); + header |= prefixMask; + dv.setUint8(0, header); + + return array; +} + +const generateStreamId = generateVarInt as (streamId: StreamId) => Uint8Array; +const parseStreamId = parseVarInt as (array: Uint8Array) => Parsed; + +// StreamMessage + +function parseStreamMessageType(input: Uint8Array): Parsed { + const type = input.at(0); + if (type == null) { + throw new errors.ErrorStreamParse( + 'StreamMessage does not contain a StreamMessageType', + ); + } + switch (type) { + case StreamMessageType.Ack: + case StreamMessageType.Data: + case StreamMessageType.Close: + case StreamMessageType.Error: + return { + data: type, + remainder: input.subarray(1), + }; + default: + throw new errors.ErrorStreamParse( + `StreamMessage contains an invalid StreamMessageType: ${type}`, + ); + } +} + +function parseStreamMessageAckPayload(input: Uint8Array): Parsed { + const dv = new DataView(input.buffer, input.byteOffset, input.byteLength); + if (input.byteLength < 4) { + throw new errors.ErrorStreamParse('StreamMessageAckPayload is too short'); + } + const payload = dv.getUint32(0, false); + return { + data: payload, + remainder: input.subarray(4), + }; +} + +function parseStreamMessageClosePayload( + input: Uint8Array, +): Parsed { + const shutdown = input.at(0); + if (shutdown == null) { + throw new errors.ErrorStreamParse( + 'StreamMessageClosePayload does not contain a StreamShutdown', + ); + } + if (shutdown !== StreamShutdown.Read && shutdown !== StreamShutdown.Write) { + throw new errors.ErrorStreamParse( + `StreamMessageClosePayload contains an invalid StreamShutdown: ${shutdown}`, + ); + } + return { + data: shutdown, + remainder: input.subarray(1), + }; +} + +function parseStreamMessageErrorPayload( + input: Uint8Array, +): Parsed<{ shutdown: StreamShutdown; code: VarInt }> { + let remainder = input; + + const shutdown = input.at(0); + if (shutdown == null) { + throw new errors.ErrorStreamParse( + 'StreamMessageErrorPayload does not contain a StreamShutdown', + ); + } + if (shutdown !== StreamShutdown.Read && shutdown !== StreamShutdown.Write) { + throw new errors.ErrorStreamParse( + `StreamMessageErrorPayload contains an invalid StreamShutdown: ${shutdown}`, + ); + } + remainder = remainder.subarray(1); + + const { data: code, remainder: postCodeRemainder } = parseVarInt(remainder); + remainder = postCodeRemainder; + + return { + data: { + shutdown, + code, + }, + remainder, + }; +} + +function parseStreamMessage(input: Uint8Array): StreamMessage { + let remainder = input; + + const { data: type, remainder: postTypeRemainder } = + parseStreamMessageType(remainder); + remainder = postTypeRemainder; + + let payload: any; + if (type === StreamMessageType.Ack) { + const { data: ackPayload, remainder: postAckPayloadRemainder } = + parseStreamMessageAckPayload(remainder); + remainder = postAckPayloadRemainder; + payload = ackPayload; + } else if (type === StreamMessageType.Data) { + payload = remainder; + } else if (type === StreamMessageType.Close) { + const { data: closePayload, remainder: postClosePayloadRemainder } = + parseStreamMessageClosePayload(remainder); + remainder = postClosePayloadRemainder; + payload = closePayload; + } else if (type === StreamMessageType.Error) { + const { data: errorPayload, remainder: postErrorPayloadRemainder } = + parseStreamMessageErrorPayload(remainder); + remainder = postErrorPayloadRemainder; + payload = errorPayload; + } else { + never(); + } + + return { + type, + payload, + }; +} + +function generateStreamMessageType(type: StreamMessageType): Uint8Array { + return new Uint8Array([type]); +} + +function generateStreamMessageAckPayload(ackPayload: number): Uint8Array { + if (ackPayload > 0xffffffff) { + throw new errors.ErrorStreamGenerate( + 'StreamMessageAckPayload is too large', + ); + } + const array = new Uint8Array(4); + const dv = new DataView(array.buffer); + dv.setUint32(0, ackPayload, false); + return array; +} + +function generateStreamMessageClosePayload( + closePayload: StreamShutdown, +): Uint8Array { + return new Uint8Array([closePayload]); +} + +function generateStreamMessageErrorPayload(errorPayload: { + shutdown: StreamShutdown; + code: VarInt; +}): Uint8Array { + const generatedCode = generateVarInt(errorPayload.code); + const array = new Uint8Array(1 + generatedCode.length); + array[0] = errorPayload.shutdown; + array.set(generatedCode, 1); + return array; +} + +function generateStreamMessage(input: StreamMessage) { + const generatedType = generateStreamMessageType(input.type); + let generatedPayload: Uint8Array; + if (input.type === StreamMessageType.Ack) { + generatedPayload = generateStreamMessageAckPayload(input.payload); + } else if (input.type === StreamMessageType.Data) { + generatedPayload = input.payload; + } else if (input.type === StreamMessageType.Close) { + generatedPayload = generateStreamMessageClosePayload(input.payload); + } else if (input.type === StreamMessageType.Error) { + generatedPayload = generateStreamMessageErrorPayload(input.payload); + } else { + never(); + } + return concatUInt8Array(generatedType, generatedPayload); +} + +// Connection Message + +function parseConnectionMessage(input: Uint8Array): ConnectionMessage { + const { data: streamId, remainder } = parseStreamId(input); + const streamMessage = parseStreamMessage(remainder); + return { + streamId, + ...streamMessage, + }; +} + +function generateConnectionMessage(input: ConnectionMessage): Uint8Array { + const generatedStreamId = generateStreamId(input.streamId); + const generatedStreamMessage = generateStreamMessage(input); + return concatUInt8Array(generatedStreamId, generatedStreamMessage); +} + +export { + StreamMessageType, + StreamShutdown, + StreamErrorCode, + concatUInt8Array, + parseVarInt, + generateVarInt, + parseStreamId, + generateStreamId, + parseStreamMessageType, + parseStreamMessageAckPayload, + parseStreamMessageClosePayload, + parseStreamMessageErrorPayload, + parseStreamMessage, + generateStreamMessageType, + generateStreamMessageAckPayload, + generateStreamMessageClosePayload, + generateStreamMessageErrorPayload, + generateStreamMessage, + parseConnectionMessage, + generateConnectionMessage, +}; diff --git a/src/types.ts b/src/types.ts index 94c56352..6ada5e47 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,16 +26,6 @@ type PromiseDeconstructed = { type ConnectionId = Opaque<'ConnectionId', number>; -/** - * VarInt is a 62 bit unsigned integer - */ -type VarInt = Opaque<'VarInt', bigint>; - -/** - * StreamId is a VarInt - */ -type StreamId = VarInt; - /** * Host is always an IP address */ @@ -108,8 +98,6 @@ export type { Callback, PromiseDeconstructed, ConnectionId, - VarInt, - StreamId, Host, Hostname, Port, diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 00000000..c039bc16 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,24 @@ +import type { PromiseDeconstructed } from './types'; +import * as errors from './errors'; + +function never(message?: string): never { + throw new errors.ErrorWebSocketUndefinedBehaviour(message); +} + +/** + * Deconstructed promise + */ +function promise(): PromiseDeconstructed { + let resolveP, rejectP; + const p = new Promise((resolve, reject) => { + resolveP = resolve; + rejectP = reject; + }); + return { + p, + resolveP, + rejectP, + }; +} + +export { never, promise }; diff --git a/src/utils/types.ts b/src/utils/types.ts deleted file mode 100644 index 594a6925..00000000 --- a/src/utils/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Deconstructed promise - */ -type PromiseDeconstructed = { - p: Promise; - resolveP: (value: T | PromiseLike) => void; - rejectP: (reason?: any) => void; -}; - -export type { PromiseDeconstructed }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts deleted file mode 100644 index 60e9d6c2..00000000 --- a/src/utils/utils.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { PromiseDeconstructed } from './types'; -import type { Parsed, StreamId, VarInt } from '@/types'; -import * as errors from '../errors'; - -function never(message?: string): never { - throw new errors.ErrorWebSocketUndefinedBehaviour(message); -} - -/** - * Deconstructed promise - */ -function promise(): PromiseDeconstructed { - let resolveP, rejectP; - const p = new Promise((resolve, reject) => { - resolveP = resolve; - rejectP = reject; - }); - return { - p, - resolveP, - rejectP, - }; -} - -function toVarInt(array: Uint8Array): Parsed { - let streamId: bigint; - - // Get header and prefix - const header = array[0]; - const prefix = header >> 6; - - // Copy bytearray and remove prefix - const arrayCopy = new Uint8Array(array.length); - arrayCopy.set(array); - arrayCopy[0] &= 0b00111111; - - const dv = new DataView(arrayCopy.buffer, arrayCopy.byteOffset); - - let readBytes = 0; - - switch (prefix) { - case 0b00: - readBytes = 1; - streamId = BigInt(dv.getUint8(0)); - break; - case 0b01: - readBytes = 2; - streamId = BigInt(dv.getUint16(0, false)); - break; - case 0b10: - readBytes = 4; - streamId = BigInt(dv.getUint32(0, false)); - break; - case 0b11: - readBytes = 8; - streamId = dv.getBigUint64(0, false); - break; - } - return { - data: streamId! as VarInt, - remainder: array.subarray(readBytes), - }; -} - -function fromVarInt(varInt: VarInt): Uint8Array { - let array: Uint8Array; - let dv: DataView; - let prefixMask = 0; - - if (varInt < 0x40) { - array = new Uint8Array(1); - dv = new DataView(array.buffer); - dv.setUint8(0, Number(varInt)); - } else if (varInt < 0x4000) { - array = new Uint8Array(2); - dv = new DataView(array.buffer); - dv.setUint16(0, Number(varInt)); - prefixMask = 0b01_000000; - } else if (varInt < 0x40000000) { - array = new Uint8Array(4); - dv = new DataView(array.buffer); - dv.setUint32(0, Number(varInt)); - prefixMask = 0b10_000000; - } else { - array = new Uint8Array(8); - dv = new DataView(array.buffer); - dv.setBigUint64(0, varInt); - prefixMask = 0b11_000000; - } - - let header = dv.getUint8(0); - header |= prefixMask; - dv.setUint8(0, header); - - return array; -} - -const fromStreamId = fromVarInt as (streamId: StreamId) => Uint8Array; -const toStreamId = toVarInt as (array: Uint8Array) => Parsed; - -enum StreamMessageType { - DATA = 0, - ACK = 1, - ERROR = 2, - CLOSE = 3, -} - -enum StreamShutdown { - Read = 0, - Write = 1, -} - -enum StreamErrorCode { - Unknown = 0, - ErrorReadableStreamParse = 1, - ErrorReadableStreamBufferOverflow = 2, -} - -export { - never, - promise, - toVarInt, - fromVarInt, - toStreamId, - fromStreamId, - StreamMessageType, - StreamShutdown, - StreamErrorCode, -}; diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 1a3a1d6f..75eef06a 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -1,13 +1,13 @@ -import type { StreamId } from '@/types'; +import type { StreamId } from '@/message'; import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; import { fc, testProp } from '@fast-check/jest'; import WebSocketStream from '@/WebSocketStream'; import WebSocketConnection from '@/WebSocketConnection'; import * as events from '@/events'; -import { promise, StreamMessageType } from '@/utils'; -import * as config from '@/config'; +import { promise } from '@/utils'; import * as utils from '@/utils'; -import * as testUtils from './utils'; +import * as messageUtils from '@/message/utils'; +import { StreamMessageType } from '@/message'; // Smaller buffer size for the sake of testing const STREAM_BUFFER_SIZE = 64; @@ -42,8 +42,8 @@ jest.mock('@/WebSocketConnection', () => { let stream = instance.connectedConnection!.streamMap.get(streamId); if (stream == null) { if ( - data.at(0) === StreamMessageType.CLOSE || - data.at(0) === StreamMessageType.ERROR + data.at(0) === StreamMessageType.Close || + data.at(0) === StreamMessageType.Error ) { return; } @@ -130,9 +130,9 @@ describe(WebSocketStream.name, () => { () => { streamEndedCount += 1; if (streamEndedCount >= streamsNum) streamEndedProm.resolveP(); - } + }, ); - } + }, ); for (let i = 0; i < streamsNum; i++) { @@ -222,7 +222,7 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual(data); + expect(messageUtils.concatUInt8Array(...readChunks)).toEqual(data); await stream1.destroy(); await stream2.destroy(); @@ -261,8 +261,8 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual( - testUtils.concatUInt8Array(...data), + expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( + messageUtils.concatUInt8Array(...data), ); await stream1.destroy(); @@ -302,8 +302,8 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual( - testUtils.concatUInt8Array(...data), + expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( + messageUtils.concatUInt8Array(...data), ); await stream1.destroy(); @@ -350,8 +350,8 @@ describe(WebSocketStream.name, () => { await Promise.all([writeProm, readProm]); - expect(testUtils.concatUInt8Array(...readChunks)).toEqual( - testUtils.concatUInt8Array(...data), + expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( + messageUtils.concatUInt8Array(...data), ); await stream1.destroy(); @@ -406,8 +406,8 @@ describe(WebSocketStream.name, () => { data.reverse(); for (const [i, readResult] of readResults.entries()) { - expect(testUtils.concatUInt8Array(...readResult)).toEqual( - testUtils.concatUInt8Array(...data[i]), + expect(messageUtils.concatUInt8Array(...readResult)).toEqual( + messageUtils.concatUInt8Array(...data[i]), ); } diff --git a/tests/message/utils.test.ts b/tests/message/utils.test.ts new file mode 100644 index 00000000..32ea3330 --- /dev/null +++ b/tests/message/utils.test.ts @@ -0,0 +1,111 @@ +import type { ConnectionMessage, StreamMessage } from '@/message'; +import { fc, testProp } from '@fast-check/jest'; +import { + generateConnectionMessage, + generateStreamId, + generateStreamMessage, + generateStreamMessageAckPayload, + generateStreamMessageClosePayload, + generateStreamMessageErrorPayload, + generateStreamMessageType, + generateVarInt, + parseConnectionMessage, + parseStreamId, + parseStreamMessage, + parseStreamMessageAckPayload, + parseStreamMessageClosePayload, + parseStreamMessageErrorPayload, + parseStreamMessageType, + parseVarInt, +} from '@/message'; +import { + connectionMessageArb, + streamIdArb, + streamMessageAckPayloadArb, + streamMessageArb, + streamMessageClosePayloadArb, + streamMessageErrorPayloadArb, + streamMessageTypeArb, + varIntArb, +} from './utils'; + +describe('StreamMessage', () => { + testProp('should parse/generate VarInt', [varIntArb], (varInt) => { + const parsedVarInt = parseVarInt(generateVarInt(varInt)); + expect(parsedVarInt.data).toBe(varInt); + expect(parsedVarInt.remainder).toHaveLength(0); + }); + testProp('should parse/generate StreamId', [streamIdArb], (streamId) => { + const parsedStreamId = parseStreamId(generateStreamId(streamId)); + expect(parsedStreamId.data).toBe(streamId); + expect(parsedStreamId.remainder).toHaveLength(0); + }); + testProp( + 'should parse/generate StreamMessageType', + [streamMessageTypeArb], + (streamMessageType) => { + const parsedStreamMessageType = parseStreamMessageType( + generateStreamMessageType(streamMessageType), + ); + expect(parsedStreamMessageType.data).toBe(streamMessageType); + expect(parsedStreamMessageType.remainder).toHaveLength(0); + }, + ); + testProp( + 'should parse/generate StreamMessageAckPayload', + [streamMessageAckPayloadArb], + (ackPayload) => { + const parsedAckPayload = parseStreamMessageAckPayload( + generateStreamMessageAckPayload(ackPayload), + ); + expect(parsedAckPayload.data).toBe(ackPayload); + expect(parsedAckPayload.remainder).toHaveLength(0); + }, + ); + testProp( + 'should parse/generate StreamMessageClosePayload', + [streamMessageClosePayloadArb], + (closePayload) => { + const parsedClosePayload = parseStreamMessageClosePayload( + generateStreamMessageClosePayload(closePayload), + ); + expect(parsedClosePayload.data).toBe(closePayload); + expect(parsedClosePayload.remainder).toHaveLength(0); + }, + ); + testProp( + 'should parse/generate StreamMessageErrorPayload', + [streamMessageErrorPayloadArb], + (errorPayload) => { + const parsedClosePayload = parseStreamMessageErrorPayload( + generateStreamMessageErrorPayload(errorPayload), + ); + expect(parsedClosePayload.data).toEqual(errorPayload); + expect(parsedClosePayload.remainder).toHaveLength(0); + }, + ); + testProp( + 'should parse/generate StreamMessage', + [streamMessageArb], + (streamMessage) => { + const generatedStreamMessage = generateStreamMessage( + streamMessage as StreamMessage, + ); + const parsedStreamMessage = parseStreamMessage(generatedStreamMessage); + expect(parsedStreamMessage).toEqual(streamMessage); + }, + ); + testProp( + 'should parse/generate ConnectionMessage', + [connectionMessageArb], + (connectionMessage) => { + const generatedConnectionMessage = generateConnectionMessage( + connectionMessage as ConnectionMessage, + ); + const parsedConnectionMessage = parseConnectionMessage( + generatedConnectionMessage, + ); + expect(parsedConnectionMessage).toEqual(connectionMessage); + }, + ); +}); diff --git a/tests/message/utils.ts b/tests/message/utils.ts new file mode 100644 index 00000000..4444e642 --- /dev/null +++ b/tests/message/utils.ts @@ -0,0 +1,87 @@ +import type { StreamId, VarInt } from '@/message'; +import { fc } from '@fast-check/jest'; +import { + ConnectionMessage, + StreamMessageType, + StreamShutdown, +} from '@/message'; + +const varIntArb = fc.bigInt({ + min: 0n, + max: 2n ** 62n - 1n, +}) as fc.Arbitrary; + +const streamIdArb = varIntArb as fc.Arbitrary; + +const streamShutdownArb = fc.constantFrom( + StreamShutdown.Read, + StreamShutdown.Write, +); + +const streamMessageTypeArb = fc.constantFrom( + StreamMessageType.Ack, + StreamMessageType.Data, + StreamMessageType.Close, + StreamMessageType.Error, +); + +const streamMessageAckPayloadArb = fc.integer({ min: 0, max: 2 ** 32 - 1 }); + +const streamMessageClosePayloadArb = streamShutdownArb; + +const streamMessageErrorPayloadArb = fc.record({ + shutdown: streamShutdownArb, + code: varIntArb, +}); + +const streamMessageAckArb = fc.record({ + type: fc.constant(StreamMessageType.Ack), + payload: streamMessageAckPayloadArb, +}); + +const streamMessageDataArb = fc.record({ + type: fc.constant(StreamMessageType.Data), + payload: fc.uint8Array(), +}); + +const streamMessageCloseArb = fc.record({ + type: fc.constant(StreamMessageType.Close), + payload: streamMessageClosePayloadArb, +}); + +const streamMessageErrorArb = fc.record({ + type: fc.constant(StreamMessageType.Error), + payload: streamMessageErrorPayloadArb, +}); + +const streamMessageArb = fc.oneof( + streamMessageAckArb, + streamMessageDataArb, + streamMessageCloseArb, + streamMessageErrorArb, +); + +const connectionMessageArb = streamIdArb.chain((streamId) => { + return streamMessageArb.map((streamMessage) => { + return { + streamId, + ...streamMessage, + }; + }); +}); + +export { + varIntArb, + streamIdArb, + streamShutdownArb, + streamMessageTypeArb, + streamMessageAckPayloadArb, + streamMessageClosePayloadArb, + streamMessageErrorPayloadArb, + streamMessageAckArb, + streamMessageDataArb, + streamMessageCloseArb, + streamMessageErrorArb, + streamMessageArb, + connectionMessageArb, +}; diff --git a/tests/utils.ts b/tests/utils.ts index ec391eac..6ad771ce 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -599,17 +599,6 @@ function toReadableStream(iterator: IterableIterator) { }); } -function concatUInt8Array(...arrays: Array) { - const totalLength = arrays.reduce((acc, val) => acc + val.length, 0); - const result = new Uint8Array(totalLength); - let offset = 0; - for (const arr of arrays) { - result.set(arr, offset); - offset += arr.length; - } - return result; -} - export { sleep, randomBytes, @@ -626,7 +615,6 @@ export { verifyHMAC, generateConfig, toReadableStream, - concatUInt8Array, }; export type { KeyTypes, TLSConfigs }; From d65262ad0d7d2d68edb07007166831b59ec86ed0 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:58:41 +1000 Subject: [PATCH 049/149] lintfix --- src/WebSocketClient.ts | 5 ++--- tests/message/utils.ts | 6 +----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index b39cff90..5c26574b 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -10,7 +10,6 @@ import WebSocketConnection from './WebSocketConnection'; import WebSocketConnectionMap from './WebSocketConnectionMap'; import { clientDefault } from './config'; import * as events from './events'; -import * as utils from './utils'; interface WebSocketClient extends createDestroy.CreateDestroy {} /** @@ -170,12 +169,12 @@ class WebSocketClient extends EventTarget { }); connection.addEventListener( events.EventWebSocketConnectionStopped.name, - (event: events.EventWebSocketConnectionStopped) => { + async (event: events.EventWebSocketConnectionStopped) => { connection.removeEventListener( EventAll.name, client.handleEventWebSocketConnection, ); - client.handleEventWebSocketConnection(event); + await client.handleEventWebSocketConnection(event); }, { once: true }, ); diff --git a/tests/message/utils.ts b/tests/message/utils.ts index 4444e642..06b77efc 100644 --- a/tests/message/utils.ts +++ b/tests/message/utils.ts @@ -1,10 +1,6 @@ import type { StreamId, VarInt } from '@/message'; import { fc } from '@fast-check/jest'; -import { - ConnectionMessage, - StreamMessageType, - StreamShutdown, -} from '@/message'; +import { StreamMessageType, StreamShutdown } from '@/message'; const varIntArb = fc.bigInt({ min: 0n, From 1c1eeaa1f319a5af9b8a87aae51ff204923b8e6c Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:02:59 +1000 Subject: [PATCH 050/149] fix: exported every thing at root on index.ts --- src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/index.ts b/src/index.ts index 9a61d906..b7a9cdd9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,10 @@ export { default as WebSocketClient } from './WebSocketClient'; export { default as WebSocketConnection } from './WebSocketConnection'; export { default as WebSocketStream } from './WebSocketStream'; +export * as types from './types'; export * as utils from './utils'; export * as events from './events'; export * as errors from './errors'; +export * as config from './config'; +export * as message from './message'; + From b6c4b7fb967947c9c2e8898a2c29f02f7f5cb4da Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:01:10 +1000 Subject: [PATCH 051/149] feat: WebSocketStreamQueue --- src/WebSocketStreamQueue.ts | 64 ++++++++++++++++++++++++++++++ src/index.ts | 1 + tests/WebSocketStreamQueue.test.ts | 37 +++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 src/WebSocketStreamQueue.ts create mode 100644 tests/WebSocketStreamQueue.test.ts diff --git a/src/WebSocketStreamQueue.ts b/src/WebSocketStreamQueue.ts new file mode 100644 index 00000000..615260e4 --- /dev/null +++ b/src/WebSocketStreamQueue.ts @@ -0,0 +1,64 @@ +class WebSocketStreamQueue { + protected head?: WebSocketStreamQueueItem; + protected tail?: WebSocketStreamQueueItem; + protected _byteLength: number; + protected _length: number; + protected _count: number; + + public get byteLength(): Readonly { + return this._byteLength; + } + public get length(): Readonly { + return this._length + } + public get count(): Readonly { + return this._count; + } + constructor() { + this._byteLength = 0; + this._length = 0; + this._count = 0; + } + public queue(data: Uint8Array) { + const item = { + data + }; + if (this.head == null) { + this.head = item; + } + if (this.tail != null) { + this.tail.next = item; + } + this.tail = item; + this._byteLength += data.byteLength; + this._length += data.length; + this._count++; + } + public dequeue() { + const oldData = this.head?.data; + const newHead = this.head?.next; + if (this.head === this.tail) { + delete this.tail; + } + delete this.head; + this.head = newHead; + this._count = this._count === 0 ? 0 : this._count - 1; + this._byteLength -= oldData?.byteLength ?? 0; + this._length -= oldData?.length ?? 0; + return oldData; + } + public clear() { + this._byteLength = 0; + this._length = 0; + this._count = 0; + delete this.head; + delete this.tail; + } +} + +type WebSocketStreamQueueItem = { + data: Uint8Array; + next?: WebSocketStreamQueueItem; +} + +export default WebSocketStreamQueue; diff --git a/src/index.ts b/src/index.ts index b7a9cdd9..47747c2f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ export { default as WebSocketServer } from './WebSocketServer'; export { default as WebSocketClient } from './WebSocketClient'; export { default as WebSocketConnection } from './WebSocketConnection'; export { default as WebSocketStream } from './WebSocketStream'; +export { default as WebSocketStreamQueue } from './WebSocketStreamQueue'; export * as types from './types'; export * as utils from './utils'; diff --git a/tests/WebSocketStreamQueue.test.ts b/tests/WebSocketStreamQueue.test.ts new file mode 100644 index 00000000..aebe7fd7 --- /dev/null +++ b/tests/WebSocketStreamQueue.test.ts @@ -0,0 +1,37 @@ +import WebSocketStreamQueue from "@/WebSocketStreamQueue"; +import { fc, testProp } from "@fast-check/jest"; + +describe(WebSocketStreamQueue.name, () => { + testProp( + "should queue items", + [fc.array(fc.uint8Array())], + (array) => { + const queue = new WebSocketStreamQueue(); + let totalLength = 0; + let totalByteLength = 0; + for (const buffer of array) { + queue.queue(buffer); + totalByteLength += buffer.byteLength; + totalLength += buffer.length; + } + expect(queue.count).toBe(array.length); + expect(queue.byteLength).toBe(totalByteLength); + expect(queue.length).toBe(totalLength); + } + ); + testProp( + "should dequeue items", + [fc.array(fc.uint8Array())], + (array) => { + const queue = new WebSocketStreamQueue(); + for (const buffer of array) { + queue.queue(buffer); + } + const result: Array = []; + for (let i = 0; i < array.length; i++) { + result.push(queue.dequeue()!); + } + expect(result).toEqual(array); + } + ); +}); From 8c1ddfcf488320a9054565c9d3accc71e683e4f0 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:01:36 +1000 Subject: [PATCH 052/149] feat: changed backpressure to work additively rather than by setting desired size --- src/WebSocketStream.ts | 59 +++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index f6e129a5..e7e26661 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -13,6 +13,13 @@ import { StreamMessageType, StreamShutdown, } from './message'; +import { + ReadableStream, + WritableStream, + CountQueuingStrategy, + ReadableByteStreamController, +} from 'stream/web'; +import WebSocketStreamQueue from './WebSocketStreamQueue'; interface WebSocketStream extends CreateDestroy {} /** @@ -36,9 +43,12 @@ class WebSocketStream implements ReadableWritablePair { protected connection: WebSocketConnection; protected reasonToCode: StreamReasonToCode; protected codeToReason: StreamCodeToReason; - protected readableController: ReadableStreamController; + protected readableController: ReadableStreamDefaultController; protected writableController: WritableStreamDefaultController; + protected readableQueue: WebSocketStreamQueue = new WebSocketStreamQueue(); + protected readableQueueBufferSize = 0; + protected readableBufferReadable = promise(); protected writableDesiredSize = 0; protected writableDesiredSizeProm = promise(); @@ -95,6 +105,10 @@ class WebSocketStream implements ReadableWritablePair { this.reasonToCode = reasonToCode; this.codeToReason = codeToReason; + this.readableQueueBufferSize = bufferSize; + + let initAckSent = false; + this.readable = new ReadableStream( { start: async (controller) => { @@ -105,14 +119,31 @@ class WebSocketStream implements ReadableWritablePair { if (this._readableEnded) { return; } - // If desiredSize is less than or equal to 0, it means that the buffer is still full after a read - if (controller.desiredSize != null && controller.desiredSize <= 0) { + + if (!initAckSent) { + await this.streamSend({ + type: StreamMessageType.Ack, + payload: bufferSize, + }); + initAckSent = true; return; } - // Send ACK on every read as there will be more usable space on the buffer. + + if (this.readableQueue.count === 0) { + this.readableBufferReadable.resolveP(); + this.readableBufferReadable = promise(); + } + await this.readableBufferReadable.p; + + const data = this.readableQueue.dequeue(); + const readBytes = data!.length; + controller.enqueue(data!); + + this.logger.debug(`${readBytes} bytes have been pushed onto stream buffer`) + await this.streamSend({ type: StreamMessageType.Ack, - payload: controller.desiredSize!, + payload: readBytes, }); }, cancel: async (reason) => { @@ -120,9 +151,9 @@ class WebSocketStream implements ReadableWritablePair { await this.signalReadableEnd(true, reason); }, }, - new ByteLengthQueuingStrategy({ - highWaterMark: bufferSize, - }), + { + highWaterMark: 1, + } ); const writeHandler = async ( @@ -155,6 +186,7 @@ class WebSocketStream implements ReadableWritablePair { } // Decrement the desired size by the amount of bytes written this.writableDesiredSize -= bytesWritten; + this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to write`); await this.streamSend({ type: StreamMessageType.Data, payload: data, @@ -247,15 +279,15 @@ class WebSocketStream implements ReadableWritablePair { } if (parsedMessage.type === StreamMessageType.Ack) { - this.writableDesiredSize = parsedMessage.payload; + this.writableDesiredSize += parsedMessage.payload; this.writableDesiredSizeProm.resolveP(); + this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to ACK`); } else if (parsedMessage.type === StreamMessageType.Data) { if (this._readableEnded) { return; } if ( - this.readableController.desiredSize != null && - parsedMessage.payload.length > this.readableController.desiredSize + parsedMessage.payload.length > (this.readableQueueBufferSize - this.readableQueue.length) ) { await this.signalReadableEnd( true, @@ -263,7 +295,8 @@ class WebSocketStream implements ReadableWritablePair { ); return; } - this.readableController.enqueue(parsedMessage.payload); + this.readableQueue.queue(parsedMessage.payload); + this.readableBufferReadable.resolveP(); } else if (parsedMessage.type === StreamMessageType.Error) { const { shutdown, code } = parsedMessage.payload; let reason: any; @@ -369,6 +402,8 @@ class WebSocketStream implements ReadableWritablePair { payload: StreamShutdown.Write, }); } + // clear the readable queue + this.readableQueue.clear(); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); if (this[status] !== 'destroying') await this.destroy(); From 34fbee3d26f587059daa3c24d8115e920db7759a Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:12:51 +1000 Subject: [PATCH 053/149] fix: WebSocketStream shutdowns WebSocketStreamQueue --- src/WebSocketStream.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index e7e26661..8443e2ac 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -48,7 +48,7 @@ class WebSocketStream implements ReadableWritablePair { protected readableQueue: WebSocketStreamQueue = new WebSocketStreamQueue(); protected readableQueueBufferSize = 0; - protected readableBufferReadable = promise(); + protected readableBufferReady = promise(); protected writableDesiredSize = 0; protected writableDesiredSizeProm = promise(); @@ -129,15 +129,22 @@ class WebSocketStream implements ReadableWritablePair { return; } + // reset the promise before a read from the queue to wait until the queue has items if (this.readableQueue.count === 0) { - this.readableBufferReadable.resolveP(); - this.readableBufferReadable = promise(); + this.readableBufferReady.resolveP(); + this.readableBufferReady = promise(); } - await this.readableBufferReadable.p; + await this.readableBufferReady.p; + // data will be null in the case of stream destruction before the readable buffer is blocked + // we're going to just enqueue an empty buffer in case it is null for some other reason, so that the next read is able to complete const data = this.readableQueue.dequeue(); - const readBytes = data!.length; - controller.enqueue(data!); + if (data == null) { + controller.enqueue(new Uint8Array(0)); + return; + } + const readBytes = data.length; + controller.enqueue(data); this.logger.debug(`${readBytes} bytes have been pushed onto stream buffer`) @@ -296,7 +303,7 @@ class WebSocketStream implements ReadableWritablePair { return; } this.readableQueue.queue(parsedMessage.payload); - this.readableBufferReadable.resolveP(); + this.readableBufferReady.resolveP(); } else if (parsedMessage.type === StreamMessageType.Error) { const { shutdown, code } = parsedMessage.payload; let reason: any; @@ -374,6 +381,8 @@ class WebSocketStream implements ReadableWritablePair { if (this._readableEnded) return; // Indicate that receiving side is closed this._readableEnded = true; + // Resolve readable promise in case blocking + this.readableBufferReady.resolveP(); // Shutdown the write side of the other stream if (isError) { let code: VarInt; From 188f4ab30df14e908416783dd289b2d37a41fff2 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:34:11 +1000 Subject: [PATCH 054/149] fix: changed 'delete's in WebSocketStreamQueue to set to undefined for performance --- src/WebSocketStreamQueue.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/WebSocketStreamQueue.ts b/src/WebSocketStreamQueue.ts index 615260e4..164fd38b 100644 --- a/src/WebSocketStreamQueue.ts +++ b/src/WebSocketStreamQueue.ts @@ -38,9 +38,8 @@ class WebSocketStreamQueue { const oldData = this.head?.data; const newHead = this.head?.next; if (this.head === this.tail) { - delete this.tail; + this.tail = undefined; } - delete this.head; this.head = newHead; this._count = this._count === 0 ? 0 : this._count - 1; this._byteLength -= oldData?.byteLength ?? 0; @@ -51,8 +50,8 @@ class WebSocketStreamQueue { this._byteLength = 0; this._length = 0; this._count = 0; - delete this.head; - delete this.tail; + this.head = undefined; + this.tail = undefined; } } From 94768886a97c28209dba44b3cd983aa14339aa55 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:20:05 +1000 Subject: [PATCH 055/149] fix: WebSocketStream WritableStream write is now looped rather than recursive --- src/WebSocketStream.ts | 80 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 8443e2ac..8dfca445 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -163,53 +163,49 @@ class WebSocketStream implements ReadableWritablePair { } ); - const writeHandler = async ( - chunk: Uint8Array, - controller: WritableStreamDefaultController, - ) => { - // Do not bother to write or wait for ACK if the writable has ended - if (this._writableEnded) { - return; - } - await this.writableDesiredSizeProm.p; - this.logger.debug( - `${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`, - ); - let data: Uint8Array; - const isChunkable = chunk.length > this.writableDesiredSize; - if (isChunkable) { - this.logger.debug( - `this chunk will be split into sizes of ${this.writableDesiredSize} bytes`, - ); - data = chunk.subarray(0, this.writableDesiredSize); - } else { - data = chunk; - } - const bytesWritten = data.length; - if (this.writableDesiredSize === bytesWritten) { - this.logger.debug(`this chunk will trigger receiver to send an ACK`); - // Reset the promise to wait for another ACK - this.writableDesiredSizeProm = promise(); - } - // Decrement the desired size by the amount of bytes written - this.writableDesiredSize -= bytesWritten; - this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to write`); - await this.streamSend({ - type: StreamMessageType.Data, - payload: data, - }); - - if (isChunkable) { - await writeHandler(chunk.subarray(bytesWritten), controller); - } - }; - this.writable = new WritableStream( { start: (controller) => { this.writableController = controller; }, - write: writeHandler, + write: async (chunk) => { + while (chunk.length > 0) { + // Do not bother to write or wait for ACK if the writable has ended + if (this._writableEnded) { + return; + } + this.logger.debug( + `${chunk.length} bytes need to be written into a receiver buffer of ${this.writableDesiredSize} bytes`, + ); + + await this.writableDesiredSizeProm.p; + + // chunking + let data: Uint8Array; + if (chunk.length > this.writableDesiredSize) { + this.logger.debug( + `this chunk will be split into sizes of ${this.writableDesiredSize} bytes`, + ); + } + // .subarray parameters begin and end are clamped to the size of the Uint8Array + data = chunk.subarray(0, this.writableDesiredSize); + + const bytesWritten = data.length; + if (this.writableDesiredSize === bytesWritten) { + this.logger.debug(`this chunk will trigger receiver to send an ACK`); + // Reset the promise to wait for another ACK + this.writableDesiredSizeProm = promise(); + } + // Decrement the desired size by the amount of bytes written + this.writableDesiredSize -= bytesWritten; + this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to write`); + await this.streamSend({ + type: StreamMessageType.Data, + payload: data, + }); + chunk = chunk.subarray(bytesWritten); + } + }, close: async () => { await this.signalWritableEnd(); }, From 8e66969ae7aecfc158fc37daee45ab9b9a3fcece Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:23:20 +1000 Subject: [PATCH 056/149] fix: readable queue is now cleared earlier on closing ReadableStream --- src/WebSocketStream.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 8dfca445..9ab7e67d 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -379,6 +379,8 @@ class WebSocketStream implements ReadableWritablePair { this._readableEnded = true; // Resolve readable promise in case blocking this.readableBufferReady.resolveP(); + // clear the readable queue + this.readableQueue.clear(); // Shutdown the write side of the other stream if (isError) { let code: VarInt; @@ -407,8 +409,6 @@ class WebSocketStream implements ReadableWritablePair { payload: StreamShutdown.Write, }); } - // clear the readable queue - this.readableQueue.clear(); if (this._readableEnded && this._writableEnded) { this.destroyProm.resolveP(); if (this[status] !== 'destroying') await this.destroy(); From ba2c3af4ca8e91ea233dbd05a8a5964dc11649de Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:58:15 +1000 Subject: [PATCH 057/149] chore: WebSocketStreamQueue documentation --- src/WebSocketStreamQueue.ts | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/WebSocketStreamQueue.ts b/src/WebSocketStreamQueue.ts index 164fd38b..5c50ae9b 100644 --- a/src/WebSocketStreamQueue.ts +++ b/src/WebSocketStreamQueue.ts @@ -1,3 +1,7 @@ +// WebSocketStreamQueue can have 3 states regarding the head and the tail +// if (head == null && head === tail) then the queue is empty +// if (head != null && head === tail) then the queue has 1 item +// if (head != null && head !== tail) then the queue has 2 or more items class WebSocketStreamQueue { protected head?: WebSocketStreamQueueItem; protected tail?: WebSocketStreamQueueItem; @@ -5,51 +9,73 @@ class WebSocketStreamQueue { protected _length: number; protected _count: number; + /** + * The combined byteLength of all queued `Uint8Array`. + */ public get byteLength(): Readonly { return this._byteLength; } + /** + * The combined length of the queued `Uint8Array`s. + */ public get length(): Readonly { return this._length } + /** + * The number of queued `Uint8Array`. + */ public get count(): Readonly { return this._count; } + constructor() { this._byteLength = 0; this._length = 0; this._count = 0; } - public queue(data: Uint8Array) { + public queue(data: Uint8Array): void { const item = { data }; + // if there is no head, then this is the first item in the queue if (this.head == null) { this.head = item; } + // if the tail exists, then set the next item on the tail to the new item if (this.tail != null) { this.tail.next = item; } + // set the tail to the new item this.tail = item; + // update the byteLength, length, and count this._byteLength += data.byteLength; this._length += data.length; this._count++; } - public dequeue() { + /** + * Returns the data of the head and removes the head from the queue. + * If the queue is empty, then undefined is returned. + */ + public dequeue(): Uint8Array | undefined { + // get the data of the head const oldData = this.head?.data; const newHead = this.head?.next; + // if the head and the tail are the same, then the queue is either empty or only have one item if (this.head === this.tail) { this.tail = undefined; } this.head = newHead; + // decrement the count, but don't let it go below 0 in case the queue is empty this._count = this._count === 0 ? 0 : this._count - 1; this._byteLength -= oldData?.byteLength ?? 0; this._length -= oldData?.length ?? 0; return oldData; } - public clear() { + public clear(): void { this._byteLength = 0; this._length = 0; this._count = 0; + // clearing head and tail should cause the garbage collector to clean up all the items in the queue this.head = undefined; this.tail = undefined; } From 0b282586e0609bd477faa5f0e22f5dff5aa192af Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:03:05 +1000 Subject: [PATCH 058/149] fix: WebSocketStreamQueue dequeue tests to include byteLengths --- tests/WebSocketStreamQueue.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/WebSocketStreamQueue.test.ts b/tests/WebSocketStreamQueue.test.ts index aebe7fd7..d94179a9 100644 --- a/tests/WebSocketStreamQueue.test.ts +++ b/tests/WebSocketStreamQueue.test.ts @@ -32,6 +32,9 @@ describe(WebSocketStreamQueue.name, () => { result.push(queue.dequeue()!); } expect(result).toEqual(array); + expect(queue.count).toBe(0) + expect(queue.byteLength).toBe(0); + expect(queue.length).toBe(0); } ); }); From 425a58ad4a4bd117fa682d84b665d8d925ef76dd Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:12:09 +1000 Subject: [PATCH 059/149] chore: execute WebSocketStream test write/read functions more closely --- tests/WebSocketStream.test.ts | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 75eef06a..c8a8ad6c 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -169,21 +169,21 @@ describe(WebSocketStream.name, () => { const writer = stream2Writable.getWriter(); const reader = stream1Readable.getReader(); - const writeProm = (async () => { + const writeF = async () => { await writer.write(data); await writer.close(); - })(); + }; const readChunks: Array = []; - const readProm = (async () => { + const readF = async () => { while (true) { const { done, value } = await reader.read(); if (done) break; readChunks.push(value); } - })(); + }; - await Promise.all([writeProm, readProm]); + await Promise.all([writeF(), readF()]); expect(readChunks).toEqual([data]); @@ -206,21 +206,21 @@ describe(WebSocketStream.name, () => { const writer = stream2Writable.getWriter(); const reader = stream1Readable.getReader(); - const writeProm = (async () => { + const writeF = async () => { await writer.write(data); await writer.close(); - })(); + }; const readChunks: Array = []; - const readProm = (async () => { + const readF = async () => { while (true) { const { done, value } = await reader.read(); if (done) break; readChunks.push(value); } - })(); + }; - await Promise.all([writeProm, readProm]); + await Promise.all([writeF(), readF()]); expect(messageUtils.concatUInt8Array(...readChunks)).toEqual(data); @@ -243,23 +243,23 @@ describe(WebSocketStream.name, () => { const writer = stream2Writable.getWriter(); const reader = stream1Readable.getReader(); - const writeProm = (async () => { + const writeF = async () => { for (const chunk of data) { await writer.write(chunk); } await writer.close(); - })(); + }; const readChunks: Array = []; - const readProm = (async () => { + const readProm = async () => { while (true) { const { done, value } = await reader.read(); if (done) break; readChunks.push(value); } - })(); + }; - await Promise.all([writeProm, readProm]); + await Promise.all([writeF(), readProm()]); expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( messageUtils.concatUInt8Array(...data), @@ -284,23 +284,23 @@ describe(WebSocketStream.name, () => { const writer = stream2Writable.getWriter(); const reader = stream1Readable.getReader(); - const writeProm = (async () => { + const writeF = async () => { for (const chunk of data) { await writer.write(chunk); } await writer.close(); - })(); + }; const readChunks: Array = []; - const readProm = (async () => { + const readF = async () => { while (true) { const { done, value } = await reader.read(); if (done) break; readChunks.push(value); } - })(); + }; - await Promise.all([writeProm, readProm]); + await Promise.all([writeF(), readF()]); expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( messageUtils.concatUInt8Array(...data), @@ -332,23 +332,23 @@ describe(WebSocketStream.name, () => { const writer = stream2Writable.getWriter(); const reader = stream1Readable.getReader(); - const writeProm = (async () => { + const writeF = async () => { for (const chunk of data) { await writer.write(chunk); } await writer.close(); - })(); + }; const readChunks: Array = []; - const readProm = (async () => { + const readF = async () => { while (true) { const { done, value } = await reader.read(); if (done) break; readChunks.push(value); } - })(); + }; - await Promise.all([writeProm, readProm]); + await Promise.all([writeF(), readF()]); expect(messageUtils.concatUInt8Array(...readChunks)).toEqual( messageUtils.concatUInt8Array(...data), @@ -383,13 +383,13 @@ describe(WebSocketStream.name, () => { for (const [i, stream] of streams.entries()) { const reader = stream.readable.getReader(); const writer = stream.writable.getWriter(); - const writeProm = (async () => { + const writeF = async () => { for (const chunk of data[i]) { await writer.write(chunk); } await writer.close(); - })(); - const readProm = (async () => { + }; + const readF = async () => { const readChunks: Array = []; while (true) { const { done, value } = await reader.read(); @@ -397,9 +397,9 @@ describe(WebSocketStream.name, () => { readChunks.push(value); } return readChunks; - })(); - readProms.push(readProm); - writeProms.push(writeProm); + }; + readProms.push(readF()); + writeProms.push(writeF()); } await Promise.all(writeProms); const readResults = await Promise.all(readProms); From c34755f60e614c156a809809d9a89980aa04ded9 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:52:51 +1000 Subject: [PATCH 060/149] fix: readableController is automatically closed off of error in signalReadableEnd --- src/WebSocketStream.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 9ab7e67d..0f39dfdd 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -318,15 +318,8 @@ class WebSocketStream implements ReadableWritablePair { reason = await this.codeToReason('recv', code); } if (shutdown === StreamShutdown.Read) { - if (this._readableEnded) { - return; - } await this.signalReadableEnd(true, reason); - this.readableController.close(); } else if (shutdown === StreamShutdown.Write) { - if (this._writableEnded) { - return; - } await this.signalWritableEnd(true, reason); } } else if (parsedMessage.type === StreamMessageType.Close) { From e2b46461d3d77ab9ecf6cc603c66ac5ec0b48f66 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:57:00 +1000 Subject: [PATCH 061/149] feat: error propagation test for WebSocketStream --- tests/WebSocketStream.test.ts | 84 ++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index c8a8ad6c..74cd4e2e 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -9,6 +9,8 @@ import * as utils from '@/utils'; import * as messageUtils from '@/message/utils'; import { StreamMessageType } from '@/message'; +type StreamOptions = Partial[0]>; + // Smaller buffer size for the sake of testing const STREAM_BUFFER_SIZE = 64; @@ -25,7 +27,7 @@ const logger2 = new Logger('stream 2', LogLevel.WARN, [ ]); jest.mock('@/WebSocketConnection', () => { - return jest.fn().mockImplementation(() => { + return jest.fn().mockImplementation((streamOptions: StreamOptions = {}) => { const instance = new EventTarget() as EventTarget & { connectedConnection: WebSocketConnection | undefined; connectTo: (connection: WebSocketConnection) => void; @@ -52,6 +54,7 @@ jest.mock('@/WebSocketConnection', () => { bufferSize: STREAM_BUFFER_SIZE, connection: instance.connectedConnection!, logger: logger2, + ...streamOptions }); instance.connectedConnection!.dispatchEvent( new events.EventWebSocketConnectionStream({ @@ -68,27 +71,31 @@ jest.mock('@/WebSocketConnection', () => { const connectionMock = jest.mocked(WebSocketConnection, true); describe(WebSocketStream.name, () => { - let connection1: WebSocketConnection; - let connection2: WebSocketConnection; let streamIdCounter = 0n; beforeEach(async () => { connectionMock.mockClear(); streamIdCounter = 0n; - connection1 = new (WebSocketConnection as any)(); - connection2 = new (WebSocketConnection as any)(); - (connection1 as any).connectTo(connection2); }); - async function createStreamPair( + async function createConnectionPair(streamOptions: StreamOptions = {}): Promise<[WebSocketConnection, WebSocketConnection]> { + const connection1 = new (WebSocketConnection as any)(streamOptions); + const connection2 = new (WebSocketConnection as any)(streamOptions); + (connection1 as any).connectTo(connection2); + return [connection1, connection2]; + } + + async function createStreamPairFrom( connection1: WebSocketConnection, connection2: WebSocketConnection, - ) { + streamOptions: StreamOptions = {}, + ): Promise<[WebSocketStream, WebSocketStream]> { const stream1 = await WebSocketStream.createWebSocketStream({ streamId: streamIdCounter as StreamId, bufferSize: STREAM_BUFFER_SIZE, connection: connection1, logger: logger1, + ...streamOptions }); const createStream2Prom = promise(); connection2.addEventListener( @@ -102,14 +109,21 @@ describe(WebSocketStream.name, () => { streamIdCounter++; return [stream1, stream2]; } + + async function createStreamPair(streamOptions: StreamOptions = {}) { + const [connection1, connection2] = await createConnectionPair(streamOptions); + return createStreamPairFrom(connection1, connection2, streamOptions); + } + test('should create stream', async () => { - const streams = await createStreamPair(connection1, connection2); + const streams = await createStreamPair(); expect(streams.length).toEqual(2); for (const stream of streams) { await stream.destroy(); } }); test('destroying stream should clean up on both ends while streams are used', async () => { + const [connection1, connection2] = await createConnectionPair(); const streamsNum = 10; let streamCreatedCount = 0; @@ -154,14 +168,36 @@ describe(WebSocketStream.name, () => { await stream.destroy(); } }); + test( + 'should propagate errors over stream for writable', + async () => { + const testReason = Symbol('TestReason'); + const codeToReason = (type, code: bigint) => { + switch (code) { + case 4002n: + return testReason; + default: + return new Error(`${type.toString()} ${code.toString()}`); + } + }; + const reasonToCode = (type, reason) => { + if (reason === testReason) return 4002n; + return 0n; + }; + const [stream1, stream2] = await createStreamPair({ codeToReason, reasonToCode }); + + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + await stream2Writable.abort(testReason); + await expect(stream1Readable.getReader().read()).rejects.toBe(testReason); + } + ); testProp( 'should send data over stream - single write within buffer size', [fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE })], async (data) => { - const [stream1, stream2] = await createStreamPair( - connection1, - connection2, - ); + data = new Uint8Array(0); + const [stream1, stream2] = await createStreamPair(); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -195,10 +231,7 @@ describe(WebSocketStream.name, () => { 'should send data over stream - single write outside buffer size', [fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 })], async (data) => { - const [stream1, stream2] = await createStreamPair( - connection1, - connection2, - ); + const [stream1, stream2] = await createStreamPair(); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -232,10 +265,7 @@ describe(WebSocketStream.name, () => { 'should send data over stream - multiple writes within buffer size', [fc.array(fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE }))], async (data) => { - const [stream1, stream2] = await createStreamPair( - connection1, - connection2, - ); + const [stream1, stream2] = await createStreamPair(); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -273,10 +303,7 @@ describe(WebSocketStream.name, () => { 'should send data over stream - multiple writes outside buffer size', [fc.array(fc.uint8Array({ minLength: STREAM_BUFFER_SIZE + 1 }))], async (data) => { - const [stream1, stream2] = await createStreamPair( - connection1, - connection2, - ); + const [stream1, stream2] = await createStreamPair(); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -321,10 +348,7 @@ describe(WebSocketStream.name, () => { ), ], async (data) => { - const [stream1, stream2] = await createStreamPair( - connection1, - connection2, - ); + const [stream1, stream2] = await createStreamPair(); const stream1Readable = stream1.readable; const stream2Writable = stream2.writable; @@ -375,7 +399,7 @@ describe(WebSocketStream.name, () => { ), ], async (...data) => { - const streams = await createStreamPair(connection1, connection2); + const streams = await createStreamPair(); const readProms: Array>> = []; const writeProms: Array> = []; From 7367eebc33d7779e852f387ad3765d1fdafac277 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:56:43 +1000 Subject: [PATCH 062/149] fix: closing ReadableStream should correctly close the opposing WritableStream --- src/WebSocketStream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 0f39dfdd..5110453d 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -391,7 +391,7 @@ class WebSocketStream implements ReadableWritablePair { await this.streamSend({ type: StreamMessageType.Error, payload: { - shutdown: StreamShutdown.Read, + shutdown: StreamShutdown.Write, code, }, }); From ec2218f24fe401250b4549f11e34a774985cb62e Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:24:12 +1000 Subject: [PATCH 063/149] feat: stream cancelling tests --- tests/WebSocketStream.test.ts | 151 +++++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 28 deletions(-) diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index 74cd4e2e..fda4d65d 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -9,7 +9,9 @@ import * as utils from '@/utils'; import * as messageUtils from '@/message/utils'; import { StreamMessageType } from '@/message'; -type StreamOptions = Partial[0]>; +type StreamOptions = Partial< + Parameters[0] +>; // Smaller buffer size for the sake of testing const STREAM_BUFFER_SIZE = 64; @@ -54,7 +56,7 @@ jest.mock('@/WebSocketConnection', () => { bufferSize: STREAM_BUFFER_SIZE, connection: instance.connectedConnection!, logger: logger2, - ...streamOptions + ...streamOptions, }); instance.connectedConnection!.dispatchEvent( new events.EventWebSocketConnectionStream({ @@ -78,7 +80,9 @@ describe(WebSocketStream.name, () => { streamIdCounter = 0n; }); - async function createConnectionPair(streamOptions: StreamOptions = {}): Promise<[WebSocketConnection, WebSocketConnection]> { + async function createConnectionPair( + streamOptions: StreamOptions = {}, + ): Promise<[WebSocketConnection, WebSocketConnection]> { const connection1 = new (WebSocketConnection as any)(streamOptions); const connection2 = new (WebSocketConnection as any)(streamOptions); (connection1 as any).connectTo(connection2); @@ -95,7 +99,7 @@ describe(WebSocketStream.name, () => { bufferSize: STREAM_BUFFER_SIZE, connection: connection1, logger: logger1, - ...streamOptions + ...streamOptions, }); const createStream2Prom = promise(); connection2.addEventListener( @@ -111,7 +115,9 @@ describe(WebSocketStream.name, () => { } async function createStreamPair(streamOptions: StreamOptions = {}) { - const [connection1, connection2] = await createConnectionPair(streamOptions); + const [connection1, connection2] = await createConnectionPair( + streamOptions, + ); return createStreamPairFrom(connection1, connection2, streamOptions); } @@ -168,30 +174,31 @@ describe(WebSocketStream.name, () => { await stream.destroy(); } }); - test( - 'should propagate errors over stream for writable', - async () => { - const testReason = Symbol('TestReason'); - const codeToReason = (type, code: bigint) => { - switch (code) { - case 4002n: - return testReason; - default: - return new Error(`${type.toString()} ${code.toString()}`); - } - }; - const reasonToCode = (type, reason) => { - if (reason === testReason) return 4002n; - return 0n; - }; - const [stream1, stream2] = await createStreamPair({ codeToReason, reasonToCode }); + test('should propagate errors over stream for writable', async () => { + const testReason = Symbol('TestReason'); + const codeToReason = (type, code: bigint) => { + switch (code) { + case 4002n: + return testReason; + default: + return new Error(`${type.toString()} ${code.toString()}`); + } + }; + const reasonToCode = (type, reason) => { + if (reason === testReason) return 4002n; + return 0n; + }; + const [stream1, stream2] = await createStreamPair({ + codeToReason, + reasonToCode, + }); - const stream1Readable = stream1.readable; - const stream2Writable = stream2.writable; - await stream2Writable.abort(testReason); - await expect(stream1Readable.getReader().read()).rejects.toBe(testReason); - } - ); + const stream1Readable = stream1.readable; + const stream2Writable = stream2.writable; + await stream2Writable.abort(testReason); + await expect(stream1Readable.getReader().read()).rejects.toBe(testReason); + await expect(stream2Writable.getWriter().write()).rejects.toBe(testReason); + }); testProp( 'should send data over stream - single write within buffer size', [fc.uint8Array({ maxLength: STREAM_BUFFER_SIZE })], @@ -440,4 +447,92 @@ describe(WebSocketStream.name, () => { } }, ); + test('streams can be cancelled after data sent', async () => { + const cancelReason = Symbol('CancelReason'); + const codeToReason = (type, code: bigint) => { + switch (code) { + case 4001n: + return cancelReason; + default: + return new Error(`${type.toString()} ${code.toString()}`); + } + }; + const reasonToCode = (_type, reason) => { + if (reason === cancelReason) return 4001n; + return 0n; + }; + const [_stream1, stream2] = await createStreamPair({ + codeToReason, + reasonToCode, + }); + + const writer = stream2.writable.getWriter(); + await writer.write(new Uint8Array(2)); + writer.releaseLock(); + await stream2.cancel(cancelReason); + + await expect(stream2.readable.getReader().read()).rejects.toBe( + cancelReason, + ); + await expect(stream2.writable.getWriter().write()).rejects.toBe( + cancelReason, + ); + }); + test('streams can be cancelled with no data sent', async () => { + const cancelReason = Symbol('CancelReason'); + const codeToReason = (type, code: bigint) => { + switch (code) { + case 4001n: + return cancelReason; + default: + return new Error(`${type.toString()} ${code.toString()}`); + } + }; + const reasonToCode = (_type, reason) => { + if (reason === cancelReason) return 4001n; + return 0n; + }; + const [_stream1, stream2] = await createStreamPair({ + codeToReason, + reasonToCode, + }); + + await stream2.cancel(cancelReason); + + await expect(stream2.readable.getReader().read()).rejects.toBe( + cancelReason, + ); + await expect(stream2.writable.getWriter().write()).rejects.toBe( + cancelReason, + ); + }); + test('streams can be cancelled concurrently after data sent', async () => { + const [stream1, stream2] = await createStreamPair(); + + const writer = stream2.writable.getWriter(); + await writer.write(new Uint8Array(2)); + + const reader = stream1.readable.getReader(); + reader.releaseLock(); + + await Promise.all([writer.close(), stream1.writable.close()]); + }); + // test('stream can error when blocked on data', async () => { + // const [stream1, stream2] = await createStreamPair(); + + // const message = new Uint8Array(STREAM_BUFFER_SIZE * 2); + + // const stream1Writer = stream1.writable.getWriter(); + // void stream1Writer.write(message); + // stream1Writer.releaseLock(); + + // const stream2Writer = stream2.writable.getWriter(); + // void stream2Writer.write(message); + // stream2Writer.releaseLock(); + + // await Promise.all([ + // stream1Writer.abort(Error('some error')), + // stream1Writer.abort(Error('some error')), + // ]); + // }); }); From 8592b0ce9ac5979cff457bc7a5560d89b6a633d4 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:26:11 +1000 Subject: [PATCH 064/149] lintfix --- src/WebSocketStream.ts | 47 ++++++++++++---------- src/WebSocketStreamQueue.ts | 22 +++++------ src/index.ts | 1 - tests/WebSocketStream.test.ts | 2 +- tests/WebSocketStreamQueue.test.ts | 62 +++++++++++++----------------- 5 files changed, 66 insertions(+), 68 deletions(-) diff --git a/src/WebSocketStream.ts b/src/WebSocketStream.ts index 5110453d..f136001f 100644 --- a/src/WebSocketStream.ts +++ b/src/WebSocketStream.ts @@ -1,6 +1,11 @@ import type { StreamCodeToReason, StreamReasonToCode } from './types'; import type WebSocketConnection from './WebSocketConnection'; import type { StreamId, StreamMessage, VarInt } from './message'; +import { + ReadableStream, + WritableStream, + CountQueuingStrategy, +} from 'stream/web'; import { CreateDestroy, status } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; import { promise } from './utils'; @@ -13,12 +18,6 @@ import { StreamMessageType, StreamShutdown, } from './message'; -import { - ReadableStream, - WritableStream, - CountQueuingStrategy, - ReadableByteStreamController, -} from 'stream/web'; import WebSocketStreamQueue from './WebSocketStreamQueue'; interface WebSocketStream extends CreateDestroy {} @@ -129,14 +128,14 @@ class WebSocketStream implements ReadableWritablePair { return; } - // reset the promise before a read from the queue to wait until the queue has items + // Reset the promise before a read from the queue to wait until the queue has items if (this.readableQueue.count === 0) { this.readableBufferReady.resolveP(); this.readableBufferReady = promise(); } await this.readableBufferReady.p; - // data will be null in the case of stream destruction before the readable buffer is blocked + // Data will be null in the case of stream destruction before the readable buffer is blocked // we're going to just enqueue an empty buffer in case it is null for some other reason, so that the next read is able to complete const data = this.readableQueue.dequeue(); if (data == null) { @@ -146,7 +145,9 @@ class WebSocketStream implements ReadableWritablePair { const readBytes = data.length; controller.enqueue(data); - this.logger.debug(`${readBytes} bytes have been pushed onto stream buffer`) + this.logger.debug( + `${readBytes} bytes have been pushed onto stream buffer`, + ); await this.streamSend({ type: StreamMessageType.Ack, @@ -158,9 +159,9 @@ class WebSocketStream implements ReadableWritablePair { await this.signalReadableEnd(true, reason); }, }, - { + new CountQueuingStrategy({ highWaterMark: 1, - } + }), ); this.writable = new WritableStream( @@ -180,25 +181,28 @@ class WebSocketStream implements ReadableWritablePair { await this.writableDesiredSizeProm.p; - // chunking - let data: Uint8Array; + // Chunking + // .subarray parameters begin and end are clamped to the size of the Uint8Array + const data = chunk.subarray(0, this.writableDesiredSize); if (chunk.length > this.writableDesiredSize) { this.logger.debug( `this chunk will be split into sizes of ${this.writableDesiredSize} bytes`, ); } - // .subarray parameters begin and end are clamped to the size of the Uint8Array - data = chunk.subarray(0, this.writableDesiredSize); const bytesWritten = data.length; if (this.writableDesiredSize === bytesWritten) { - this.logger.debug(`this chunk will trigger receiver to send an ACK`); + this.logger.debug( + `this chunk will trigger receiver to send an ACK`, + ); // Reset the promise to wait for another ACK this.writableDesiredSizeProm = promise(); } // Decrement the desired size by the amount of bytes written this.writableDesiredSize -= bytesWritten; - this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to write`); + this.logger.debug( + `writableDesiredSize is now ${this.writableDesiredSize} due to write`, + ); await this.streamSend({ type: StreamMessageType.Data, payload: data, @@ -284,13 +288,16 @@ class WebSocketStream implements ReadableWritablePair { if (parsedMessage.type === StreamMessageType.Ack) { this.writableDesiredSize += parsedMessage.payload; this.writableDesiredSizeProm.resolveP(); - this.logger.debug(`writableDesiredSize is now ${this.writableDesiredSize} due to ACK`); + this.logger.debug( + `writableDesiredSize is now ${this.writableDesiredSize} due to ACK`, + ); } else if (parsedMessage.type === StreamMessageType.Data) { if (this._readableEnded) { return; } if ( - parsedMessage.payload.length > (this.readableQueueBufferSize - this.readableQueue.length) + parsedMessage.payload.length > + this.readableQueueBufferSize - this.readableQueue.length ) { await this.signalReadableEnd( true, @@ -372,7 +379,7 @@ class WebSocketStream implements ReadableWritablePair { this._readableEnded = true; // Resolve readable promise in case blocking this.readableBufferReady.resolveP(); - // clear the readable queue + // Clear the readable queue this.readableQueue.clear(); // Shutdown the write side of the other stream if (isError) { diff --git a/src/WebSocketStreamQueue.ts b/src/WebSocketStreamQueue.ts index 5c50ae9b..5d4566e8 100644 --- a/src/WebSocketStreamQueue.ts +++ b/src/WebSocketStreamQueue.ts @@ -19,7 +19,7 @@ class WebSocketStreamQueue { * The combined length of the queued `Uint8Array`s. */ public get length(): Readonly { - return this._length + return this._length; } /** * The number of queued `Uint8Array`. @@ -35,19 +35,19 @@ class WebSocketStreamQueue { } public queue(data: Uint8Array): void { const item = { - data + data, }; - // if there is no head, then this is the first item in the queue + // If there is no head, then this is the first item in the queue if (this.head == null) { this.head = item; } - // if the tail exists, then set the next item on the tail to the new item + // If the tail exists, then set the next item on the tail to the new item if (this.tail != null) { this.tail.next = item; } - // set the tail to the new item + // Set the tail to the new item this.tail = item; - // update the byteLength, length, and count + // Update the byteLength, length, and count this._byteLength += data.byteLength; this._length += data.length; this._count++; @@ -57,15 +57,15 @@ class WebSocketStreamQueue { * If the queue is empty, then undefined is returned. */ public dequeue(): Uint8Array | undefined { - // get the data of the head + // Get the data of the head const oldData = this.head?.data; const newHead = this.head?.next; - // if the head and the tail are the same, then the queue is either empty or only have one item + // If the head and the tail are the same, then the queue is either empty or only have one item if (this.head === this.tail) { this.tail = undefined; } this.head = newHead; - // decrement the count, but don't let it go below 0 in case the queue is empty + // Decrement the count, but don't let it go below 0 in case the queue is empty this._count = this._count === 0 ? 0 : this._count - 1; this._byteLength -= oldData?.byteLength ?? 0; this._length -= oldData?.length ?? 0; @@ -75,7 +75,7 @@ class WebSocketStreamQueue { this._byteLength = 0; this._length = 0; this._count = 0; - // clearing head and tail should cause the garbage collector to clean up all the items in the queue + // Clearing head and tail should cause the garbage collector to clean up all the items in the queue this.head = undefined; this.tail = undefined; } @@ -84,6 +84,6 @@ class WebSocketStreamQueue { type WebSocketStreamQueueItem = { data: Uint8Array; next?: WebSocketStreamQueueItem; -} +}; export default WebSocketStreamQueue; diff --git a/src/index.ts b/src/index.ts index 47747c2f..aeb3cb13 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,4 +10,3 @@ export * as events from './events'; export * as errors from './errors'; export * as config from './config'; export * as message from './message'; - diff --git a/tests/WebSocketStream.test.ts b/tests/WebSocketStream.test.ts index fda4d65d..2a9f2283 100644 --- a/tests/WebSocketStream.test.ts +++ b/tests/WebSocketStream.test.ts @@ -517,7 +517,7 @@ describe(WebSocketStream.name, () => { await Promise.all([writer.close(), stream1.writable.close()]); }); - // test('stream can error when blocked on data', async () => { + // Test('stream can error when blocked on data', async () => { // const [stream1, stream2] = await createStreamPair(); // const message = new Uint8Array(STREAM_BUFFER_SIZE * 2); diff --git a/tests/WebSocketStreamQueue.test.ts b/tests/WebSocketStreamQueue.test.ts index d94179a9..95740fe1 100644 --- a/tests/WebSocketStreamQueue.test.ts +++ b/tests/WebSocketStreamQueue.test.ts @@ -1,40 +1,32 @@ -import WebSocketStreamQueue from "@/WebSocketStreamQueue"; -import { fc, testProp } from "@fast-check/jest"; +import { fc, testProp } from '@fast-check/jest'; +import WebSocketStreamQueue from '@/WebSocketStreamQueue'; describe(WebSocketStreamQueue.name, () => { - testProp( - "should queue items", - [fc.array(fc.uint8Array())], - (array) => { - const queue = new WebSocketStreamQueue(); - let totalLength = 0; - let totalByteLength = 0; - for (const buffer of array) { - queue.queue(buffer); - totalByteLength += buffer.byteLength; - totalLength += buffer.length; - } - expect(queue.count).toBe(array.length); - expect(queue.byteLength).toBe(totalByteLength); - expect(queue.length).toBe(totalLength); + testProp('should queue items', [fc.array(fc.uint8Array())], (array) => { + const queue = new WebSocketStreamQueue(); + let totalLength = 0; + let totalByteLength = 0; + for (const buffer of array) { + queue.queue(buffer); + totalByteLength += buffer.byteLength; + totalLength += buffer.length; } - ); - testProp( - "should dequeue items", - [fc.array(fc.uint8Array())], - (array) => { - const queue = new WebSocketStreamQueue(); - for (const buffer of array) { - queue.queue(buffer); - } - const result: Array = []; - for (let i = 0; i < array.length; i++) { - result.push(queue.dequeue()!); - } - expect(result).toEqual(array); - expect(queue.count).toBe(0) - expect(queue.byteLength).toBe(0); - expect(queue.length).toBe(0); + expect(queue.count).toBe(array.length); + expect(queue.byteLength).toBe(totalByteLength); + expect(queue.length).toBe(totalLength); + }); + testProp('should dequeue items', [fc.array(fc.uint8Array())], (array) => { + const queue = new WebSocketStreamQueue(); + for (const buffer of array) { + queue.queue(buffer); } - ); + const result: Array = []; + for (let i = 0; i < array.length; i++) { + result.push(queue.dequeue()!); + } + expect(result).toEqual(array); + expect(queue.count).toBe(0); + expect(queue.byteLength).toBe(0); + expect(queue.length).toBe(0); + }); }); From 2e7716f51f768ff28f05a2739e5f73d26e831405 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 16:43:56 +1000 Subject: [PATCH 065/149] fix: propagate logger on WebSocketClient --- src/WebSocketClient.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index 5c26574b..cf03ad52 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -163,6 +163,9 @@ class WebSocketClient extends EventTarget { socket: webSocket, verifyCallback, client: client, + logger: logger.getChild( + `${WebSocketConnection.name} ${connectionId}`, + ), }); await connection.start({ timer: wsConfig.connectTimeoutTime, From dfd378405b5f1e23fd74e0bbd4c328d04df91bdd Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:20:11 +1000 Subject: [PATCH 066/149] feat: benchmarks for baseline + streams --- benches/index.ts | 46 +++ .../results/baseline/baseline_1KB.chart.html | 116 +++++++ benches/results/baseline/baseline_1KB.json | 119 +++++++ .../results/baseline/baseline_1KB_metrics.txt | 8 + benches/results/metrics.txt | 0 benches/results/stream/stream_1KB.chart.html | 116 +++++++ benches/results/stream/stream_1KB.json | 117 +++++++ benches/results/stream/stream_1KB_metrics.txt | 8 + benches/results/system.json | 41 +++ benches/suites/baseline/baseline_1KB.ts | 62 ++++ benches/suites/stream/stream_1KB.ts | 94 ++++++ benches/utils.ts | 100 ++++++ package-lock.json | 291 ++++++++++++++++++ package.json | 9 +- src/WebSocketClient.ts | 4 +- 15 files changed, 1125 insertions(+), 6 deletions(-) create mode 100644 benches/index.ts create mode 100644 benches/results/baseline/baseline_1KB.chart.html create mode 100644 benches/results/baseline/baseline_1KB.json create mode 100644 benches/results/baseline/baseline_1KB_metrics.txt create mode 100644 benches/results/metrics.txt create mode 100644 benches/results/stream/stream_1KB.chart.html create mode 100644 benches/results/stream/stream_1KB.json create mode 100644 benches/results/stream/stream_1KB_metrics.txt create mode 100644 benches/results/system.json create mode 100644 benches/suites/baseline/baseline_1KB.ts create mode 100644 benches/suites/stream/stream_1KB.ts create mode 100644 benches/utils.ts diff --git a/benches/index.ts b/benches/index.ts new file mode 100644 index 00000000..8b7f143b --- /dev/null +++ b/benches/index.ts @@ -0,0 +1,46 @@ +#!/usr/bin/env ts-node + +import fs from 'fs'; +import path from 'path'; +import si from 'systeminformation'; +import Stream1KB from './suites/stream/stream_1KB'; +import Baseline1KB from './suites/baseline/baseline_1KB'; + +async function main(): Promise { + await fs.promises.mkdir(path.join(__dirname, 'results'), { recursive: true }); + // Running benches + await Stream1KB(); + await Baseline1KB(); + const resultFilenames = await fs.promises.readdir( + path.join(__dirname, 'results'), + ); + const metricsFile = await fs.promises.open( + path.join(__dirname, 'results', 'metrics.txt'), + 'w', + ); + let concatenating = false; + for (const resultFilename of resultFilenames) { + if (/.+_metrics\.txt$/.test(resultFilename)) { + const metricsData = await fs.promises.readFile( + path.join(__dirname, 'results', resultFilename), + ); + if (concatenating) { + await metricsFile.write('\n'); + } + await metricsFile.write(metricsData); + concatenating = true; + } + } + await metricsFile.close(); + const systemData = await si.get({ + cpu: '*', + osInfo: 'platform, distro, release, kernel, arch', + system: 'model, manufacturer', + }); + await fs.promises.writeFile( + path.join(__dirname, 'results', 'system.json'), + JSON.stringify(systemData, null, 2), + ); +} + +void main(); diff --git a/benches/results/baseline/baseline_1KB.chart.html b/benches/results/baseline/baseline_1KB.chart.html new file mode 100644 index 00000000..46fcbfbf --- /dev/null +++ b/benches/results/baseline/baseline_1KB.chart.html @@ -0,0 +1,116 @@ + + + + + + + + baseline.baseline_1KB + + + +

+ +
+ + + \ No newline at end of file diff --git a/benches/results/baseline/baseline_1KB.json b/benches/results/baseline/baseline_1KB.json new file mode 100644 index 00000000..6def8859 --- /dev/null +++ b/benches/results/baseline/baseline_1KB.json @@ -0,0 +1,119 @@ +{ + "name": "baseline.baseline_1KB", + "date": "2023-09-07T07:19:25.478Z", + "version": "1.0.0", + "results": [ + { + "name": "send 1Kib of data over ws", + "ops": 30108, + "margin": 2.73, + "options": { + "delay": 0.005, + "initCount": 1, + "minTime": 0.05, + "maxTime": 5, + "minSamples": 5 + }, + "samples": 75, + "promise": true, + "details": { + "min": 0.000029252882078225336, + "max": 0.00005232727777777778, + "mean": 0.00003321381603117393, + "median": 0.00003224295329830706, + "standardDeviation": 0.000004006190686929889, + "marginOfError": 9.066863064373856e-7, + "relativeMarginOfError": 2.7298468371908395, + "standardErrorOfMean": 4.625950543047886e-7, + "sampleVariance": 1.6049563820043778e-11, + "sampleResults": [ + 0.000029252882078225336, + 0.00002929751021599533, + 0.00002936853634232122, + 0.000029405826364692217, + 0.00002951502276707531, + 0.000029526803048065652, + 0.00002960880246189918, + 0.000029788255107997664, + 0.000030102925861062467, + 0.000030135480255516838, + 0.000030136439605110337, + 0.000030152616144018583, + 0.00003018484610917538, + 0.00003026503093987157, + 0.00003033932749562172, + 0.00003037961004086398, + 0.000030418737886748396, + 0.000030514216803760284, + 0.000030524466433158205, + 0.00003066791827203736, + 0.00003077566374781086, + 0.00003091431348511383, + 0.00003121467308814944, + 0.00003128154290718038, + 0.000031282152989449, + 0.00003131152072387624, + 0.000031480760653823704, + 0.00003149354232340922, + 0.00003150761938120257, + 0.00003173931300813008, + 0.000031768224501758504, + 0.00003179268826619965, + 0.000031795018096906016, + 0.00003180415176470588, + 0.00003184091418563923, + 0.000031990409730363424, + 0.0000322288606271777, + 0.00003224295329830706, + 0.00003236030869565217, + 0.0000323851321974148, + 0.00003241311500291885, + 0.000032427120256859314, + 0.00003243595887191539, + 0.00003245894570928196, + 0.00003247941140776699, + 0.00003255369410391127, + 0.00003260386701509872, + 0.00003269247285464098, + 0.000032795169293636896, + 0.00003307173730297723, + 0.00003321116637478109, + 0.00003351214652656159, + 0.00003351629268292683, + 0.00003365487224157956, + 0.00003365600175131348, + 0.000033904696438995916, + 0.00003391641973146527, + 0.000034282498235294115, + 0.000034517285714285715, + 0.000034927875664893616, + 0.0000349384786923526, + 0.00003515318855808523, + 0.00003525902802101576, + 0.00003531050262697023, + 0.000036608705676243866, + 0.000036668130764740224, + 0.00003854895621716287, + 0.0000394337381489842, + 0.000039615970811441917, + 0.00003965940163455925, + 0.00004029793870402802, + 0.00004058788494208494, + 0.00004340250204319907, + 0.00004539873263280794, + 0.00005232727777777778 + ] + }, + "completed": true, + "percentSlower": 0 + } + ], + "fastest": { + "name": "send 1Kib of data over ws", + "index": 0 + }, + "slowest": { + "name": "send 1Kib of data over ws", + "index": 0 + } +} \ No newline at end of file diff --git a/benches/results/baseline/baseline_1KB_metrics.txt b/benches/results/baseline/baseline_1KB_metrics.txt new file mode 100644 index 00000000..9e608461 --- /dev/null +++ b/benches/results/baseline/baseline_1KB_metrics.txt @@ -0,0 +1,8 @@ +# TYPE baseline.baseline_1KB_ops gauge +baseline.baseline_1KB_ops{name="send 1Kib of data over ws"} 30108 + +# TYPE baseline.baseline_1KB_margin gauge +baseline.baseline_1KB_margin{name="send 1Kib of data over ws"} 2.73 + +# TYPE baseline.baseline_1KB_samples counter +baseline.baseline_1KB_samples{name="send 1Kib of data over ws"} 75 diff --git a/benches/results/metrics.txt b/benches/results/metrics.txt new file mode 100644 index 00000000..e69de29b diff --git a/benches/results/stream/stream_1KB.chart.html b/benches/results/stream/stream_1KB.chart.html new file mode 100644 index 00000000..50dcb9cf --- /dev/null +++ b/benches/results/stream/stream_1KB.chart.html @@ -0,0 +1,116 @@ + + + + + + + + stream.stream_1KB + + + +
+ +
+ + + \ No newline at end of file diff --git a/benches/results/stream/stream_1KB.json b/benches/results/stream/stream_1KB.json new file mode 100644 index 00000000..4b54bbcb --- /dev/null +++ b/benches/results/stream/stream_1KB.json @@ -0,0 +1,117 @@ +{ + "name": "stream.stream_1KB", + "date": "2023-09-07T07:19:18.204Z", + "version": "1.0.0", + "results": [ + { + "name": "send 1Kib of data over stream", + "ops": 426, + "margin": 5.06, + "options": { + "delay": 0.005, + "initCount": 1, + "minTime": 0.05, + "maxTime": 5, + "minSamples": 5 + }, + "samples": 73, + "promise": true, + "details": { + "min": 0.0017477503103448277, + "max": 0.0038379029285714288, + "mean": 0.0023471357312319016, + "median": 0.00215252308, + "standardDeviation": 0.0005182063925109443, + "marginOfError": 0.00011887688250114453, + "relativeMarginOfError": 5.064763870249276, + "standardErrorOfMean": 0.00006065147066384925, + "sampleVariance": 2.6853786523920685e-7, + "sampleResults": [ + 0.0017477503103448277, + 0.0017967768214285714, + 0.0018129551785714285, + 0.0018137015, + 0.0018192088214285715, + 0.001821007892857143, + 0.001821310357142857, + 0.0018235127142857143, + 0.0018691560740740742, + 0.0019064947857142856, + 0.0019268388214285714, + 0.0019428332962962962, + 0.0019438307857142856, + 0.001947187107142857, + 0.0019526353703703703, + 0.0019581806666666667, + 0.0019664113333333335, + 0.0019679321785714284, + 0.0019703470357142856, + 0.001991393259259259, + 0.0019938596785714284, + 0.0020057217407407407, + 0.0020108668214285715, + 0.002012867074074074, + 0.002023275, + 0.00202838416, + 0.0020436227142857145, + 0.00204693492, + 0.002052115392857143, + 0.00205514228, + 0.0020569102962962964, + 0.002079562962962963, + 0.0020855759200000003, + 0.0021088123571428572, + 0.0021170235, + 0.00212085862962963, + 0.00215252308, + 0.00215609404, + 0.0021769194642857143, + 0.002218121392857143, + 0.0022943591363636362, + 0.0022965015555555554, + 0.0023230972142857143, + 0.0023294068888888887, + 0.002330770888888889, + 0.002355380592592593, + 0.002362644107142857, + 0.0023833217142857142, + 0.002386992590909091, + 0.0024229923703703703, + 0.0024297490454545452, + 0.002452695, + 0.0025563194, + 0.0025570125714285716, + 0.002564137, + 0.0026277650952380954, + 0.0026469360357142856, + 0.0026939087142857144, + 0.002698705095238095, + 0.002751616052631579, + 0.002779424, + 0.0028510274444444443, + 0.0029493395789473684, + 0.0030082369285714286, + 0.003019166142857143, + 0.003035778, + 0.003149590392857143, + 0.003401115892857143, + 0.003438066714285714, + 0.0035103203809523807, + 0.0037650444375, + 0.0037869287333333335, + 0.0038379029285714288 + ] + }, + "completed": true, + "percentSlower": 0 + } + ], + "fastest": { + "name": "send 1Kib of data over stream", + "index": 0 + }, + "slowest": { + "name": "send 1Kib of data over stream", + "index": 0 + } +} \ No newline at end of file diff --git a/benches/results/stream/stream_1KB_metrics.txt b/benches/results/stream/stream_1KB_metrics.txt new file mode 100644 index 00000000..9f46257e --- /dev/null +++ b/benches/results/stream/stream_1KB_metrics.txt @@ -0,0 +1,8 @@ +# TYPE stream.stream_1KB_ops gauge +stream.stream_1KB_ops{name="send 1Kib of data over stream"} 426 + +# TYPE stream.stream_1KB_margin gauge +stream.stream_1KB_margin{name="send 1Kib of data over stream"} 5.06 + +# TYPE stream.stream_1KB_samples counter +stream.stream_1KB_samples{name="send 1Kib of data over stream"} 73 diff --git a/benches/results/system.json b/benches/results/system.json new file mode 100644 index 00000000..853f4aff --- /dev/null +++ b/benches/results/system.json @@ -0,0 +1,41 @@ +{ + "cpu": { + "manufacturer": "Intel", + "brand": "Gen Intel® Core™ i5-11320H", + "vendor": "Intel", + "family": "6", + "model": "140", + "stepping": "2", + "revision": "", + "voltage": "", + "speed": 3.2, + "speedMin": 0.4, + "speedMax": 4.5, + "governor": "powersave", + "cores": 8, + "physicalCores": 4, + "performanceCores": 4, + "efficiencyCores": 0, + "processors": 1, + "socket": "", + "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l2 invpcid_single cdp_l2 ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves split_lock_detect dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp hwp_pkg_req avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid movdiri movdir64b fsrm avx512_vp2intersect md_clear flush_l1d arch_capabilities", + "virtualization": true, + "cache": { + "l1d": 196608, + "l1i": 131072, + "l2": 5242880, + "l3": 8388608 + } + }, + "osInfo": { + "platform": "linux", + "distro": "nixos", + "release": "22.11", + "kernel": "5.10.177", + "arch": "x64" + }, + "system": { + "model": "Vostro 14 5410", + "manufacturer": "Dell Inc." + } +} \ No newline at end of file diff --git a/benches/suites/baseline/baseline_1KB.ts b/benches/suites/baseline/baseline_1KB.ts new file mode 100644 index 00000000..1c63523e --- /dev/null +++ b/benches/suites/baseline/baseline_1KB.ts @@ -0,0 +1,62 @@ +import type { Host } from '../../../src/types'; +import type { AddressInfo } from 'net'; +import * as https from 'https'; +import b from 'benny'; +import * as ws from 'ws'; +import { promise } from '@/utils'; +import { suiteCommon, summaryName } from '../../utils'; +import * as testsUtils from '../../../tests/utils'; + +async function main() { + // Setting up initial state + const data1KiB = Buffer.alloc(1024, 0xf0); + const host = '127.0.0.1' as Host; + const tlsConfig = await testsUtils.generateConfig('RSA'); + + const listenProm = promise(); + + const httpsServer = https.createServer({ + ...tlsConfig, + }); + const wsServer = new ws.WebSocketServer({ + server: httpsServer, + }); + httpsServer.listen(0, host, listenProm.resolveP); + + await listenProm.p; + + const address = httpsServer.address() as AddressInfo; + + const openProm = promise(); + + const client = new ws.WebSocket(`wss://${host}:${address.port}`, { + rejectUnauthorized: false, + }); + + client.on('open', openProm.resolveP); + + await openProm.p; + + // Running benchmark + const summary = await b.suite( + summaryName(__filename), + b.add('send 1Kib of data over ws', async () => { + const sendProm = promise(); + client.send(data1KiB, () => { + sendProm.resolveP(); + }); + await sendProm.p; + }), + ...suiteCommon, + ); + client.close(); + wsServer.close(); + httpsServer.close(); + return summary; +} + +if (require.main === module) { + void main(); +} + +export default main; diff --git a/benches/suites/stream/stream_1KB.ts b/benches/suites/stream/stream_1KB.ts new file mode 100644 index 00000000..d9327bec --- /dev/null +++ b/benches/suites/stream/stream_1KB.ts @@ -0,0 +1,94 @@ +import type { Host } from '../../../src/types'; +import b from 'benny'; +import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; +import { suiteCommon, summaryName } from '../../utils'; +import * as events from '../../../src/events'; +import * as testsUtils from '../../../tests/utils'; +import WebSocketServer from '../../../src/WebSocketServer'; +import WebSocketClient from '../../../src/WebSocketClient'; + +async function main() { + const logger = new Logger(`Stream1KB Bench`, LogLevel.WARN, [ + new StreamHandler( + formatting.format`${formatting.level}:${formatting.keys}:${formatting.msg}`, + ), + ]); + // Setting up initial state + const data1KiB = Buffer.alloc(1024, 0xf0); + const host = '127.0.0.1' as Host; + const tlsConfig = await testsUtils.generateConfig('RSA'); + + const wsServer = new WebSocketServer({ + config: { + key: tlsConfig.key, + cert: tlsConfig.cert, + }, + logger, + }); + + wsServer.addEventListener( + events.EventWebSocketServerConnection.name, + async (e: events.EventWebSocketServerConnection) => { + const conn = e.detail; + conn.addEventListener( + events.EventWebSocketConnectionStream.name, + (streamEvent: events.EventWebSocketConnectionStream) => { + const stream = streamEvent.detail; + void Promise.allSettled([ + (async () => { + // Consume data + for await (const _ of stream.readable) { + // Do nothing, only consume + } + })(), + (async () => { + // End writable immediately + await stream.writable.close(); + })(), + ]); + }, + ); + }, + ); + await wsServer.start({ + host, + }); + const client = await WebSocketClient.createWebSocketClient({ + host, + port: wsServer.getPort(), + logger, + verifyCallback: async () => {}, + }); + + // Running benchmark + const summary = await b.suite( + summaryName(__filename), + b.add('send 1Kib of data over stream', async () => { + const stream = await client.connection.streamNew(); + await Promise.all([ + (async () => { + // Consume data + for await (const _ of stream.readable) { + // Do nothing, only consume + } + })(), + (async () => { + // Write data + const writer = stream.writable.getWriter(); + await writer.write(data1KiB); + await writer.close(); + })(), + ]); + }), + ...suiteCommon, + ); + await wsServer.stop({ force: true }); + await client.destroy({ force: true }); + return summary; +} + +if (require.main === module) { + void main(); +} + +export default main; diff --git a/benches/utils.ts b/benches/utils.ts new file mode 100644 index 00000000..b8d7758a --- /dev/null +++ b/benches/utils.ts @@ -0,0 +1,100 @@ +import fs from 'fs'; +import path from 'path'; +import b from 'benny'; +import { codeBlock } from 'common-tags'; +import packageJson from '../package.json'; + +const suitesPath = path.join(__dirname, 'suites'); +const resultsPath = path.join(__dirname, 'results'); + +function summaryName(suitePath: string) { + return path + .relative(suitesPath, suitePath) + .replace(/\.[^.]*$/, '') + .replace(/\//g, '.'); +} + +const suiteCommon = [ + b.cycle(), + b.complete(), + b.save({ + file: (summary) => { + // Replace dots with slashes + const relativePath = summary.name.replace(/\./g, '/'); + // To `results/path/to/suite` + const resultPath = path.join(resultsPath, relativePath); + // This creates directory `results/path/to` + fs.mkdirSync(path.dirname(resultPath), { recursive: true }); + return relativePath; + }, + folder: resultsPath, + version: packageJson.version, + details: true, + }), + b.save({ + file: (summary) => { + // Replace dots with slashes + const relativePath = summary.name.replace(/\./g, '/'); + // To `results/path/to/suite` + const resultPath = path.join(resultsPath, relativePath); + // This creates directory `results/path/to` + fs.mkdirSync(path.dirname(resultPath), { recursive: true }); + return relativePath; + }, + folder: resultsPath, + version: packageJson.version, + format: 'chart.html', + }), + b.complete((summary) => { + // Replace dots with slashes + const relativePath = summary.name.replace(/\./g, '/'); + // To `results/path/to/suite_metrics.txt` + const resultPath = path.join(resultsPath, relativePath) + '_metrics.txt'; + // This creates directory `results/path/to` + fs.mkdirSync(path.dirname(resultPath), { recursive: true }); + fs.writeFileSync( + resultPath, + codeBlock` + # TYPE ${summary.name}_ops gauge + ${summary.results + .map( + (result) => + `${summary.name}_ops{name="${result.name}"} ${result.ops}`, + ) + .join('\n')} + + # TYPE ${summary.name}_margin gauge + ${summary.results + .map( + (result) => + `${summary.name}_margin{name="${result.name}"} ${result.margin}`, + ) + .join('\n')} + + # TYPE ${summary.name}_samples counter + ${summary.results + .map( + (result) => + `${summary.name}_samples{name="${result.name}"} ${result.samples}`, + ) + .join('\n')} + ` + '\n', + ); + // eslint-disable-next-line no-console + console.log('\nSaved to:', path.resolve(resultPath)); + }), +]; + +async function* fsWalk(dir: string): AsyncGenerator { + const dirents = await fs.promises.readdir(dir, { withFileTypes: true }); + for (const dirent of dirents) { + const res = path.resolve(dir, dirent.name); + if (dirent.isDirectory()) { + yield* fsWalk(res); + } else { + yield res; + } + } +} + +export { suitesPath, resultsPath, summaryName, suiteCommon, fsWalk }; diff --git a/package-lock.json b/package-lock.json index f9f4ccbb..0ffaa168 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "@types/ws": "^8.5.5", "@typescript-eslint/eslint-plugin": "^5.45.1", "@typescript-eslint/parser": "^5.45.1", + "benny": "^3.7.1", "eslint": "^8.15.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-import": "^2.26.0", @@ -46,6 +47,7 @@ "prettier": "^2.6.2", "semver": "^7.3.7", "shx": "^0.3.4", + "systeminformation": "^5.21.4", "ts-jest": "^28.0.5", "ts-node": "^10.9.1", "tsconfig-paths": "^3.9.0", @@ -75,6 +77,48 @@ "node": ">=6.0.0" } }, + "node_modules/@arrows/array": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@arrows/array/-/array-1.4.1.tgz", + "integrity": "sha512-MGYS8xi3c4tTy1ivhrVntFvufoNzje0PchjEz6G/SsWRgUKxL4tKwS6iPdO8vsaJYldagAeWMd5KRD0aX3Q39g==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.2.2" + } + }, + "node_modules/@arrows/composition": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@arrows/composition/-/composition-1.2.2.tgz", + "integrity": "sha512-9fh1yHwrx32lundiB3SlZ/VwuStPB4QakPsSLrGJFH6rCXvdrd060ivAZ7/2vlqPnEjBkPRRXOcG1YOu19p2GQ==", + "dev": true + }, + "node_modules/@arrows/dispatch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@arrows/dispatch/-/dispatch-1.0.3.tgz", + "integrity": "sha512-v/HwvrFonitYZM2PmBlAlCqVqxrkIIoiEuy5bQgn0BdfvlL0ooSBzcPzTMrtzY8eYktPyYcHg8fLbSgyybXEqw==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.2.2" + } + }, + "node_modules/@arrows/error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@arrows/error/-/error-1.0.2.tgz", + "integrity": "sha512-yvkiv1ay4Z3+Z6oQsUkedsQm5aFdyPpkBUQs8vejazU/RmANABx6bMMcBPPHI4aW43VPQmXFfBzr/4FExwWTEA==", + "dev": true + }, + "node_modules/@arrows/multimethod": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@arrows/multimethod/-/multimethod-1.4.1.tgz", + "integrity": "sha512-AZnAay0dgPnCJxn3We5uKiB88VL+1ZIF2SjZohLj6vqY2UyvB/sKdDnFP+LZNVsTC5lcnGPmLlRRkAh4sXkXsQ==", + "dev": true, + "dependencies": { + "@arrows/array": "^1.4.1", + "@arrows/composition": "^1.2.2", + "@arrows/error": "^1.0.2", + "fast-deep-equal": "^3.1.3" + } + }, "node_modules/@babel/code-frame": { "version": "7.22.10", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.10.tgz", @@ -2985,6 +3029,15 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "dev": true }, + "node_modules/astral-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", @@ -3103,6 +3156,45 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/benchmark": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz", + "integrity": "sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==", + "dev": true, + "dependencies": { + "lodash": "^4.17.4", + "platform": "^1.3.3" + } + }, + "node_modules/benny": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/benny/-/benny-3.7.1.tgz", + "integrity": "sha512-USzYxODdVfOS7JuQq/L0naxB788dWCiUgUTxvN+WLPt/JfcDURNNj8kN/N+uK6PDvuR67/9/55cVKGPleFQINA==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.0.0", + "@arrows/dispatch": "^1.0.2", + "@arrows/multimethod": "^1.1.6", + "benchmark": "^2.1.4", + "common-tags": "^1.8.0", + "fs-extra": "^10.0.0", + "json2csv": "^5.0.6", + "kleur": "^4.1.4", + "log-update": "^4.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/benny/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/bitset": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/bitset/-/bitset-5.1.1.tgz", @@ -3289,6 +3381,18 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "dev": true }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -3337,6 +3441,24 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/commander": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", + "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4173,6 +4295,20 @@ "is-callable": "^1.1.3" } }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -6830,6 +6966,25 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/json2csv": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.7.tgz", + "integrity": "sha512-YRZbUnyaJZLZUJSRi2G/MqahCyRv9n/ds+4oIetjDF3jWQA7AG7iSeKTiZiCNqtMZM7HDyt0e/W6lEnoGEmMGA==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true, + "dependencies": { + "commander": "^6.1.0", + "jsonparse": "^1.3.1", + "lodash.get": "^4.4.2" + }, + "bin": { + "json2csv": "bin/json2csv.js" + }, + "engines": { + "node": ">= 10", + "npm": ">= 6.13.0" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -6848,6 +7003,27 @@ "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", "dev": true }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "dev": true, + "engines": [ + "node >= 0.2.0" + ] + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -6900,6 +7076,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -6912,6 +7100,38 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/log-update": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-4.0.0.tgz", + "integrity": "sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==", + "dev": true, + "dependencies": { + "ansi-escapes": "^4.3.0", + "cli-cursor": "^3.1.0", + "slice-ansi": "^4.0.0", + "wrap-ansi": "^6.2.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -7415,6 +7635,12 @@ "node": ">=8" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "dev": true + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -7701,6 +7927,19 @@ "node": ">=6.4.0" } }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -7915,6 +8154,23 @@ "node": ">=8" } }, + "node_modules/slice-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -8112,6 +8368,32 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/systeminformation": { + "version": "5.21.4", + "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.21.4.tgz", + "integrity": "sha512-fLW6j47UoAJDlZPEqykkWTKxubxb8IFuow6pMQlqf4irZ2lBgCrCReavMkH2t8VxxjOcg6wBlZ2EPQcluAT6xg==", + "dev": true, + "os": [ + "darwin", + "linux", + "win32", + "freebsd", + "openbsd", + "netbsd", + "sunos", + "android" + ], + "bin": { + "systeminformation": "lib/cli.js" + }, + "engines": { + "node": ">=8.0.0" + }, + "funding": { + "type": "Buy me a coffee", + "url": "https://www.buymeacoffee.com/systeminfo" + } + }, "node_modules/terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -8523,6 +8805,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true, + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", diff --git a/package.json b/package.json index bfe06184..c8dd062d 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,11 @@ "build": "shx rm -rf ./dist && tsc -p ./tsconfig.build.json", "ts-node": "ts-node", "test": "jest", - "lint": "eslint '{src,tests,scripts}/**/*.{js,ts}'", - "lintfix": "eslint '{src,tests,scripts}/**/*.{js,ts}' --fix", + "lint": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}'", + "lintfix": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}' --fix", "lint-shell": "find ./src ./tests ./scripts -type f -regextype posix-extended -regex '.*\\.(sh)' -exec shellcheck {} +", - "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src" + "docs": "shx rm -rf ./docs && typedoc --gitRevision master --tsconfig ./tsconfig.build.json --out ./docs src", + "bench": "rimraf ./benches/results && ts-node ./benches" }, "dependencies": { "@matrixai/async-cancellable": "^1.1.1", @@ -56,6 +57,7 @@ "@types/ws": "^8.5.5", "@typescript-eslint/eslint-plugin": "^5.45.1", "@typescript-eslint/parser": "^5.45.1", + "benny": "^3.7.1", "eslint": "^8.15.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-import": "^2.26.0", @@ -66,6 +68,7 @@ "prettier": "^2.6.2", "semver": "^7.3.7", "shx": "^0.3.4", + "systeminformation": "^5.21.4", "ts-jest": "^28.0.5", "ts-node": "^10.9.1", "tsconfig-paths": "^3.9.0", diff --git a/src/WebSocketClient.ts b/src/WebSocketClient.ts index cf03ad52..3be0a367 100644 --- a/src/WebSocketClient.ts +++ b/src/WebSocketClient.ts @@ -163,9 +163,7 @@ class WebSocketClient extends EventTarget { socket: webSocket, verifyCallback, client: client, - logger: logger.getChild( - `${WebSocketConnection.name} ${connectionId}`, - ), + logger: logger.getChild(`${WebSocketConnection.name} ${connectionId}`), }); await connection.start({ timer: wsConfig.connectTimeoutTime, From e63478dfa09b509fbb1aa4cc95e04fc272ffcef1 Mon Sep 17 00:00:00 2001 From: Amy <50583248+amydevs@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:35:52 +1000 Subject: [PATCH 067/149] fix: changed to KiB naming for benches --- benches/index.ts | 53 ++++---- benches/results/baseline/baseline_1KB.json | 119 ----------------- .../results/baseline/baseline_1KB_metrics.txt | 8 -- .../baseline_tcp_1KiB.chart.html} | 16 +-- .../baseline_tcp/baseline_tcp_1KiB.json | 124 ++++++++++++++++++ .../baseline_tcp_1KiB_metrics.txt | 8 ++ .../baseline_websocket_1KiB.chart.html | 116 ++++++++++++++++ .../baseline_websocket_1KiB.json | 119 +++++++++++++++++ .../baseline_websocket_1KiB_metrics.txt | 8 ++ benches/results/metrics.txt | 26 ++++ benches/results/stream/stream_1KB.json | 117 ----------------- benches/results/stream/stream_1KB_metrics.txt | 8 -- ..._1KB.chart.html => stream_1KiB.chart.html} | 16 +-- benches/results/stream/stream_1KiB.json | 122 +++++++++++++++++ .../results/stream/stream_1KiB_metrics.txt | 8 ++ .../baseline_tcp_1KiB.ts} | 2 +- .../baseline_websocket_1KiB.ts | 62 +++++++++ .../stream/{stream_1KB.ts => stream_1KiB.ts} | 2 +- src/WebSocketConnection.ts | 2 +- 19 files changed, 641 insertions(+), 295 deletions(-) delete mode 100644 benches/results/baseline/baseline_1KB.json delete mode 100644 benches/results/baseline/baseline_1KB_metrics.txt rename benches/results/{baseline/baseline_1KB.chart.html => baseline_tcp/baseline_tcp_1KiB.chart.html} (87%) create mode 100644 benches/results/baseline_tcp/baseline_tcp_1KiB.json create mode 100644 benches/results/baseline_tcp/baseline_tcp_1KiB_metrics.txt create mode 100644 benches/results/baseline_websocket/baseline_websocket_1KiB.chart.html create mode 100644 benches/results/baseline_websocket/baseline_websocket_1KiB.json create mode 100644 benches/results/baseline_websocket/baseline_websocket_1KiB_metrics.txt delete mode 100644 benches/results/stream/stream_1KB.json delete mode 100644 benches/results/stream/stream_1KB_metrics.txt rename benches/results/stream/{stream_1KB.chart.html => stream_1KiB.chart.html} (87%) create mode 100644 benches/results/stream/stream_1KiB.json create mode 100644 benches/results/stream/stream_1KiB_metrics.txt rename benches/suites/{baseline/baseline_1KB.ts => baseline_tcp/baseline_tcp_1KiB.ts} (96%) create mode 100644 benches/suites/baseline_websocket/baseline_websocket_1KiB.ts rename benches/suites/stream/{stream_1KB.ts => stream_1KiB.ts} (97%) diff --git a/benches/index.ts b/benches/index.ts index 8b7f143b..dd2d64dd 100644 --- a/benches/index.ts +++ b/benches/index.ts @@ -1,37 +1,38 @@ #!/usr/bin/env ts-node +import type { Summary } from 'benny/lib/internal/common-types'; import fs from 'fs'; import path from 'path'; import si from 'systeminformation'; -import Stream1KB from './suites/stream/stream_1KB'; -import Baseline1KB from './suites/baseline/baseline_1KB'; +import { fsWalk, resultsPath, suitesPath } from './utils'; async function main(): Promise { await fs.promises.mkdir(path.join(__dirname, 'results'), { recursive: true }); - // Running benches - await Stream1KB(); - await Baseline1KB(); - const resultFilenames = await fs.promises.readdir( - path.join(__dirname, 'results'), - ); - const metricsFile = await fs.promises.open( - path.join(__dirname, 'results', 'metrics.txt'), - 'w', - ); + // Running all suites + for await (const suitePath of fsWalk(suitesPath)) { + // Skip over non-ts and non-js files + const ext = path.extname(suitePath); + if (ext !== '.ts' && ext !== '.js') { + continue; + } + const suite: () => Promise = (await import(suitePath)).default; + await suite(); + } + // Concatenating metrics + const metricsPath = path.join(resultsPath, 'metrics.txt'); let concatenating = false; - for (const resultFilename of resultFilenames) { - if (/.+_metrics\.txt$/.test(resultFilename)) { - const metricsData = await fs.promises.readFile( - path.join(__dirname, 'results', resultFilename), - ); - if (concatenating) { - await metricsFile.write('\n'); - } - await metricsFile.write(metricsData); - concatenating = true; + for await (const metricPath of fsWalk(resultsPath)) { + // Skip over non-metrics files + if (!metricPath.endsWith('_metrics.txt')) { + continue; } + const metricData = await fs.promises.readFile(metricPath); + if (concatenating) { + await fs.promises.appendFile(metricsPath, '\n'); + } + await fs.promises.appendFile(metricsPath, metricData); + concatenating = true; } - await metricsFile.close(); const systemData = await si.get({ cpu: '*', osInfo: 'platform, distro, release, kernel, arch', @@ -43,4 +44,8 @@ async function main(): Promise { ); } -void main(); +if (require.main === module) { + void main(); +} + +export default main; diff --git a/benches/results/baseline/baseline_1KB.json b/benches/results/baseline/baseline_1KB.json deleted file mode 100644 index 6def8859..00000000 --- a/benches/results/baseline/baseline_1KB.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "name": "baseline.baseline_1KB", - "date": "2023-09-07T07:19:25.478Z", - "version": "1.0.0", - "results": [ - { - "name": "send 1Kib of data over ws", - "ops": 30108, - "margin": 2.73, - "options": { - "delay": 0.005, - "initCount": 1, - "minTime": 0.05, - "maxTime": 5, - "minSamples": 5 - }, - "samples": 75, - "promise": true, - "details": { - "min": 0.000029252882078225336, - "max": 0.00005232727777777778, - "mean": 0.00003321381603117393, - "median": 0.00003224295329830706, - "standardDeviation": 0.000004006190686929889, - "marginOfError": 9.066863064373856e-7, - "relativeMarginOfError": 2.7298468371908395, - "standardErrorOfMean": 4.625950543047886e-7, - "sampleVariance": 1.6049563820043778e-11, - "sampleResults": [ - 0.000029252882078225336, - 0.00002929751021599533, - 0.00002936853634232122, - 0.000029405826364692217, - 0.00002951502276707531, - 0.000029526803048065652, - 0.00002960880246189918, - 0.000029788255107997664, - 0.000030102925861062467, - 0.000030135480255516838, - 0.000030136439605110337, - 0.000030152616144018583, - 0.00003018484610917538, - 0.00003026503093987157, - 0.00003033932749562172, - 0.00003037961004086398, - 0.000030418737886748396, - 0.000030514216803760284, - 0.000030524466433158205, - 0.00003066791827203736, - 0.00003077566374781086, - 0.00003091431348511383, - 0.00003121467308814944, - 0.00003128154290718038, - 0.000031282152989449, - 0.00003131152072387624, - 0.000031480760653823704, - 0.00003149354232340922, - 0.00003150761938120257, - 0.00003173931300813008, - 0.000031768224501758504, - 0.00003179268826619965, - 0.000031795018096906016, - 0.00003180415176470588, - 0.00003184091418563923, - 0.000031990409730363424, - 0.0000322288606271777, - 0.00003224295329830706, - 0.00003236030869565217, - 0.0000323851321974148, - 0.00003241311500291885, - 0.000032427120256859314, - 0.00003243595887191539, - 0.00003245894570928196, - 0.00003247941140776699, - 0.00003255369410391127, - 0.00003260386701509872, - 0.00003269247285464098, - 0.000032795169293636896, - 0.00003307173730297723, - 0.00003321116637478109, - 0.00003351214652656159, - 0.00003351629268292683, - 0.00003365487224157956, - 0.00003365600175131348, - 0.000033904696438995916, - 0.00003391641973146527, - 0.000034282498235294115, - 0.000034517285714285715, - 0.000034927875664893616, - 0.0000349384786923526, - 0.00003515318855808523, - 0.00003525902802101576, - 0.00003531050262697023, - 0.000036608705676243866, - 0.000036668130764740224, - 0.00003854895621716287, - 0.0000394337381489842, - 0.000039615970811441917, - 0.00003965940163455925, - 0.00004029793870402802, - 0.00004058788494208494, - 0.00004340250204319907, - 0.00004539873263280794, - 0.00005232727777777778 - ] - }, - "completed": true, - "percentSlower": 0 - } - ], - "fastest": { - "name": "send 1Kib of data over ws", - "index": 0 - }, - "slowest": { - "name": "send 1Kib of data over ws", - "index": 0 - } -} \ No newline at end of file diff --git a/benches/results/baseline/baseline_1KB_metrics.txt b/benches/results/baseline/baseline_1KB_metrics.txt deleted file mode 100644 index 9e608461..00000000 --- a/benches/results/baseline/baseline_1KB_metrics.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TYPE baseline.baseline_1KB_ops gauge -baseline.baseline_1KB_ops{name="send 1Kib of data over ws"} 30108 - -# TYPE baseline.baseline_1KB_margin gauge -baseline.baseline_1KB_margin{name="send 1Kib of data over ws"} 2.73 - -# TYPE baseline.baseline_1KB_samples counter -baseline.baseline_1KB_samples{name="send 1Kib of data over ws"} 75 diff --git a/benches/results/baseline/baseline_1KB.chart.html b/benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html similarity index 87% rename from benches/results/baseline/baseline_1KB.chart.html rename to benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html index 46fcbfbf..d4aeb922 100644 --- a/benches/results/baseline/baseline_1KB.chart.html +++ b/benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html @@ -5,7 +5,7 @@ - baseline.baseline_1KB + baseline_tcp.baseline_tcp_1KiB + + +
+ +
+ + + \ No newline at end of file diff --git a/benches/results/baseline_websocket/baseline_websocket_1KiB.json b/benches/results/baseline_websocket/baseline_websocket_1KiB.json new file mode 100644 index 00000000..3f93686c --- /dev/null +++ b/benches/results/baseline_websocket/baseline_websocket_1KiB.json @@ -0,0 +1,119 @@ +{ + "name": "baseline_websocket.baseline_websocket_1KiB", + "date": "2023-09-07T07:35:01.344Z", + "version": "1.0.0", + "results": [ + { + "name": "send 1KiB of data over ws", + "ops": 29078, + "margin": 2.97, + "options": { + "delay": 0.005, + "initCount": 1, + "minTime": 0.05, + "maxTime": 5, + "minSamples": 5 + }, + "samples": 75, + "promise": true, + "details": { + "min": 0.000028390809604519775, + "max": 0.00006071488248587571, + "mean": 0.00003439081653279893, + "median": 0.000033497117514124294, + "standardDeviation": 0.000004515476879162089, + "marginOfError": 0.0000010219486223478753, + "relativeMarginOfError": 2.9715741740915944, + "standardErrorOfMean": 5.214023583407526e-7, + "sampleVariance": 2.03895314462474e-11, + "sampleResults": [ + 0.000028390809604519775, + 0.000028933514124293786, + 0.000029006232768361582, + 0.00002934612186788155, + 0.000029754626399528578, + 0.000029764674720094285, + 0.000029879536158192093, + 0.000030296065536723163, + 0.00003056736271186441, + 0.00003061092937853107, + 0.00003088936848559166, + 0.000030890580790960454, + 0.00003089739540365351, + 0.00003114020056497175, + 0.00003117618418079096, + 0.00003130708644067797, + 0.00003158498474576271, + 0.0000315960506833713, + 0.000031705362711864404, + 0.00003176232711864407, + 0.000031763728929384965, + 0.000031765007660577486, + 0.000031888101694915256, + 0.000031889248587570625, + 0.00003190926635238656, + 0.000031966005649717516, + 0.000032000460813199765, + 0.00003218490282485876, + 0.000032338068355922215, + 0.00003241981525423729, + 0.00003259228418079096, + 0.000032743301129943505, + 0.000032941703954802255, + 0.000033190504598405885, + 0.000033316031073446324, + 0.00003332061512939615, + 0.000033359078479460456, + 0.000033497117514124294, + 0.00003353896214689265, + 0.00003354978587570621, + 0.000033555286440677966, + 0.000033738032514930325, + 0.00003381700928998009, + 0.000033933316440777846, + 0.000034071425988700565, + 0.000034101769523809526, + 0.00003419915593220339, + 0.000034770849785407726, + 0.0000347998, + 0.00003487867231638418, + 0.000035111405649717514, + 0.000035115347457627115, + 0.000035325899251191284, + 0.00003614243107344633, + 0.000036756152542372876, + 0.00003693492768361582, + 0.00003712097231638418, + 0.00003722009731543624, + 0.00003727163901791639, + 0.00003736029548022599, + 0.000037586394326241137, + 0.000037668203954802265, + 0.00003813341016949153, + 0.000038207128248587574, + 0.00003854229604519774, + 0.00003863840451977401, + 0.000039042309039548024, + 0.00003939215593220339, + 0.00003952434011299435, + 0.00003974015254237289, + 0.000040374732203389835, + 0.00004051842203389831, + 0.00004166252554744526, + 0.000041667992175273864, + 0.00006071488248587571 + ] + }, + "completed": true, + "percentSlower": 0 + } + ], + "fastest": { + "name": "send 1KiB of data over ws", + "index": 0 + }, + "slowest": { + "name": "send 1KiB of data over ws", + "index": 0 + } +} \ No newline at end of file diff --git a/benches/results/baseline_websocket/baseline_websocket_1KiB_metrics.txt b/benches/results/baseline_websocket/baseline_websocket_1KiB_metrics.txt new file mode 100644 index 00000000..00667f3e --- /dev/null +++ b/benches/results/baseline_websocket/baseline_websocket_1KiB_metrics.txt @@ -0,0 +1,8 @@ +# TYPE baseline_websocket.baseline_websocket_1KiB_ops gauge +baseline_websocket.baseline_websocket_1KiB_ops{name="send 1KiB of data over ws"} 29078 + +# TYPE baseline_websocket.baseline_websocket_1KiB_margin gauge +baseline_websocket.baseline_websocket_1KiB_margin{name="send 1KiB of data over ws"} 2.97 + +# TYPE baseline_websocket.baseline_websocket_1KiB_samples counter +baseline_websocket.baseline_websocket_1KiB_samples{name="send 1KiB of data over ws"} 75 diff --git a/benches/results/metrics.txt b/benches/results/metrics.txt index e69de29b..402234e2 100644 --- a/benches/results/metrics.txt +++ b/benches/results/metrics.txt @@ -0,0 +1,26 @@ +# TYPE baseline_tcp.baseline_tcp_1KiB_ops gauge +baseline_tcp.baseline_tcp_1KiB_ops{name="send 1KiB of data over ws"} 29374 + +# TYPE baseline_tcp.baseline_tcp_1KiB_margin gauge +baseline_tcp.baseline_tcp_1KiB_margin{name="send 1KiB of data over ws"} 1.9 + +# TYPE baseline_tcp.baseline_tcp_1KiB_samples counter +baseline_tcp.baseline_tcp_1KiB_samples{name="send 1KiB of data over ws"} 80 + +# TYPE baseline_websocket.baseline_websocket_1KiB_ops gauge +baseline_websocket.baseline_websocket_1KiB_ops{name="send 1KiB of data over ws"} 29078 + +# TYPE baseline_websocket.baseline_websocket_1KiB_margin gauge +baseline_websocket.baseline_websocket_1KiB_margin{name="send 1KiB of data over ws"} 2.97 + +# TYPE baseline_websocket.baseline_websocket_1KiB_samples counter +baseline_websocket.baseline_websocket_1KiB_samples{name="send 1KiB of data over ws"} 75 + +# TYPE stream.stream_1KiB_ops gauge +stream.stream_1KiB_ops{name="send 1KiB of data over stream"} 438 + +# TYPE stream.stream_1KiB_margin gauge +stream.stream_1KiB_margin{name="send 1KiB of data over stream"} 4.99 + +# TYPE stream.stream_1KiB_samples counter +stream.stream_1KiB_samples{name="send 1KiB of data over stream"} 78 diff --git a/benches/results/stream/stream_1KB.json b/benches/results/stream/stream_1KB.json deleted file mode 100644 index 4b54bbcb..00000000 --- a/benches/results/stream/stream_1KB.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "name": "stream.stream_1KB", - "date": "2023-09-07T07:19:18.204Z", - "version": "1.0.0", - "results": [ - { - "name": "send 1Kib of data over stream", - "ops": 426, - "margin": 5.06, - "options": { - "delay": 0.005, - "initCount": 1, - "minTime": 0.05, - "maxTime": 5, - "minSamples": 5 - }, - "samples": 73, - "promise": true, - "details": { - "min": 0.0017477503103448277, - "max": 0.0038379029285714288, - "mean": 0.0023471357312319016, - "median": 0.00215252308, - "standardDeviation": 0.0005182063925109443, - "marginOfError": 0.00011887688250114453, - "relativeMarginOfError": 5.064763870249276, - "standardErrorOfMean": 0.00006065147066384925, - "sampleVariance": 2.6853786523920685e-7, - "sampleResults": [ - 0.0017477503103448277, - 0.0017967768214285714, - 0.0018129551785714285, - 0.0018137015, - 0.0018192088214285715, - 0.001821007892857143, - 0.001821310357142857, - 0.0018235127142857143, - 0.0018691560740740742, - 0.0019064947857142856, - 0.0019268388214285714, - 0.0019428332962962962, - 0.0019438307857142856, - 0.001947187107142857, - 0.0019526353703703703, - 0.0019581806666666667, - 0.0019664113333333335, - 0.0019679321785714284, - 0.0019703470357142856, - 0.001991393259259259, - 0.0019938596785714284, - 0.0020057217407407407, - 0.0020108668214285715, - 0.002012867074074074, - 0.002023275, - 0.00202838416, - 0.0020436227142857145, - 0.00204693492, - 0.002052115392857143, - 0.00205514228, - 0.0020569102962962964, - 0.002079562962962963, - 0.0020855759200000003, - 0.0021088123571428572, - 0.0021170235, - 0.00212085862962963, - 0.00215252308, - 0.00215609404, - 0.0021769194642857143, - 0.002218121392857143, - 0.0022943591363636362, - 0.0022965015555555554, - 0.0023230972142857143, - 0.0023294068888888887, - 0.002330770888888889, - 0.002355380592592593, - 0.002362644107142857, - 0.0023833217142857142, - 0.002386992590909091, - 0.0024229923703703703, - 0.0024297490454545452, - 0.002452695, - 0.0025563194, - 0.0025570125714285716, - 0.002564137, - 0.0026277650952380954, - 0.0026469360357142856, - 0.0026939087142857144, - 0.002698705095238095, - 0.002751616052631579, - 0.002779424, - 0.0028510274444444443, - 0.0029493395789473684, - 0.0030082369285714286, - 0.003019166142857143, - 0.003035778, - 0.003149590392857143, - 0.003401115892857143, - 0.003438066714285714, - 0.0035103203809523807, - 0.0037650444375, - 0.0037869287333333335, - 0.0038379029285714288 - ] - }, - "completed": true, - "percentSlower": 0 - } - ], - "fastest": { - "name": "send 1Kib of data over stream", - "index": 0 - }, - "slowest": { - "name": "send 1Kib of data over stream", - "index": 0 - } -} \ No newline at end of file diff --git a/benches/results/stream/stream_1KB_metrics.txt b/benches/results/stream/stream_1KB_metrics.txt deleted file mode 100644 index 9f46257e..00000000 --- a/benches/results/stream/stream_1KB_metrics.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TYPE stream.stream_1KB_ops gauge -stream.stream_1KB_ops{name="send 1Kib of data over stream"} 426 - -# TYPE stream.stream_1KB_margin gauge -stream.stream_1KB_margin{name="send 1Kib of data over stream"} 5.06 - -# TYPE stream.stream_1KB_samples counter -stream.stream_1KB_samples{name="send 1Kib of data over stream"} 73 diff --git a/benches/results/stream/stream_1KB.chart.html b/benches/results/stream/stream_1KiB.chart.html similarity index 87% rename from benches/results/stream/stream_1KB.chart.html rename to benches/results/stream/stream_1KiB.chart.html index 50dcb9cf..23c176db 100644 --- a/benches/results/stream/stream_1KB.chart.html +++ b/benches/results/stream/stream_1KiB.chart.html @@ -5,7 +5,7 @@ - stream.stream_1KB + stream.stream_1KiB + + +
+ +
+ + + \ No newline at end of file diff --git a/benches/results/connection/connection_1KiB.json b/benches/results/connection/connection_1KiB.json new file mode 100644 index 00000000..c741f181 --- /dev/null +++ b/benches/results/connection/connection_1KiB.json @@ -0,0 +1,113 @@ +{ + "name": "connection.connection_1KiB", + "date": "2023-09-07T09:00:38.808Z", + "version": "1.0.0", + "results": [ + { + "name": "send 1KiB of data over connection", + "ops": 5450, + "margin": 3.11, + "options": { + "delay": 0.005, + "initCount": 1, + "minTime": 0.05, + "maxTime": 5, + "minSamples": 5 + }, + "samples": 69, + "promise": true, + "details": { + "min": 0.00014635003188405798, + "max": 0.00031344865507246375, + "mean": 0.00018349627201455138, + "median": 0.0001783529304347826, + "standardDeviation": 0.00002416628391625095, + "marginOfError": 0.000005702186262134717, + "relativeMarginOfError": 3.107521585878611, + "standardErrorOfMean": 0.000002909278705170774, + "sampleVariance": 5.840092783208493e-10, + "sampleResults": [ + 0.00014635003188405798, + 0.00015379765507246378, + 0.00015541868695652175, + 0.00015693411692307692, + 0.00015702844927536233, + 0.00015718554492753623, + 0.00015771045454545453, + 0.00015849292615384615, + 0.00016137123768115942, + 0.00016357428695652174, + 0.00016379861846153846, + 0.00016391912173913044, + 0.00016395645846153847, + 0.0001653712, + 0.00016644090434782608, + 0.00016657027536231883, + 0.00016775379692307693, + 0.0001678211304347826, + 0.00016904638260869566, + 0.00016980780289855071, + 0.00017110699420289855, + 0.00017199962898550725, + 0.00017207712753623187, + 0.00017239740289855073, + 0.00017239792783505154, + 0.00017418014782608697, + 0.00017456723768115943, + 0.00017470225507246377, + 0.0001760852925170068, + 0.0001764309188405797, + 0.00017673784057971014, + 0.00017693079094076655, + 0.00017738125087108015, + 0.0001776969652173913, + 0.0001783529304347826, + 0.00017931492932862192, + 0.00018050594699646643, + 0.00018050634782608695, + 0.00018186792173913044, + 0.00018304446956521738, + 0.0001833685331010453, + 0.0001833872231884058, + 0.00018511281449275362, + 0.0001865513971014493, + 0.00018670893333333334, + 0.00018751547826086957, + 0.0001890432927536232, + 0.00019061585217391305, + 0.0001921914956521739, + 0.00019225605653710247, + 0.00019258292173913042, + 0.0001935836956521739, + 0.00019463734782608697, + 0.00019636790808823528, + 0.00019958373913043478, + 0.00019979167491166076, + 0.0002005660492753623, + 0.0002018613884057971, + 0.0002028808144927536, + 0.00020298704057971012, + 0.00020755730144927538, + 0.00020906497101449276, + 0.0002094022492753623, + 0.00021408032173913047, + 0.00021466978840579712, + 0.000221389368115942, + 0.00022169565797101448, + 0.0002277073907563025, + 0.00031344865507246375 + ] + }, + "completed": true, + "percentSlower": 0 + } + ], + "fastest": { + "name": "send 1KiB of data over connection", + "index": 0 + }, + "slowest": { + "name": "send 1KiB of data over connection", + "index": 0 + } +} \ No newline at end of file diff --git a/benches/results/connection/connection_1KiB_metrics.txt b/benches/results/connection/connection_1KiB_metrics.txt new file mode 100644 index 00000000..0b773ca2 --- /dev/null +++ b/benches/results/connection/connection_1KiB_metrics.txt @@ -0,0 +1,8 @@ +# TYPE connection.connection_1KiB_ops gauge +connection.connection_1KiB_ops{name="send 1KiB of data over connection"} 5450 + +# TYPE connection.connection_1KiB_margin gauge +connection.connection_1KiB_margin{name="send 1KiB of data over connection"} 3.11 + +# TYPE connection.connection_1KiB_samples counter +connection.connection_1KiB_samples{name="send 1KiB of data over connection"} 69 diff --git a/benches/results/metrics.txt b/benches/results/metrics.txt index 0525b11d..ed655eeb 100644 --- a/benches/results/metrics.txt +++ b/benches/results/metrics.txt @@ -1,26 +1,35 @@ # TYPE baseline_tcp.baseline_tcp_1KiB_ops gauge -baseline_tcp.baseline_tcp_1KiB_ops{name="send 1KiB of data over ws"} 379014 +baseline_tcp.baseline_tcp_1KiB_ops{name="send 1KiB of data over ws"} 377058 # TYPE baseline_tcp.baseline_tcp_1KiB_margin gauge -baseline_tcp.baseline_tcp_1KiB_margin{name="send 1KiB of data over ws"} 1.63 +baseline_tcp.baseline_tcp_1KiB_margin{name="send 1KiB of data over ws"} 1.91 # TYPE baseline_tcp.baseline_tcp_1KiB_samples counter -baseline_tcp.baseline_tcp_1KiB_samples{name="send 1KiB of data over ws"} 77 +baseline_tcp.baseline_tcp_1KiB_samples{name="send 1KiB of data over ws"} 72 # TYPE baseline_websocket.baseline_websocket_1KiB_ops gauge -baseline_websocket.baseline_websocket_1KiB_ops{name="send 1KiB of data over ws"} 22668 +baseline_websocket.baseline_websocket_1KiB_ops{name="send 1KiB of data over ws"} 22863 # TYPE baseline_websocket.baseline_websocket_1KiB_margin gauge -baseline_websocket.baseline_websocket_1KiB_margin{name="send 1KiB of data over ws"} 3.42 +baseline_websocket.baseline_websocket_1KiB_margin{name="send 1KiB of data over ws"} 3.63 # TYPE baseline_websocket.baseline_websocket_1KiB_samples counter -baseline_websocket.baseline_websocket_1KiB_samples{name="send 1KiB of data over ws"} 70 +baseline_websocket.baseline_websocket_1KiB_samples{name="send 1KiB of data over ws"} 71 + +# TYPE connection.connection_1KiB_ops gauge +connection.connection_1KiB_ops{name="send 1KiB of data over connection"} 5450 + +# TYPE connection.connection_1KiB_margin gauge +connection.connection_1KiB_margin{name="send 1KiB of data over connection"} 3.11 + +# TYPE connection.connection_1KiB_samples counter +connection.connection_1KiB_samples{name="send 1KiB of data over connection"} 69 # TYPE stream.stream_1KiB_ops gauge -stream.stream_1KiB_ops{name="send 1KiB of data over stream"} 9738 +stream.stream_1KiB_ops{name="send 1KiB of data over stream"} 8390 # TYPE stream.stream_1KiB_margin gauge -stream.stream_1KiB_margin{name="send 1KiB of data over stream"} 3.08 +stream.stream_1KiB_margin{name="send 1KiB of data over stream"} 5.18 # TYPE stream.stream_1KiB_samples counter stream.stream_1KiB_samples{name="send 1KiB of data over stream"} 73 diff --git a/benches/results/stream/stream_1KiB.chart.html b/benches/results/stream/stream_1KiB.chart.html index 62717f52..d18fe306 100644 --- a/benches/results/stream/stream_1KiB.chart.html +++ b/benches/results/stream/stream_1KiB.chart.html @@ -28,7 +28,7 @@
- +