From 9e86048c41b7f4a59209a4a4fb9615742a73d2c1 Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti Date: Mon, 4 Jul 2022 17:51:18 +0300 Subject: [PATCH] refactor(util): use spawn instead of exec - Use child_process.spawn instead of child_process.exec --- lib/util/exec/common.ts | 107 +++++++++++++++++++++++++++++++++- lib/util/exec/index.spec.ts | 4 +- lib/util/exec/spawn-driver.ts | 46 +++++++++++++++ lib/util/exec/types.ts | 8 ++- package.json | 2 + tsconfig.app.json | 6 +- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 lib/util/exec/spawn-driver.ts diff --git a/lib/util/exec/common.ts b/lib/util/exec/common.ts index 80e3be2e7dfe04..fba88c9e6e21f0 100644 --- a/lib/util/exec/common.ts +++ b/lib/util/exec/common.ts @@ -1,8 +1,109 @@ -import { exec } from 'child_process'; -import { promisify } from 'util'; +import { ChildProcess, spawn } from 'child_process'; import type { ExecResult, RawExecOptions } from './types'; +// https://man7.org/linux/man-pages/man7/signal.7.html#NAME +// Non TERM/CORE signals +// The following is step 3. in https://github.com/renovatebot/renovate/issues/16197#issuecomment-1171423890 +// const NONTERM = [ +// 'SIGCHLD', +// 'SIGCLD', +// 'SIGCONT', +// 'SIGSTOP', +// 'SIGTSTP', +// 'SIGTTIN', +// 'SIGTTOU', +// 'SIGURG', +// 'SIGWINCH', +// ]; + +function stringify(stream: Buffer[], encoding: BufferEncoding): string { + return Buffer.concat(stream).toString(encoding); +} + +function initStreamListeners( + cp: ChildProcess, + opts: RawExecOptions & { maxBuffer: number; encoding: BufferEncoding } +): [Buffer[], Buffer[]] { + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let stdoutLen = 0; + let stderrLen = 0; + + cp.stdout?.on('data', (data: Buffer) => { + // process.stdout.write(data.toString()); + const len = Buffer.byteLength(data, opts.encoding); + stdoutLen += len; + if (stdoutLen > opts.maxBuffer) { + cp.emit('error', new Error('exceeded max buffer size for stdout')); + } else { + stdout.push(data); + } + }); + cp.stderr?.on('data', (data: Buffer) => { + // process.stderr.write(data.toString()); + const len = Buffer.byteLength(data, opts.encoding); + stderrLen += len; + if (stderrLen > opts.maxBuffer) { + cp.emit('error', new Error('exceeded max buffer size for stderr')); + } else { + stderr.push(data); + } + }); + return [stdout, stderr]; +} + +function promisifySpawn( + cmd: string, + opts: RawExecOptions +): Promise { + return new Promise((resolve, reject) => { + const encoding = opts.encoding as BufferEncoding; + const [command, ...args] = cmd.split(/\s+/); + const maxBuffer = opts.maxBuffer ?? 10 * 1024 * 1024; // Set default max buffer size to 10MB + // The following is step 3. in https://github.com/renovatebot/renovate/issues/16197#issuecomment-1171423890 + // const cp = spawn(command, args, { ...opts, detached: true }); // PID range hack; force detached + const cp = spawn(command, args, opts); // PID range hack; force detached + const [stdout, stderr] = initStreamListeners(cp, { + ...opts, + maxBuffer, + encoding, + }); // handle streams + // handle process events + cp.on('error', (error) => { + reject(error.message); + }); + + cp.on('exit', (code: number, signal: string) => { + // The following is step 3. in https://github.com/renovatebot/renovate/issues/16197#issuecomment-1171423890 + // if (signal && !NONTERM.includes(signal)) { + // try { + // process.kill(-(cp.pid as number), signal); // PID range hack; signal process tree + // } catch (err) { + // // cp is a single node tree, therefore -pid is invalid, + // } + // stderr.push( + // Buffer.from( + // `PID= ${cp.pid as number}\n` + + // `COMMAND= "${cp.spawnargs.join(' ')}"\n` + + // `Signaled with "${signal}"` + // ) + // ); + // reject(stringify(stderr, encoding)); + // return; + // } + if (code !== 0) { + reject(stringify(stderr, encoding)); + return; + } + resolve({ + stderr: stringify(stderr, encoding), + stdout: stringify(stdout, encoding), + }); + }); + }); +} + export const rawExec: ( cmd: string, opts: RawExecOptions -) => Promise = promisify(exec); +) => Promise = promisifySpawn; diff --git a/lib/util/exec/index.spec.ts b/lib/util/exec/index.spec.ts index 61fe5ed66b9ae1..116f47e66f2861 100644 --- a/lib/util/exec/index.spec.ts +++ b/lib/util/exec/index.spec.ts @@ -1,6 +1,6 @@ import { ExecOptions as ChildProcessExecOptions, - exec as _cpExec, + spawn as _cpSpawn, } from 'child_process'; import { envMock } from '../../../test/exec-util'; import { GlobalConfig } from '../../config/global'; @@ -10,7 +10,7 @@ import * as dockerModule from './docker'; import type { ExecOptions, RawExecOptions, VolumeOption } from './types'; import { exec } from '.'; -const cpExec: jest.Mock = _cpExec as any; +const cpExec: jest.Mock = _cpSpawn as any; jest.mock('child_process'); jest.mock('../../modules/datasource'); diff --git a/lib/util/exec/spawn-driver.ts b/lib/util/exec/spawn-driver.ts new file mode 100644 index 00000000000000..fa14a400147db9 --- /dev/null +++ b/lib/util/exec/spawn-driver.ts @@ -0,0 +1,46 @@ +import { logger } from '../../logger'; +import { rawExec } from './common'; +// import { rawExec } from './common'; +import type { RawExecOptions } from './types'; + +void (async () => { + const cmds: [string, RawExecOptions][] = []; + const opts: RawExecOptions = { + encoding: 'utf8', + shell: true, + timeout: 2000, + }; + logger.info('driver function - START'); + cmds.push(['npm run non-existent-script', opts]); + cmds.push(['docker', { ...opts, shell: false }]); + cmds.push(['docker image rm alpine', { ...opts, timeout: 0 }]); + cmds.push(['docker images', opts]); + cmds.push(['docker pull alpine', { ...opts, timeout: 0 }]); + cmds.push(['docker images', opts]); + cmds.push(['npm run spawn-testing-script', opts]); + cmds.push(['npm run spawn-testing-script', { ...opts, shell: false }]); + cmds.push(['sleep 900', opts]); + cmds.push(['sleep 900', { ...opts, shell: false }]); + cmds.push(['sleep 900', { ...opts, shell: '/bin/bash' }]); + cmds.push(['ls -l /', { ...opts, timeout: 0, maxBuffer: 100 }]); + cmds.push(['ps -auxf', opts]); + + for (const [cmd, opts] of cmds) { + logger.info('-------------------------------------------------------'); + logger.info({ opts }, `Run rawSpawn() - START - "${cmd}"`); + try { + const { stdout, stderr } = await rawExec(cmd, opts); + // const { stdout, stderr } = await rawExec(cmd, {encoding: 'utf8', timeout: 0}); + if (stdout) { + logger.info(stdout); + } + if (stderr) { + logger.warn(stderr); + } + } catch (err) { + logger.error(err as string); + } + logger.info(`run cmd - END - "${cmd}"`); + } + logger.info('driver function - END'); +})(); diff --git a/lib/util/exec/types.ts b/lib/util/exec/types.ts index b97279b2af8675..e08da665765d53 100644 --- a/lib/util/exec/types.ts +++ b/lib/util/exec/types.ts @@ -1,4 +1,4 @@ -import type { ExecOptions as ChildProcessExecOptions } from 'child_process'; +import type { SpawnOptions as ChildProcessSpawnOptions } from 'child_process'; export interface ToolConstraint { toolName: string; @@ -27,10 +27,13 @@ export interface DockerOptions { cwd?: Opt; } -export interface RawExecOptions extends ChildProcessExecOptions { +// this will be renamed later on, left as is to minimize PR diff +export interface RawExecOptions extends ChildProcessSpawnOptions { encoding: string; + maxBuffer?: number | undefined; } +// this will be renamed later on, left as is to minimize PR diff export interface ExecResult { stdout: string; stderr: string; @@ -38,6 +41,7 @@ export interface ExecResult { export type ExtraEnv = Record; +// this will be renamed later on, left as is to minimize PR diff export interface ExecOptions { cwd?: string; cwdFile?: string; diff --git a/package.json b/package.json index e6339412499fbe..efc1c1cafc31d8 100644 --- a/package.json +++ b/package.json @@ -45,10 +45,12 @@ "release:prepare": "node -r ts-node/register/transpile-only -- tools/generate-docs.ts", "release:publish": "node tools/release.mjs", "start": "node -r ts-node/register/transpile-only -- lib/renovate.ts", + "spwan-driver": "node -r ts-node/register/transpile-only -- lib/util/exec/spawn-driver.ts", "test": "run-s lint test-schema type-check strict-check jest", "test-dirty": "git diff --exit-code", "test-e2e": "yarn pack && cd test/e2e && yarn install --no-lockfile --ignore-optional --prod && yarn test", "test-schema": "run-s create-json-schema", + "spawn-testing-script": "echo 'long prepare for 900 sec' && sleep 900", "tsc": "tsc", "type-check": "run-s generate:* \"tsc --noEmit {@}\" --", "update:distro-info": "node tools/distro-json-generate.mjs", diff --git a/tsconfig.app.json b/tsconfig.app.json index 01612b0cf5c494..82433ab1e429df 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -8,6 +8,10 @@ "importHelpers": true, "types": ["node"] }, - "files": ["lib/renovate.ts", "lib/config-validator.ts"], + "files": [ + "lib/renovate.ts", + "lib/config-validator.ts", + "lib/util/exec/spawn-driver.ts" + ], "include": ["lib/**/*.d.ts"] }