fix(core): handle zombie processes

 - Use child_process.spawn instead of child_process.exec

lib/util/exec/common.ts

@@ -1,8 +1,106 @@
-import { exec } from 'child_process';
-import { promisify } from 'util';
+import { ChildProcess, spawn } from 'child_process';
 import type { ExecResult, RawExecOptions } from './types';
 
+// https://man7.org/linux/man-pages/man7/signal.7.html#NAME
+// Non TERM/CORE signals
+const NONTERM = [
+  'SIGCHLD',
+  'SIGCLD',
+  'SIGCONT',
+  'SIGSTOP',
+  'SIGTSTP',
+  'SIGTTIN',
+  'SIGTTOU',
+  'SIGURG',
+  'SIGWINCH',
+];
+
+function stringify(stream: Buffer[], encoding: BufferEncoding): string {
+  return Buffer.concat(stream).toString(encoding);
+}
+
+function initStreamListeners(
+  cp: ChildProcess,
+  opts: RawExecOptions & { maxBuffer: number; encoding: BufferEncoding }
+): [Buffer[], Buffer[]] {
+  const stdout: Buffer[] = [];
+  const stderr: Buffer[] = [];
+  let stdoutLen = 0;
+  let stderrLen = 0;
+
+  cp.stdout?.on('data', (data: Buffer) => {
+    // process.stdout.write(data.toString());
+    const len = Buffer.byteLength(data, opts.encoding);
+    stdoutLen += len;
+    if (stdoutLen > opts.maxBuffer) {
+      cp.emit('error', new Error('exceeded max buffer size for stdout'));
+    } else {
+      stdout.push(data);
+    }
+  });
+  cp.stderr?.on('data', (data: Buffer) => {
+    // process.stderr.write(data.toString());
+    const len = Buffer.byteLength(data, opts.encoding);
+    stderrLen += len;
+    if (stderrLen > opts.maxBuffer) {
+      cp.emit('error', new Error('exceeded max buffer size for stderr'));
+    } else {
+      stderr.push(data);
+    }
+  });
+  return [stdout, stderr];
+}
+
+function promisifySpawn(
+  cmd: string,
+  opts: RawExecOptions
+): Promise<ExecResult> {
+  return new Promise((resolve, reject) => {
+    const encoding = opts.encoding as BufferEncoding;
+    const [command, ...args] = cmd.split(/\s+/);
+    const maxBuffer = opts.maxBuffer ?? 10 * 1024 * 1024; // Set default max buffer size to 10MB
+    const cp = spawn(command, args, { ...opts, detached: true }); // PID range hack; force detached
+    const [stdout, stderr] = initStreamListeners(cp, {
+      ...opts,
+      maxBuffer,
+      encoding,
+    }); // handle streams
+
+    // handle process events
+    cp.on('error', (error) => {
+      reject(error.message);
+    });
+
+    cp.on('exit', (code: number, signal: string) => {
+      if (signal && !NONTERM.includes(signal)) {
+        try {
+          process.kill(-(cp.pid as number), signal); // PID range hack; signal process tree
+        } catch (err) {
+          // cp is a single node tree, therefore -pid is invalid,
+        }
+        stderr.push(
+          Buffer.from(
+            `PID= ${cp.pid as number}\n` +
+              `COMMAND= "${cp.spawnargs.join(' ')}"\n` +
+              `Signaled with "${signal}"`
+          )
+        );
+        reject(stringify(stderr, encoding));
+        return;
+      }
+      if (code !== 0) {
+        reject(stringify(stderr, encoding));
+        return;
+      }
+      resolve({
+        stderr: stringify(stderr, encoding),
+        stdout: stringify(stdout, encoding),
+      });
+    });
+  });
+}
+
 export const rawExec: (
   cmd: string,
   opts: RawExecOptions
-) => Promise<ExecResult> = promisify(exec);
+) => Promise<ExecResult> = promisifySpawn;

lib/util/exec/index.spec.ts

@@ -1,6 +1,6 @@
 import {
   ExecOptions as ChildProcessExecOptions,
-  exec as _cpExec,
+  spawn as _cpSpawn,
 } from 'child_process';
 import { envMock } from '../../../test/exec-util';
 import { GlobalConfig } from '../../config/global';
@@ -10,7 +10,7 @@ import * as dockerModule from './docker';
 import type { ExecOptions, RawExecOptions, VolumeOption } from './types';
 import { exec } from '.';
 
-const cpExec: jest.Mock<typeof _cpExec> = _cpExec as any;
+const cpExec: jest.Mock<typeof _cpSpawn> = _cpSpawn as any;
 
 jest.mock('child_process');
 jest.mock('../../modules/datasource');

lib/util/exec/spawn-driver.ts

@@ -0,0 +1,46 @@
+import { logger } from '../../logger';
+import { rawExec } from './common';
+// import { rawExec } from './common';
+import type { RawExecOptions } from './types';
+
+void (async () => {
+  const cmds: [string, RawExecOptions][] = [];
+  const opts: RawExecOptions = {
+    encoding: 'utf8',
+    shell: true,
+    timeout: 2000,
+  };
+  logger.info('driver function - START');
+  cmds.push(['npm run non-existent-script', opts]);
+  cmds.push(['docker', { ...opts, shell: false }]);
+  cmds.push(['docker image rm alpine', { ...opts, timeout: 0 }]);
+  cmds.push(['docker images', opts]);
+  cmds.push(['docker pull alpine', { ...opts, timeout: 0 }]);
+  cmds.push(['docker images', opts]);
+  cmds.push(['npm run spawn-testing-script', opts]);
+  cmds.push(['npm run spawn-testing-script', { ...opts, shell: false }]);
+  cmds.push(['sleep 900', opts]);
+  cmds.push(['sleep 900', { ...opts, shell: false }]);
+  cmds.push(['sleep 900', { ...opts, shell: '/bin/bash' }]);
+  cmds.push(['ls -l /', { ...opts, timeout: 0, maxBuffer: 100 }]);
+  cmds.push(['ps -auxf', opts]);
+
+  for (const [cmd, opts] of cmds) {
+    logger.info('-------------------------------------------------------');
+    logger.info({ opts }, `Run rawSpawn() - START - "${cmd}"`);
+    try {
+      const { stdout, stderr } = await rawExec(cmd, opts);
+      // const { stdout, stderr } = await rawExec(cmd, {encoding: 'utf8', timeout: 0});
+      if (stdout) {
+        logger.info(stdout);
+      }
+      if (stderr) {
+        logger.warn(stderr);
+      }
+    } catch (err) {
+      logger.error(err as string);
+    }
+    logger.info(`run cmd - END - "${cmd}"`);
+  }
+  logger.info('driver function - END');
+})();

lib/util/exec/types.ts

@@ -1,4 +1,4 @@
-import type { ExecOptions as ChildProcessExecOptions } from 'child_process';
+import type { SpawnOptions as ChildProcessSpawnOptions } from 'child_process';
 
 export interface ToolConstraint {
   toolName: string;
@@ -27,17 +27,21 @@ export interface DockerOptions {
   cwd?: Opt<string>;
 }
 
-export interface RawExecOptions extends ChildProcessExecOptions {
+// this will be renamed later on, left as is to minimize PR diff
+export interface RawExecOptions extends ChildProcessSpawnOptions {
   encoding: string;
+  maxBuffer?: number | undefined;
 }
 
+// this will be renamed later on, left as is to minimize PR diff
 export interface ExecResult {
   stdout: string;
   stderr: string;
 }
 
 export type ExtraEnv<T = unknown> = Record<string, T>;
 
+// this will be renamed later on, left as is to minimize PR diff
 export interface ExecOptions {
   cwd?: string;
   cwdFile?: string;

package.json

@@ -45,10 +45,12 @@
     "release:prepare": "node -r ts-node/register/transpile-only -- tools/generate-docs.ts",
     "release:publish": "node tools/release.mjs",
     "start": "node -r ts-node/register/transpile-only -- lib/renovate.ts",
+    "spwan-driver": "node -r ts-node/register/transpile-only -- lib/util/exec/spawn-driver.ts",
     "test": "run-s lint test-schema type-check strict-check jest",
     "test-dirty": "git diff --exit-code",
     "test-e2e": "yarn pack && cd test/e2e && yarn install --no-lockfile --ignore-optional --prod && yarn test",
     "test-schema": "run-s create-json-schema",
+    "spawn-testing-script": "echo 'long prepare for 900 sec' && sleep 900",
     "tsc": "tsc",
     "type-check": "run-s generate:* \"tsc --noEmit {@}\" --",
     "update:distro-info": "node tools/distro-json-generate.mjs",

tsconfig.app.json

@@ -8,6 +8,10 @@
     "importHelpers": true,
     "types": ["node"]
   },
-  "files": ["lib/renovate.ts", "lib/config-validator.ts"],
+  "files": [
+    "lib/renovate.ts",
+    "lib/config-validator.ts",
+    "lib/util/exec/spawn-driver.ts"
+  ],
   "include": ["lib/**/*.d.ts"]
 }