feat: get interactive shell into jobs running in a container

README.md

@@ -326,11 +326,9 @@ npm-install:
 ```yml
 # @Interactive
 interactive-shell:
-  rules:
-    - if: $GITLAB_CI == 'false'
-      when: manual
+  image: debian:latest
   script:
-    - docker run -it debian bash
+    - echo "this i not being executed in interactive mode"
 ```
 
 ![description-decorator](./docs/images/interactive-decorator.png)

package.json

@@ -39,7 +39,6 @@
     "jsonpointer": "5.x.x",
     "micromatch": "4.x.x",
     "object-traversal": "1.x.x",
-    "p-map": "4.x.x",
     "pretty-hrtime": "1.x.x",
     "re2": "^1.21.4",
     "split2": "4.x.x",

src/argv.ts

@@ -141,6 +141,14 @@ export class Argv {
         return this.map.get("remoteVariables");
     }
 
+    get interactiveJobs (): string[] {
+        return this.map.get("interactiveJobs") ?? [];
+    }
+
+    get debug (): boolean {
+        return this.map.get("debug") ?? false;
+    }
+
     get variable (): {[key: string]: string} {
         const val = this.map.get("variable");
         const variables: {[key: string]: string} = {};

src/executor.ts

@@ -2,18 +2,17 @@ import chalk from "chalk";
 import {Job} from "./job.js";
 import assert, {AssertionError} from "assert";
 import {Argv} from "./argv.js";
-import pMap from "p-map";
+import {runMultipleJobs} from "./multi-job-runner.js";
 
 export class Executor {
 
     static async runLoop (argv: Argv, jobs: ReadonlyArray<Job>, stages: readonly string[], potentialStarters: Job[]) {
-        let startCandidates = [];
+        let startCandidates: Job[] = [];
 
         do {
             startCandidates = Executor.getStartCandidates(jobs, stages, potentialStarters, argv.manual);
             if (startCandidates.length > 0) {
-                const mapper = async (startCandidate: Job) => startCandidate.start();
-                await pMap(startCandidates, mapper, {concurrency: argv.concurrency ?? startCandidates.length});
+                await runMultipleJobs(startCandidates, {concurrency: argv.concurrency ?? startCandidates.length});
             }
         } while (startCandidates.length > 0);
     }

src/index.ts

@@ -100,6 +100,18 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs));
             description: "Print YML with defaults, includes, extends and reference's expanded",
             requiresArg: false,
         })
+        .option("interactive-jobs", {
+            type: "array",
+            alias: "i",
+            description: "Select jobs to run interactively",
+            requiresArg: false,
+        })
+        .option("debug", {
+            type: "boolean",
+            alias: "d",
+            description: "Open an interactive shell for failed jobs",
+            requiresArg: false,
+        })
         .option("cwd", {
             type: "string",
             description: "Path to a current working directory",

src/job.ts

@@ -20,6 +20,27 @@ import {validateIncludeLocal} from "./parser-includes.js";
 
 const CI_PROJECT_DIR = "/gcl-builds";
 const GCL_SHELL_PROMPT_PLACEHOLDER = "<gclShellPromptPlaceholder>";
+const SHELL_CMD = `sh -c "
+if [ -x /usr/local/bin/bash ]; then
+    exec /usr/local/bin/bash
+elif [ -x /usr/bin/bash ]; then
+    exec /usr/bin/bash 
+elif [ -x /bin/bash ]; then
+    exec /bin/bash
+elif [ -x /usr/local/bin/sh ]; then
+    exec /usr/local/bin/sh
+elif [ -x /usr/bin/sh ]; then
+    exec /usr/bin/sh
+elif [ -x /bin/sh ]; then
+    exec /bin/sh
+elif [ -x /busybox/sh ]; then
+    exec /busybox/sh
+else
+    echo shell not found
+    exit 1
+fi"
+`;
+
 interface JobOptions {
     argv: Argv;
     writeStreams: WriteStreams;
@@ -244,11 +265,6 @@ export class Job {
 
         assert(this.scripts || this.trigger, chalk`{blueBright ${this.name}} must have script specified`);
 
-        assert(!(this.interactive && !this.argv.shellExecutorNoImage), chalk`${this.formattedJobName} @Interactive decorator cannot be used with --no-shell-executor-no-image`);
-
-        if (this.interactive && (this.when !== "manual" || this.imageName(this._variables) !== null)) {
-            throw new AssertionError({message: `${this.formattedJobName} @Interactive decorator cannot have image: and must be when:manual`});
-        }
 
         if (this.injectSSHAgent && this.imageName(this._variables) === null) {
             throw new AssertionError({message: `${this.formattedJobName} @InjectSSHAgent can only be used with image:`});
@@ -390,7 +406,9 @@ export class Job {
     }
 
     get interactive (): boolean {
-        return this.jobData["gclInteractive"] || false;
+        return this.jobData["gclInteractive"]
+            || this.argv.interactiveJobs.indexOf(this.baseName) >= 0
+            || false;
     }
 
     get injectSSHAgent (): boolean {
@@ -597,7 +615,15 @@ export class Job {
         await this.execPreScripts(expanded);
         if (this._prescriptsExitCode == null) throw Error("this._prescriptsExitCode must be defined!");
 
-        await this.execAfterScripts(expanded);
+        if (this.argv.debug && this.jobStatus === "failed") {
+            // To successfully finish the job, someone has to call debug();
+            clearInterval(this._longRunningSilentTimeout);
+            return;
+        }
+
+        if (!this.interactive) {
+            await this.execAfterScripts(expanded);
+        }
 
         this._running = false;
         this._endTime = this._endTime ?? process.hrtime(this._startTime);
@@ -613,6 +639,51 @@ export class Job {
         this.cleanupResources();
     }
 
+    async debug (): Promise<void> {
+        // stop still running message in debug mode
+        clearInterval(this._longRunningSilentTimeout);
+        this.writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright starting debug shell}\n`);
+
+        const cwd = this.argv.cwd;
+        const expanded = Utils.unscape$$Variables(Utils.expandVariables({...this._variables, ...this._dotenvVariables}));
+        const imageName = this.imageName(expanded);
+
+        if (this._containerId) {
+            await execa(`${this.argv.containerExecutable} start ${this._containerId}`, {
+                shell: "bash",
+                env: imageName ? process.env : expanded,
+            });
+        }
+
+        try {
+            await execa(this._containerId ? `DOCKER_CLI_HINTS=false ${this.argv.containerExecutable} exec -it ${this._containerId} ${SHELL_CMD}` : "bash", {
+                cwd,
+                shell: "bash",
+                stdio: "inherit",
+                env: imageName ? process.env : expanded,
+            });
+        } catch (e) {
+            // nothing to do, failing is allowed
+        }
+
+        if (!this.interactive) {
+            await this.execAfterScripts(expanded);
+        }
+
+        this._running = false;
+        this._endTime = this._endTime ?? process.hrtime(this._startTime);
+        this.printFinishedString();
+
+        await this.copyCacheOut(this.writeStreams, expanded);
+        await this.copyArtifactsOut(this.writeStreams, expanded);
+
+        if (this.jobData["coverage"]) {
+            this._coveragePercent = await Utils.getCoveragePercent(this.argv.cwd, this.argv.stateDir, this.jobData["coverage"], this.safeJobName);
+        }
+
+        this.cleanupResources();
+    }
+
     async cleanupResources () {
         clearTimeout(this._longRunningSilentTimeout);
 
@@ -732,26 +803,14 @@ export class Job {
             await Utils.rsyncTrackedFiles(cwd, stateDir, `${safeJobName}`);
         }
 
-        if (this.interactive) {
-            let iCmd = "set -eo pipefail\n";
-            iCmd += this.generateScriptCommands(scripts);
-
-            const interactiveCp = execa(iCmd, {
-                cwd,
-                shell: "bash",
-                stdio: ["inherit", "inherit", "inherit"],
-                env: {...expanded, ...process.env},
-            });
-            return new Promise<number>((resolve, reject) => {
-                void interactiveCp.on("exit", (code) => resolve(code ?? 0));
-                void interactiveCp.on("error", (err) => reject(err));
-            });
-        }
-
         this.refreshLongRunningSilentTimeout(writeStreams);
 
         if (imageName && !this._containerId) {
             let dockerCmd = `${this.argv.containerExecutable} create --interactive ${this.generateInjectSSHAgentOptions()} `;
+            if (this.interactive) {
+                dockerCmd += "--tty ";
+            }
+
             if (this.argv.privileged) {
                 dockerCmd += "--privileged ";
             }
@@ -842,25 +901,7 @@ export class Job {
                 });
             }
 
-            dockerCmd += "sh -c \"\n";
-            dockerCmd += "if [ -x /usr/local/bin/bash ]; then\n";
-            dockerCmd += "\texec /usr/local/bin/bash \n";
-            dockerCmd += "elif [ -x /usr/bin/bash ]; then\n";
-            dockerCmd += "\texec /usr/bin/bash \n";
-            dockerCmd += "elif [ -x /bin/bash ]; then\n";
-            dockerCmd += "\texec /bin/bash \n";
-            dockerCmd += "elif [ -x /usr/local/bin/sh ]; then\n";
-            dockerCmd += "\texec /usr/local/bin/sh \n";
-            dockerCmd += "elif [ -x /usr/bin/sh ]; then\n";
-            dockerCmd += "\texec /usr/bin/sh \n";
-            dockerCmd += "elif [ -x /bin/sh ]; then\n";
-            dockerCmd += "\texec /bin/sh \n";
-            dockerCmd += "elif [ -x /busybox/sh ]; then\n";
-            dockerCmd += "\texec /busybox/sh \n";
-            dockerCmd += "else\n";
-            dockerCmd += "\techo shell not found\n";
-            dockerCmd += "\texit 1\n";
-            dockerCmd += "fi\n\"";
+            dockerCmd += SHELL_CMD;
 
             const {stdout: containerId} = await Utils.bash(dockerCmd, cwd);
 
@@ -907,9 +948,14 @@ export class Job {
             await Utils.spawn([this.argv.containerExecutable, "cp", `${stateDir}/scripts/${safeJobName}_${this.jobId}`, `${this._containerId}:/gcl-cmd`], cwd);
         }
 
+        if (this.interactive) {
+            this.writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright starting interactive shell}\n`);
+        }
+
         const cp = execa(this._containerId ? `${this.argv.containerExecutable} start --attach -i ${this._containerId}` : "bash", {
             cwd,
             shell: "bash",
+            stdio: this.interactive ? "inherit" : undefined,
             env: imageName ? process.env : expanded,
         });
 
@@ -930,17 +976,22 @@ export class Job {
         const quiet = this.argv.quiet;
 
         return await new Promise<number>((resolve, reject) => {
-            if (!quiet) {
+            if (!quiet && !this.interactive) {
                 cp.stdout?.pipe(split2()).on("data", (e: string) => outFunc(e, writeStreams.stdout.bind(writeStreams), (s) => chalk`{greenBright ${s}}`));
                 cp.stderr?.pipe(split2()).on("data", (e: string) => outFunc(e, writeStreams.stderr.bind(writeStreams), (s) => chalk`{redBright ${s}}`));
             }
             void cp.on("exit", (code) => resolve(code ?? 0));
             void cp.on("error", (err) => reject(err));
 
-            if (imageName) {
-                cp.stdin?.end("/gcl-cmd");
+            if (this.interactive) {
+                // stop from showing still running message in interactive mode
+                clearTimeout(this._longRunningSilentTimeout);
             } else {
-                cp.stdin?.end(`./${stateDir}/scripts/${safeJobName}_${this.jobId}`);
+                if (imageName) {
+                    cp.stdin?.end("/gcl-cmd");
+                } else {
+                    cp.stdin?.end(`./${stateDir}/scripts/${safeJobName}_${this.jobId}`);
+                }
             }
         });
     }

src/multi-job-runner.ts

@@ -0,0 +1,62 @@
+import {Job} from "./job";
+
+
+export type MultiJobRunnerOptions = {
+    concurrency: number;
+};
+
+/**
+ * Run multiple jobs in parallel, if a job is interactive it will no be run in parallel.
+ *
+ * @param jobs the rubs to run in parallel
+ * @param options parallelization options
+ */
+export async function runMultipleJobs (jobs: Job[], options: MultiJobRunnerOptions): Promise<void> {
+    const activeJobsById: Map<number, Promise<void>> = new Map();
+    const jobsToDebug: Job[] = [];
+    const exceptions: unknown[] = [];
+
+    for (const job of jobs) {
+        await debugJobs(jobsToDebug);
+
+        if (job.interactive) {
+            await Promise.all(activeJobsById.values());
+        }
+
+        if (activeJobsById.size >= options.concurrency) {
+            await Promise.any(activeJobsById.values());
+        }
+
+        const execution = job.start();
+        activeJobsById.set(job.jobId, execution);
+        execution.then(() => {
+            activeJobsById.delete(job.jobId);
+            if (job.argv.debug && job.jobStatus === "failed") {
+                jobsToDebug.push(job);
+            }
+        }).catch((e) => exceptions.push(e));
+
+        if (job.interactive) {
+            await execution;
+        }
+    }
+
+    await Promise.all(activeJobsById.values());
+    await throwExceptions(exceptions);
+    await debugJobs(jobsToDebug);
+}
+
+async function throwExceptions (exceptions: unknown[]): Promise<void> {
+    if (exceptions.length > 0) {
+        throw exceptions[0];
+    }
+}
+
+async function debugJobs (jobs: Job[]): Promise<void> {
+    while (jobs.length > 0) {
+        const job = jobs.shift();
+        if (job) {
+            await job.debug();
+        }
+    }
+}