feat: support disabling serving

This is useful when environments don't have a server ready (or are
targeting other languages without a web UI).

Author: devversion

runner/orchestration/executors/executor.ts

@@ -72,7 +72,7 @@ export const executorSchema = z.object({
           )
           .describe('Call this function while the server is running'),
       ]),
-      z.promise(z.custom<ServeTestingResult>()),
+      z.promise(z.union([z.custom<ServeTestingResult>(), z.null()])),
     )
     .nullable(),
   executeProjectTests: z.function(

runner/orchestration/executors/local-executor-config.ts

@@ -21,9 +21,11 @@ export const localExecutorConfigSchema = z.strictObject({
   buildCommand: z.string().optional(),
   /**
    * Command to run when starting a development server inside the app.
-   * Defaults to `<package manager> run start --port 0`.
+   *
+   * When `undefined`, defaults to `<package manager> run start --port 0`.
+   * When `null`, the app has no server and no runtime testing will occur.
    */
-  serveCommand: z.string().optional(),
+  serveCommand: z.string().optional().nullable(),
   /**
    * Optional command for executing project tests.
    */

runner/orchestration/executors/local-executor.ts

@@ -27,6 +27,7 @@ import {callWithTimeout} from '../../utils/timeout.js';
 import {executeCommand} from '../../utils/exec.js';
 import {cleanupBuildMessage} from '../../workers/builder/worker.js';
 import {combineAbortSignals} from '../../utils/abort-signal.js';
+import {ServeTestingResult} from '../../workers/serve-testing/worker-types.js';
 
 let uniqueIDs = 0;
 
@@ -172,13 +173,18 @@ export class LocalExecutor implements Executor {
     } satisfies TestExecutionResult;
   }
 
-  async serveWebApplication<T>(
+  async serveWebApplication(
     _id: EvalID,
     appDirectoryPath: string,
     rootPromptDef: RootPromptDefinition,
     progress: ProgressLogger,
-    logicWhileServing: (serveUrl: string) => Promise<T>,
-  ): Promise<T> {
+    logicWhileServing: (serveUrl: string) => Promise<ServeTestingResult>,
+  ): Promise<ServeTestingResult | null> {
+    // Serve testing is explicitly disabled.
+    if (this.config.serveCommand === null) {
+      return null;
+    }
+
     return await serveApp(
       this.getServeCommand(),
       rootPromptDef,
@@ -207,7 +213,7 @@ export class LocalExecutor implements Executor {
   }
 
   getServeCommand(): string {
-    if (this.config.serveCommand !== undefined) {
+    if (this.config.serveCommand != null) {
       return this.config.serveCommand;
     }
 

runner/orchestration/serve-testing-worker.ts

@@ -87,6 +87,12 @@ export async function serveAndTestApp(
       },
     );
 
+    // An executor might define `serveWebApplication` but conditionally decide
+    // that no web application can be started/served.
+    if (result === null) {
+      return null;
+    }
+
     if (result.errorMessage === undefined) {
       progress.log(rootPromptDef, 'success', 'Validation of running app is successful');
     } else {