fix: OOM error

src/CodeBlockExtractor.ts

@@ -1,45 +1,60 @@
-import * as fsExtra from "fs-extra";
+import * as fs from "fs";
 
 // eslint-disable-next-line @typescript-eslint/no-extraneous-class
 export class CodeBlockExtractor {
+  /**
+   * Matches TS/TSX fences not preceded by an ignore directive.
+   * Captures:
+   *   1: 'tsx' if TSX, undefined if 'typescript' or 'ts'
+   *   2: the code inside the fence
+   */
   static readonly TYPESCRIPT_CODE_PATTERN =
-    /(?<!(?:<!--\s*ts-docs-verifier:ignore\s*-->[\r?\n]*))(?:```(?:(?:typescript)|(tsx?))\r?\n)((?:\r?\n|.)*?)(?:(?=```))/gi;
+    /(?<!(?:<!--\s*ts-docs-verifier:ignore\s*-->[\r?\n]*))```(?:(?:typescript)|(tsx?))\r?\n([\s\S]*?)(?=```)/gi;
 
-  /* istanbul ignore next */
-  private constructor() {
-    //
-  }
+  private constructor() {}
 
+  /**
+   * Extract all code blocks into an array (backward-compatible).
+   */
   static async extract(
     markdownFilePath: string
   ): Promise<{ code: string; type: "tsx" | "ts" }[]> {
-    try {
-      const contents = await CodeBlockExtractor.readFile(markdownFilePath);
-      return CodeBlockExtractor.extractCodeBlocksFromMarkdown(contents);
-    } catch (error) {
-      throw new Error(
-        `Error extracting code blocks from ${markdownFilePath}: ${
-          error instanceof Error ? error.message : error
-        }`
-      );
+    const blocks: { code: string; type: "tsx" | "ts" }[] = [];
+    for await (const block of this.iterateBlocks(markdownFilePath)) {
+      blocks.push(block);
     }
+    return blocks;
   }
 
-  private static async readFile(path: string): Promise<string> {
-    return await fsExtra.readFile(path, "utf-8");
-  }
+  /**
+   * Async generator that yields code blocks one-by-one with streaming regex parsing.
+   */
+  static async *iterateBlocks(
+    markdownFilePath: string
+  ): AsyncGenerator<{ code: string; type: "tsx" | "ts" }> {
+    const pattern = this.TYPESCRIPT_CODE_PATTERN;
+    // eslint-disable-next-line functional/no-let
+    let buffer = "";
 
-  private static extractCodeBlocksFromMarkdown(
-    markdown: string
-  ): { code: string; type: "tsx" | "ts" }[] {
-    const codeBlocks: { code: string; type: "tsx" | "ts" }[] = [];
-    markdown.replace(this.TYPESCRIPT_CODE_PATTERN, (_, type, code) => {
-      codeBlocks.push({
-        code,
-        type: type === "tsx" ? "tsx" : "ts",
-      });
-      return code;
-    });
-    return codeBlocks;
+    const stream = fs.createReadStream(markdownFilePath, { encoding: "utf-8" });
+    for await (const chunk of stream) {
+      buffer += chunk;
+      // reset regex state
+      pattern.lastIndex = 0;
+      // eslint-disable-next-line functional/no-let
+      let match: RegExpExecArray | null;
+
+      // pull out all complete code blocks
+      while ((match = pattern.exec(buffer))) {
+        const tsxType = match[1];
+        const code = match[2];
+        const type = tsxType === "tsx" ? "tsx" : "ts";
+        yield { code, type };
+
+        // drop processed segment
+        buffer = buffer.slice(match.index + match[0].length);
+        pattern.lastIndex = 0;
+      }
+    }
   }
 }

src/SnippetCompiler.ts

@@ -25,7 +25,7 @@ export type SnippetCompilationResult = {
 
 export class SnippetCompiler {
   private readonly compilerConfig: TSNode.CreateOptions;
-
+  private readonly compiler: TSNode.Service;
   constructor(
     private readonly workingDirectory: string,
     private readonly packageDefinition: PackageDefinition,
@@ -39,6 +39,7 @@ export class SnippetCompiler {
       ...(configOptions.config as TSNode.CreateOptions),
       transpileOnly: false,
     };
+    this.compiler = TSNode.create(this.compilerConfig);
   }
 
   private static loadTypeScriptConfig(
@@ -61,16 +62,58 @@ export class SnippetCompiler {
     return rawString.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   }
 
+  private async *generateBlocks(
+    files: string[],
+    substituter: LocalImportSubstituter
+  ): AsyncGenerator<CodeBlock> {
+    for (const file of files) {
+      yield* this.extractFileCodeBlocks(file, substituter);
+    }
+  }
+
   async compileSnippets(
-    documentationFiles: string[]
+    documentationFiles: string[],
+    { concurrency = 4 } = {}
   ): Promise<SnippetCompilationResult[]> {
     try {
       await this.cleanWorkingDirectory();
       await fsExtra.ensureDir(this.workingDirectory);
-      const examples = await this.extractAllCodeBlocks(documentationFiles);
-      return await Promise.all(
-        examples.map(async (example) => await this.testCodeCompilation(example))
+
+      const results: SnippetCompilationResult[] = [];
+      const importSubstituter = new LocalImportSubstituter(
+        this.packageDefinition
+      );
+      const blockIterator = this.generateBlocks(
+        documentationFiles,
+        importSubstituter
       );
+
+      // eslint-disable-next-line functional/no-let
+      let hasFailed = false;
+
+      const worker = async () => {
+        while (!hasFailed) {
+          const { value: block, done } = await blockIterator.next();
+          if (done || !block) {
+            return;
+          }
+
+          try {
+            const result = await this.testCodeCompilation(block);
+            results.push(result);
+            if (result.error) {
+              hasFailed = true;
+            }
+          } catch (err) {
+            hasFailed = true;
+            throw err;
+          }
+        }
+      };
+
+      await Promise.all(Array.from({ length: concurrency }, () => worker()));
+
+      return results.sort((a, b) => a.index - b.index);
     } finally {
       await this.cleanWorkingDirectory();
     }
@@ -80,34 +123,21 @@ export class SnippetCompiler {
     return await fsExtra.remove(this.workingDirectory);
   }
 
-  private async extractAllCodeBlocks(documentationFiles: string[]) {
-    const importSubstituter = new LocalImportSubstituter(
-      this.packageDefinition
-    );
-
-    const codeBlocks = await Promise.all(
-      documentationFiles.map(
-        async (file) =>
-          await this.extractFileCodeBlocks(file, importSubstituter)
-      )
-    );
-    return codeBlocks.flat();
-  }
-
-  private async extractFileCodeBlocks(
+  private async *extractFileCodeBlocks(
     file: string,
     importSubstituter: LocalImportSubstituter
-  ): Promise<CodeBlock[]> {
-    const blocks = await CodeBlockExtractor.extract(file);
-    return blocks.map(({ code, type }, index) => {
-      return {
+  ): AsyncGenerator<CodeBlock> {
+    // eslint-disable-next-line functional/no-let
+    let index = 0;
+    for await (const { code, type } of CodeBlockExtractor.iterateBlocks(file)) {
+      yield {
         file,
         type,
         snippet: code,
-        index: index + 1,
+        index: ++index,
         sanitisedCode: this.sanitiseCodeBlock(importSubstituter, code),
       };
-    });
+    }
   }
 
   private sanitiseCodeBlock(
@@ -116,15 +146,26 @@ export class SnippetCompiler {
   ): string {
     const localisedBlock =
       importSubstituter.substituteLocalPackageImports(block);
-    return localisedBlock;
+
+    const moduleSyntaxRegex = /\b(import|export|declare\s+module|export\s*=)\b/;
+
+    const isModuleCode = moduleSyntaxRegex.test(localisedBlock);
+
+    // TODO: allow preventing of wrapping if the block is marked with <!-- ts-docs-verifier:no-wrap -->
+    if (isModuleCode) {
+      // keep block as is if recognized as module code (it won't be valid if wrapped in an IIFE)
+      return localisedBlock;
+    }
+
+    // otherwise wrap in function scope to isolate types, @see https://github.com/bbc/typescript-docs-verifier/issues/30
+    return `(function wrap() {\n${localisedBlock}\n})();`;
   }
 
   private async compile(code: string, type: "ts" | "tsx"): Promise<void> {
     const id = process.hrtime.bigint().toString();
     const codeFile = path.join(this.workingDirectory, `block-${id}.${type}`);
     await fsExtra.writeFile(codeFile, code);
-    const compiler = TSNode.create(this.compilerConfig);
-    compiler.compile(code, codeFile);
+    this.compiler.compile(code, codeFile);
   }
 
   private removeTemporaryFilePaths(
@@ -183,7 +224,13 @@ export class SnippetCompiler {
             [...example.sanitisedCode.substring(0, start)].filter(
               (char) => char === "\n"
             ).length + 1;
-          linesWithErrors.add(lineNumber);
+          const iifeOffset = example.sanitisedCode.startsWith(
+            "(function wrap() {"
+          )
+            ? 1
+            : 0;
+
+          linesWithErrors.add(lineNumber - iifeOffset);
         });
       }