Explain why we need to stream and why not use readline

github · Nov 19, 2024 · bb1da9c · bb1da9c
1 parent 2cde3b9
commit bb1da9c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/extensions/ql-vscode/src/common/jsonl-reader.ts b/extensions/ql-vscode/src/common/jsonl-reader.ts
@@ -15,6 +15,8 @@ export async function readJsonlFile<T>(
   handler: (value: T) => Promise<void>,
   logger?: { log: (message: string) => void },
 ): Promise<void> {
+  // Stream the data as large evaluator logs won't fit in memory.
+  // Also avoid using 'readline' as it is slower than our manual line splitting.
   void logger?.log(
     `Parsing ${path} (${(await stat(path)).size / 1024 / 1024} MB)...`,
   );