Skip to content

Commit

Permalink
Do not load too many events (#1484)
Browse files Browse the repository at this point in the history
## Changes
While preloading updates in order to generate datasets and their
schemas,
it's possible to be stuck for a while in the loading (e.g. in the case
of continuous pipelines with a lot of events).


## Tests
<!-- How is this tested? -->
  • Loading branch information
ilia-db authored Dec 12, 2024
1 parent 8a19a41 commit 79ad92c
Showing 1 changed file with 7 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ type SourceLocation = {
notebook_cell_number?: number;
};

const MAX_EVENTS_TO_LOAD = 1000;

export class BundlePipelinesManager {
private disposables: Disposable[] = [];
private readonly triggeredState: Map<string, PipelineState> = new Map();
Expand Down Expand Up @@ -303,11 +305,16 @@ export class BundlePipelinesManager {
pipelineId,
runs
);
let loadedEventsCount = 0;
for await (const event of listing) {
const runState = runs.get(event.origin?.update_id ?? "");
if (runState?.events) {
runState.events.push(event);
}
loadedEventsCount++;
if (loadedEventsCount >= MAX_EVENTS_TO_LOAD) {
break;
}
}
const extractedData = extractPipelineDatasets(
new Set(runs.values())
Expand Down Expand Up @@ -358,8 +365,6 @@ export class BundlePipelinesManager {
if (oldestUpdateTime) {
const timestamp = new Date(oldestUpdateTime).toISOString();
listEventsOptions.filter = `timestamp >= '${timestamp}'`;
} else {
listEventsOptions.max_results = 100;
}
return client.pipelines.listPipelineEvents(listEventsOptions);
}
Expand Down

0 comments on commit 79ad92c

Please sign in to comment.