From 3c3fc9c5ff86b6e5e9b852e4e36e3baa7fd81686 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Wed, 13 Nov 2024 12:44:08 +0100 Subject: [PATCH] Add Event Log section to the pipeline subtree (#1444) ## Changes Running state: Screenshot 2024-11-08 at 15 15 23 Completed state: Screenshot 2024-11-08 at 15 15 50 Currently event log items aren't interactive. Requires this JS SDK PR to be merged first: https://github.com/databricks/databricks-sdk-js/pull/272 ## Tests Manually --- .../src/bundle/run/PipelineRunStatus.ts | 48 ++++++-- .../PipelineRunEventsTreeNode.ts | 107 ++++++++++++++++++ .../PipelineRunStatusTreeNode.ts | 24 ++-- .../PipelineTreeNode.ts | 29 +++-- .../src/ui/bundle-resource-explorer/types.ts | 1 + 5 files changed, 174 insertions(+), 35 deletions(-) create mode 100644 packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts diff --git a/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts b/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts index 6beece33d..0d248487e 100644 --- a/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts +++ b/packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts @@ -2,7 +2,7 @@ import {BundleRunStatus} from "./BundleRunStatus"; import {AuthProvider} from "../../configuration/auth/AuthProvider"; import {onError} from "../../utils/onErrorDecorator"; -import {pipelines} from "@databricks/databricks-sdk"; +import {pipelines, WorkspaceClient} from "@databricks/databricks-sdk"; function isRunning(status?: pipelines.UpdateInfoState) { if (status === undefined) { @@ -12,9 +12,12 @@ function isRunning(status?: pipelines.UpdateInfoState) { } export class PipelineRunStatus extends BundleRunStatus { - readonly type = "pipelines"; + public readonly type = "pipelines"; + public data: pipelines.GetUpdateResponse | undefined; + public events: pipelines.PipelineEvent[] | undefined; + private interval?: NodeJS.Timeout; - data: pipelines.GetUpdateResponse | undefined; + constructor( private readonly authProvider: AuthProvider, private readonly pipelineId: string @@ -22,7 +25,6 @@ export class PipelineRunStatus extends BundleRunStatus { super(); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars parseId(output: string): void { if (this.runId !== undefined || this.runState !== "unknown") { return; @@ -57,19 +59,25 @@ export class PipelineRunStatus extends BundleRunStatus { if (this.runId === undefined) { throw new Error("No update id"); } - const update = await client.pipelines.getUpdate({ + this.data = await client.pipelines.getUpdate({ pipeline_id: this.pipelineId, update_id: this.runId, }); - this.data = update; + + if (this.data.update?.creation_time !== undefined) { + this.events = await this.fetchUpdateEvents( + client, + this.data.update.creation_time, + this.data.update?.update_id + ); + } // If update is completed, we stop polling. - if (!isRunning(update.update?.state)) { + if (!isRunning(this.data.update?.state)) { this.markCompleted(); - return; + } else { + this.onDidChangeEmitter.fire(); } - - this.onDidChangeEmitter.fire(); } catch (e) { this.runState = "error"; throw e; @@ -77,6 +85,26 @@ export class PipelineRunStatus extends BundleRunStatus { }, 5_000); } + private async fetchUpdateEvents( + client: WorkspaceClient, + creationTime: number, + updateId?: string + ) { + const events = []; + const timestamp = new Date(creationTime).toISOString(); + const listEvents = client.pipelines.listPipelineEvents({ + pipeline_id: this.pipelineId, + order_by: ["timestamp asc"], + filter: `timestamp >= '${timestamp}'`, + }); + for await (const event of listEvents) { + if (!updateId || event.origin?.update_id === updateId) { + events.push(event); + } + } + return events; + } + private markCompleted() { if (this.interval !== undefined) { clearInterval(this.interval); diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts new file mode 100644 index 000000000..9a9457287 --- /dev/null +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts @@ -0,0 +1,107 @@ +import { + BundleResourceExplorerTreeItem, + BundleResourceExplorerTreeNode, +} from "./types"; +import {ThemeColor, ThemeIcon, TreeItemCollapsibleState} from "vscode"; +import {ContextUtils} from "./utils"; +import {PipelineRunStatus} from "../../bundle/run/PipelineRunStatus"; +import {TreeItemTreeNode} from "../TreeItemTreeNode"; +import {ConnectionManager} from "../../configuration/ConnectionManager"; +import {EventLevel} from "@databricks/databricks-sdk/dist/apis/pipelines"; + +export class PipelineRunEventsTreeNode + implements BundleResourceExplorerTreeNode +{ + readonly type = "pipeline_run_events"; + + private get update() { + return this.runMonitor?.data?.update; + } + + private get events() { + return this.runMonitor?.events; + } + + public get url() { + const {host} = this.connectionManager.databricksWorkspace ?? {}; + // eslint-disable-next-line @typescript-eslint/naming-convention + const {pipeline_id, update_id} = this.update ?? {}; + if (!host || !pipeline_id || !update_id) { + return undefined; + } + return `${host}#joblist/pipelines/${pipeline_id}/updates/${update_id}`; + } + + constructor( + private readonly connectionManager: ConnectionManager, + private readonly runMonitor: PipelineRunStatus, + public parent?: BundleResourceExplorerTreeNode + ) {} + + getChildren(): BundleResourceExplorerTreeNode[] { + if (this.events === undefined || this.events.length === 0) { + return []; + } + const children: BundleResourceExplorerTreeNode[] = []; + + for (const event of this.events) { + children.push( + new TreeItemTreeNode( + { + label: event.message ?? event.event_type ?? "unknown", + iconPath: getEventIcon(event.level), + tooltip: event.message, + contextValue: "pipeline_event", + }, + this + ) + ); + } + + return children; + } + + getTreeItem(): BundleResourceExplorerTreeItem { + if (this.events === undefined || this.events.length === 0) { + return { + label: "Event Log", + iconPath: new ThemeIcon("loading~spin"), + contextValue: ContextUtils.getContextString({ + nodeType: this.type, + }), + collapsibleState: TreeItemCollapsibleState.None, + }; + } + + return { + label: "Event Log", + iconPath: new ThemeIcon("inbox"), + contextValue: ContextUtils.getContextString({ + nodeType: this.type, + hasUrl: this.url !== undefined, + }), + collapsibleState: TreeItemCollapsibleState.Expanded, + }; + } +} + +function getEventIcon(level: EventLevel | undefined): ThemeIcon { + switch (level) { + case "ERROR": + return new ThemeIcon( + "error", + new ThemeColor("list.errorForeground") + ); + case "INFO": + return new ThemeIcon("info"); + case "METRICS": + return new ThemeIcon("dashboard"); + case "WARN": + return new ThemeIcon( + "warning", + new ThemeColor("list.warningForeground") + ); + default: + return new ThemeIcon("question"); + } +} diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts index 4ae26c45e..099b58188 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts @@ -2,7 +2,7 @@ import { BundleResourceExplorerTreeItem, BundleResourceExplorerTreeNode, } from "./types"; -import {TreeItemCollapsibleState} from "vscode"; +import {ThemeIcon, TreeItemCollapsibleState} from "vscode"; import {ContextUtils, RunStateUtils} from "./utils"; import {SimplifiedRunState, sentenceCase} from "./utils/RunStateUtils"; import {GetUpdateResponse} from "@databricks/databricks-sdk/dist/apis/pipelines"; @@ -44,24 +44,19 @@ export class PipelineRunStatusTreeNode implements BundleResourceExplorerTreeNode { readonly type = "pipeline_run_status"; + private get update() { return this.runMonitor?.data?.update; } + public get url() { - if (this.type !== this.type) { - return undefined; - } - const host = this.connectionManager.databricksWorkspace?.host; - if ( - host === undefined || - this.update?.pipeline_id === undefined || - this.update?.update_id === undefined - ) { + const {host} = this.connectionManager.databricksWorkspace ?? {}; + // eslint-disable-next-line @typescript-eslint/naming-convention + const {pipeline_id, update_id} = this.update ?? {}; + if (!host || !pipeline_id || !update_id) { return undefined; } - return `${host.toString()}#joblist/pipelines/${ - this.update.pipeline_id - }/updates/${this.update.update_id}`; + return `${host}#joblist/pipelines/${pipeline_id}/updates/${update_id}`; } constructor( @@ -122,8 +117,7 @@ export class PipelineRunStatusTreeNode if (this.update === undefined) { return { label: "Run Status", - iconPath: RunStateUtils.getThemeIconForStatus("Unknown"), - description: "Run status not available", + iconPath: new ThemeIcon("loading~spin"), contextValue: ContextUtils.getContextString({ nodeType: this.type, }), diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineTreeNode.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineTreeNode.ts index ae755f5e6..c13a88380 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineTreeNode.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineTreeNode.ts @@ -12,6 +12,8 @@ import {ConnectionManager} from "../../configuration/ConnectionManager"; import {PipelineRunStatus} from "../../bundle/run/PipelineRunStatus"; import {TreeItemTreeNode} from "../TreeItemTreeNode"; import {PipelineRunStatusTreeNode} from "./PipelineRunStatusTreeNode"; +import {PipelineRunEventsTreeNode} from "./PipelineRunEventsTreeNode"; +import {ThemeIcon} from "vscode"; export class PipelineTreeNode implements BundleResourceExplorerTreeNode { readonly type = "pipelines"; @@ -67,22 +69,13 @@ export class PipelineTreeNode implements BundleResourceExplorerTreeNode { const runMonitor = this.bundleRunStatusManager.runStatuses.get( this.resourceKey ) as PipelineRunStatus | undefined; - if (runMonitor) { - children.push( - new PipelineRunStatusTreeNode( - this.connectionManager, - runMonitor, - this - ) - ); - } - if (this.data.catalog) { children.push( new TreeItemTreeNode( { label: "Catalog", description: this.data.catalog, + iconPath: new ThemeIcon("book"), contextValue: "catalog", }, this @@ -96,6 +89,7 @@ export class PipelineTreeNode implements BundleResourceExplorerTreeNode { { label: "Target", description: this.data.target, + iconPath: new ThemeIcon("target"), contextValue: "target", }, this @@ -103,6 +97,21 @@ export class PipelineTreeNode implements BundleResourceExplorerTreeNode { ); } + if (runMonitor) { + children.push( + new PipelineRunStatusTreeNode( + this.connectionManager, + runMonitor, + this + ), + new PipelineRunEventsTreeNode( + this.connectionManager, + runMonitor, + this + ) + ); + } + return children; } diff --git a/packages/databricks-vscode/src/ui/bundle-resource-explorer/types.ts b/packages/databricks-vscode/src/ui/bundle-resource-explorer/types.ts index a564dbf6c..0c0312af4 100644 --- a/packages/databricks-vscode/src/ui/bundle-resource-explorer/types.ts +++ b/packages/databricks-vscode/src/ui/bundle-resource-explorer/types.ts @@ -15,6 +15,7 @@ export interface BundleResourceExplorerTreeNode { | "treeItem" | "task_run_status" | "pipeline_run_status" + | "pipeline_run_events" | "resource_type_header" | "task" | "job_run_status"