From a71a3c5d41e4c92394f713c33359bd483f0f9703 Mon Sep 17 00:00:00 2001 From: Kyle Sammons <1023070+kyle-sammons@users.noreply.github.com> Date: Tue, 16 Jan 2024 15:46:37 -0800 Subject: [PATCH] Add a custom logs panel (#46) * Rewrote the logs panel to be fully custom * Styling feedback and bug fix for light mode * Fixed a bug when no datasource exists, fixed a timezone regression * PR fixes --- .gitignore | 1 - package.json | 1 + .../components/FieldValueFrequency.tsx | 26 +- src/datasource/components/Logs/LogsCell.tsx | 325 ++++++++++++++++++ src/datasource/components/Logs/LogsTable.tsx | 94 +++++ src/datasource/components/Logs/LogsView.tsx | 54 +++ src/datasource/components/Logs/context.ts | 20 ++ src/datasource/components/Logs/styles.ts | 7 + src/datasource/components/Logs/types.ts | 12 + .../PPLFormatEditor/HelpMessage.tsx | 40 +-- src/datasource/types.ts | 2 + src/pages/explore.tsx | 294 +++++++++------- yarn.lock | 7 + 13 files changed, 729 insertions(+), 154 deletions(-) create mode 100644 src/datasource/components/Logs/LogsCell.tsx create mode 100644 src/datasource/components/Logs/LogsTable.tsx create mode 100644 src/datasource/components/Logs/LogsView.tsx create mode 100644 src/datasource/components/Logs/context.ts create mode 100644 src/datasource/components/Logs/styles.ts create mode 100644 src/datasource/components/Logs/types.ts diff --git a/.gitignore b/.gitignore index 4093b61..56ad4cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ # Logs -logs *.log npm-debug.log* yarn-debug.log* diff --git a/package.json b/package.json index 41fe930..912ecd2 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@types/react": "17.0.14", "@types/react-test-renderer": "16.9.2", "@types/react-virtualized-auto-sizer": "1.0.0", + "@types/react-window": "^1.8.8", "@types/semver": "7.3.7", "@typescript-eslint/eslint-plugin": "4.0.1", "@typescript-eslint/parser": "4.0.1", diff --git a/src/datasource/components/FieldValueFrequency.tsx b/src/datasource/components/FieldValueFrequency.tsx index 01bf488..518b988 100644 --- a/src/datasource/components/FieldValueFrequency.tsx +++ b/src/datasource/components/FieldValueFrequency.tsx @@ -51,17 +51,15 @@ function getValueCountsForField(unmappedFieldValuesArray: any): ValueFrequency[] const InnerTitle = (field: Field) => { return ( -
- -
- {field.name} -
-
-
+ +
+ {field.name} +
+
); }; @@ -72,7 +70,7 @@ const InnerContent = ( ) => { let mostCommonValues = getValueCountsForField(field.unmappedFieldValuesArray) return ( -
+ <> Top {mostCommonValues.length} {mostCommonValues.length > 1 ? 'values' : 'value'} @@ -117,7 +115,7 @@ const InnerContent = ( ); })} -
+ ); }; @@ -135,7 +133,7 @@ const InnerFooter = (field: Field) => { const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick }: Props) => { // This doesn't make sense for this field if (field.name === '_source') { - return
; + return <>; } return ( diff --git a/src/datasource/components/Logs/LogsCell.tsx b/src/datasource/components/Logs/LogsCell.tsx new file mode 100644 index 0000000..076bfea --- /dev/null +++ b/src/datasource/components/Logs/LogsCell.tsx @@ -0,0 +1,325 @@ +import React from 'react' +import { Log } from 'datasource/types' +import { LogColumnType, LogColumn } from 'datasource/components/Logs/types' +import getLogTableContext from 'datasource/components/Logs/context' +import { Button, useTheme2 } from '@grafana/ui' +import { dateTimeParse, getTimeZone } from '@grafana/data' +import { DARK_THEME_HIGHLIGHTED_BACKGROUND, DARK_THEME_OUTLINE, LIGHT_THEME_HIGHLIGHTED_BACKGROUND, LIGHT_THEME_OUTLINE } from './styles' + + +interface LogKeyValProps { + field: string, + val: any +} + + +const LogKeyVal = ({ field, val }: LogKeyValProps) => { + return (
+
+ {field + ":"} + +
+
+ {val} +
+
); +} + +interface ExpandedLogKeyValProps { + field: string, + val: any +} + + +const ExpandedLogKeyVal = ({ field, val }: ExpandedLogKeyValProps) => { + return ( + + + {field + ":"} + + + + {val} + + ); +} + +interface ExpandedDocumentProps { + log: Log, + index: number, + datasourceUid: string, + datasourceName: string, + datasourceField: string, +} + + +const ExpandedDocument = ({ log, index, datasourceUid, datasourceName, datasourceField }: ExpandedDocumentProps) => { + const { setSize, windowWidth } = getLogTableContext(); + const root = React.useRef(); + React.useEffect(() => { + if (root.current) { + setSize(index, root.current.getBoundingClientRect().height); + } + }, [windowWidth]); + + const link = { + datasource: datasourceUid, + queries: [{ + query: log.get(datasourceField), + refId: "A", + }], + range:{ + from: "now-15m", // TODO: This shouldn't be hardcoded + to: "now", // TODO: This also shouldn't be hardcoded + }, + }; + + const orgId = 1; // TODO: This shouldn't be hardcoded + const protocol = window.location.protocol.toString(); + const hostname = window.location.hostname.toString(); + const port = window.location.port.toString(); + + /* + * The format for the Grafana Explore UI (in a urlencoded form): + *'https:///explore?left={"datasource":"","queries":[{"query":"","refId":"A"}],"range":{"from":"now-15m"," to":"now"}}&orgId=' + */ + const formattedLink = `${protocol}//${hostname}${port ? ":" + port : ""}/explore?left=${encodeURIComponent(JSON.stringify(link))}&orgId=${orgId}` + + // TODO: We should add an icon here as well + return ( +
+ + Expanded Document + + + + + + + + { + Array.from(log.keys()).map((key) => ( + + )) + } +
+ Field + + Value +
+ { + log.has(datasourceField) && datasourceName && datasourceUid ? + ( + + + + ) + : '' + } + +
+ ); +} + + +const DocumentCell = (log: Log, style: any, rowIndex: number, expanded: boolean, datasourceUid: string, datasourceName: string, datasourceField: string) => ( +
+
+ { + Array.from(log.keys()).map((key) => ( + + )) + } +
+ { + expanded ? + () + + : '' + } +
+) + +const TimestampCell = (timestamp: number, style: any, rowIndex: number, expandedRows: boolean[], onClick: ((index: number) => void)) => { + const getFoldIcon = () => { + if (expandedRows[rowIndex]) { + return 'angle-down'; + } + return 'angle-right'; + }; + + return ( +
+
+
+ <> + {dateTimeParse(timestamp, {timeZone: getTimeZone()}).format("YYYY-MM-DD @ HH:mm:ss:SSS Z").toString()} + +
+ ); +} + +const FieldCell = () => { + return (<>); +} + +const HeaderCell = (column: LogColumn, style) => { + // TODO: Handle field types + // const log = data.logs[rowIndex]; + // const _timeField = data.timeField; + + // TODO: Implement sorting? + return ( +
+ {column.logColumnType === LogColumnType.TIME ? 'Time' : 'Document'} +
+ ) +} + +// Either expands or collapses the row, depending on the state its currently in +const invertRow = (expandedRows: boolean[], rowIndex: number): boolean[] => { + expandedRows[rowIndex] = !expandedRows[rowIndex] + return expandedRows +} + +// If the row isn't being expanded, then shrink it back to its original size. +// NOTE: Because of how we handle it down below, "shrink it to its original size" +// effectively means set it to 0. +const shrinkRows = (expandedRows: boolean[], rowIndex: number, setSize: (index: number, value: number) => void): boolean => { + if (!expandedRows[rowIndex]) { + setSize(rowIndex, 0); + return true; + } + return false; +} + + + +const LogCell = ({ columnIndex, rowIndex, style, data }) => { + const log = data.logs[rowIndex]; + const timestamp = data.timestamps[rowIndex]; + const column = data.columns[columnIndex]; + const setExpandedRowsAndReRender = data.setExpandedRowsAndReRender; + const expandedRows = data.expandedRows; + const datasourceUid: string = data.datasourceUid; + const datasourceName: string = data.datasourceName; + const datasourceField: string = data.datasourceField; + const { setSize } = getLogTableContext(); + const darkModeEnabled = useTheme2().isDark ; + + + // TODO: Ignoring for now as these will be used in a future pass + // const _timeField = data.timeField; + // const _setColumns = data.setColumns + + const handleOnClick = (rowIndex: number): any => { + const newExpandedRows = invertRow(expandedRows, rowIndex); + shrinkRows(newExpandedRows, rowIndex, setSize); + setExpandedRowsAndReRender([...newExpandedRows], rowIndex); + } + + const outline = darkModeEnabled ? DARK_THEME_OUTLINE : LIGHT_THEME_OUTLINE; + + // Handle drawing the borders for the entire row + // Only draw a borderon the left if we're on the left-most cell + if (columnIndex === 0) { + style['borderLeft'] = outline; + } + + // Only draw a border on the top if we're on the top-most cell + if (rowIndex === 0) { + style['borderTop'] = outline; + } + + // Only draw a border on the right if we're on the right-most cell + if (columnIndex === data.columns.length - 1) { + style['borderRight'] = outline; + } + + style['borderBottom'] = outline; + + + + // Header row + if (rowIndex === 0) { + return HeaderCell(column, style) + + } + + if (column.logColumnType === LogColumnType.TIME) { + return TimestampCell(timestamp, style, rowIndex, expandedRows, handleOnClick); + } else if (column.logColumnType === LogColumnType.DOCUMENT) { + return DocumentCell(log, style, rowIndex, expandedRows[rowIndex], datasourceUid, datasourceName, datasourceField); + } else { + return FieldCell(); + } +}; + +export default LogCell diff --git a/src/datasource/components/Logs/LogsTable.tsx b/src/datasource/components/Logs/LogsTable.tsx new file mode 100644 index 0000000..7b39e76 --- /dev/null +++ b/src/datasource/components/Logs/LogsTable.tsx @@ -0,0 +1,94 @@ +import React, { useLayoutEffect, useState } from 'react' +import { Log } from 'datasource/types' +import { VariableSizeGrid as Grid } from 'react-window' +import AutoSizer from 'react-virtualized-auto-sizer' +import { LogColumn, LogColumnType } from 'datasource/components/Logs/types' +import { LogTableContext } from 'datasource/components/Logs/context' +import LogCell from 'datasource/components/Logs/LogsCell' + +// Used for dynamically resizing our statically sized Grid when a resize +// operation happens +const useWindowSize = () => { + let [size, setSize] = useState([0, 0]); + useLayoutEffect(() => { + function updateSize() { + setSize([window.innerWidth, window.innerHeight]); + } + window.addEventListener("resize", updateSize); + updateSize(); + return () => window.removeEventListener("resize", updateSize); + }, []); + return size; +}; + + +interface LogsTableProps { + logs: Log[]; + timeField: string; + columns: LogColumn[]; + timestamps: number[]; + expandedRows: boolean[]; + datasourceUid: string; + datasourceName: string; + setExpandedRows: ((value: boolean[] | ((preVar: boolean[]) => boolean[])) => void); + setColumns: ((value: LogColumn[] | ((preVar: LogColumn[]) => LogColumn[])) => void); + datasourceField: string; +} + +const LogsTable = ({ logs, timeField, columns, timestamps, expandedRows, setColumns, setExpandedRows, datasourceUid, datasourceName, datasourceField }: LogsTableProps) => { + let gridRef: React.RefObject = React.createRef(); + + // In order to get highly variable (and unknown at the time of rendering) row heights in a virtualized environment + // where we need to know the _exact_ size of the rows ahead of time, we need to do just in time sizing. Essentially + // we can accomplish that by having this map that gets updated whenever we know the size of the thing we need to render. + // After we know that size, we can force a re-render, passing in the proper size into our Grid down below. Hopefully + // one day we don't need to do that, but that's entirely dependent on [this](https://github.com/bvaughn/react-window/issues/6) PR + // getting merged in. + const sizeMap = React.useRef({}); + const setSize = (index, size) => { + sizeMap.current = { ...sizeMap.current, [index]: size }; + if (gridRef.current) { + gridRef.current.resetAfterRowIndex(index); + } + } + + // * The header row should have a height of 25 + // * Any expanded row should have the height of whatever the height of the thing rendered is, plus 135 for padding for the already + // rendered row + // * Every other row should only have a height of 125 + const getSize = React.useCallback(index => index === 0 ? 25 : sizeMap.current[index] + 135 || 125, []); + const [windowWidth] = useWindowSize(); + + let setExpandedRowsAndReRender = (expandedRows: boolean[], rowIndex: number) => { + if (gridRef.current) { + gridRef.current.resetAfterRowIndex(rowIndex); + } + + setExpandedRows(expandedRows); + } + return ( + + + { + ({height, width}) => ( + columns[index].logColumnType === LogColumnType.TIME ? 300 : width-310} + height={height} + rowCount={logs.length} + rowHeight={getSize} + width={width} + itemData={{logs, timestamps, columns, timeField, setColumns, setExpandedRowsAndReRender, expandedRows, datasourceUid, datasourceName, datasourceField}} + > + {LogCell} + + + ) + } + + + ); +} + +export default LogsTable diff --git a/src/datasource/components/Logs/LogsView.tsx b/src/datasource/components/Logs/LogsView.tsx new file mode 100644 index 0000000..7357557 --- /dev/null +++ b/src/datasource/components/Logs/LogsView.tsx @@ -0,0 +1,54 @@ +import React from 'react' +import { Log } from 'datasource/types' +import { LogColumn, LogColumnType } from 'datasource/components/Logs/types' +import LogsTable from 'datasource/components/Logs/LogsTable' + + +interface LogsViewProps { + logs: Log[]; + timeField: string; + timestamps: number[]; + datasourceUid: string; + datasourceName: string; + datasourceField: string; +} + +const LogsView = ({ logs, timeField, timestamps, datasourceUid, datasourceName, datasourceField }: LogsViewProps) => { + const [columns, setColumns] = React.useState([ + { + logColumnType: LogColumnType.TIME, + underlyingFieldName: timeField + + }, + { + logColumnType: LogColumnType.DOCUMENT, + underlyingFieldName: null + }]); + + const [ expandedRows, setExpandedRows ] = React.useState(Array(logs.length).fill(false)); + + if (logs.length === 0) { + return ( + <> + ); + } + + + return ( + + ) + +}; + +export default LogsView diff --git a/src/datasource/components/Logs/context.ts b/src/datasource/components/Logs/context.ts new file mode 100644 index 0000000..a61fd92 --- /dev/null +++ b/src/datasource/components/Logs/context.ts @@ -0,0 +1,20 @@ +import { createContext, useContext } from 'react' + +// Used for updating the row size after we've figured out how large it is +type LoggingContext = { + setSize: (index: number, size: number) => void; + windowWidth: number +} +export const LogTableContext = createContext(null); + +const getLogTableContext = (): LoggingContext => { + // eslint-disable-next-line react-hooks/rules-of-hooks + const context = useContext(LogTableContext); + + if (!context) { + throw Error("Unable to get LogTableContext!"); + } + return context; +} + +export default getLogTableContext diff --git a/src/datasource/components/Logs/styles.ts b/src/datasource/components/Logs/styles.ts new file mode 100644 index 0000000..ff34949 --- /dev/null +++ b/src/datasource/components/Logs/styles.ts @@ -0,0 +1,7 @@ + +export const DARK_THEME_BACKGROUND = ''; +export const DARK_THEME_HIGHLIGHTED_BACKGROUND = '#343741' +export const LIGHT_THEME_BACKGROUND = '' +export const LIGHT_THEME_HIGHLIGHTED_BACKGROUND = '#E6F1FA' +export const LIGHT_THEME_OUTLINE = '1px solid rgba(36, 41, 46, 0.12)'; +export const DARK_THEME_OUTLINE = '1px solid rgba(204, 204, 220, 0.07)'; diff --git a/src/datasource/components/Logs/types.ts b/src/datasource/components/Logs/types.ts new file mode 100644 index 0000000..78cc619 --- /dev/null +++ b/src/datasource/components/Logs/types.ts @@ -0,0 +1,12 @@ + +export enum LogColumnType { + TIME, + DOCUMENT, + FIELD +}; + +export type LogColumn = { + logColumnType: LogColumnType, + underlyingFieldName?: string +}; + diff --git a/src/datasource/components/QueryEditor/PPLFormatEditor/HelpMessage.tsx b/src/datasource/components/QueryEditor/PPLFormatEditor/HelpMessage.tsx index bd686be..4d4d34f 100644 --- a/src/datasource/components/QueryEditor/PPLFormatEditor/HelpMessage.tsx +++ b/src/datasource/components/QueryEditor/PPLFormatEditor/HelpMessage.tsx @@ -2,26 +2,24 @@ import React from 'react'; export const HelpMessage = () => (
-
-
Table
-
    -
  • return any set of columns
  • -
-
-
Logs
-
    -
  • return any set of columns
  • -
-
-
Time series
-
    -
  • return column as date, datetime, or timestamp
  • -
  • return column with numeric datatype as values
  • -
-
- Example PPL query for time series: -
- source=<index> | eval dateValue=timestamp(timestamp) | stats count(response) by dateValue -
+
Table
+
    +
  • return any set of columns
  • +
+
+
Logs
+
    +
  • return any set of columns
  • +
+
+
Time series
+
    +
  • return column as date, datetime, or timestamp
  • +
  • return column with numeric datatype as values
  • +
+
+ Example PPL query for time series: +
+ source=<index> | eval dateValue=timestamp(timestamp) | stats count(response) by dateValue
); diff --git a/src/datasource/types.ts b/src/datasource/types.ts index 85091a9..8326cac 100644 --- a/src/datasource/types.ts +++ b/src/datasource/types.ts @@ -106,3 +106,5 @@ export interface Field { numberOfLogsFieldIsIn: number; unmappedFieldValuesArray: any[]; } + +export type Log = Map; diff --git a/src/pages/explore.tsx b/src/pages/explore.tsx index 43b0bb2..2029982 100644 --- a/src/pages/explore.tsx +++ b/src/pages/explore.tsx @@ -22,7 +22,7 @@ import { TextBoxVariable, VariableValueSelectors, } from '@grafana/scenes'; -import { AppRootProps, ArrayVector, DataFrame } from '@grafana/data'; +import { AppRootProps, DataFrame } from '@grafana/data'; import { Button, DrawStyle, @@ -39,9 +39,11 @@ import { import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; import { VariableHide } from '@grafana/schema'; -import { Field } from 'datasource/types'; +import { Field, Log } from 'datasource/types'; import FieldValueFrequency from '../datasource/components/FieldValueFrequency'; +import LogsView from 'datasource/components/Logs/LogsView'; import { FixedSizeList as List } from 'react-window' +import { DARK_THEME_HIGHLIGHTED_BACKGROUND, LIGHT_THEME_HIGHLIGHTED_BACKGROUND } from 'datasource/components/Logs/styles'; /** * The main explore component for KalDB, using the new Grafana scenes implementation. @@ -75,6 +77,14 @@ interface FieldStatsState extends SceneObjectState { loading: boolean; } +interface LogsState extends SceneObjectState { + logs: Log[]; + timestamps: number[]; + loading: boolean; + totalCount: number; + totalFailed: number; +} + const NodeStatsRenderer = ({ model }: SceneComponentProps) => { const { total, failed } = model.useState(); return ( @@ -282,7 +292,7 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => {
@@ -295,7 +305,7 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { Popular { }} > { }; } +const KalDBLogsRenderer = ({ model }: SceneComponentProps) => { + const { logs, loading, timestamps } = model.useState(); + + // TODO: This should be whatever the user set + const timeField = "_timesinceepoch" + const currentDataSource = dataSourceVariable + ['getDataSourceTypes']() // This is gross, but we need to access this private property and this is the only real typesafe way to do so in TypeScript + .filter((ele) => ele.name === dataSourceVariable.getValueText())[0]; + + let linkedDatasourceUid = ''; + let linkedDatasource = null; + let linkedDatasourceName = ''; + let linkedDatasourceField = ''; + + if (currentDataSource && currentDataSource.jsonData.dataLinks?.length > 0) { + linkedDatasourceUid = currentDataSource.jsonData.dataLinks[0].datasourceUid; + linkedDatasourceField = currentDataSource.jsonData.dataLinks[0].field; + linkedDatasource = dataSourceVariable + ['getDataSourceTypes']() // This is gross, but we need to access this private property and this is the only real typesafe way to do so in TypeScript + .filter((ele) => ele.uid === linkedDatasourceUid)[0]; + + // linkedDatasource will be undefined for external links + if (linkedDatasource) { + linkedDatasourceName = linkedDatasource.name; + } + } + + return ( + <> + {loading ? ( + + ) : ( +
+ +
+ )} + + ); +} + +class KalDBLogs extends SceneObjectBase { + static Component = KalDBLogsRenderer; + constructor(state?: Partial) { + super({ + logs: [], + loading: true, + totalCount: 0, + totalFailed: 0, + timestamps: [], + ...state, + }); + } + + setTimestamps = (timestamps: number[]) => { + this.setState({ + timestamps: timestamps, + }); + }; + + + setLogs = (logs: Log[]) => { + this.setState({ + logs: logs, + }); + }; + + setTotalCount = (totalCount: number) => { + this.setState({ + totalCount: totalCount, + }); + }; + + setTotalFailed = (totalFailed: number) => { + this.setState({ + totalFailed: totalFailed, + }); + }; + + setLoading = (loading: boolean) => { + this.setState({ + loading: loading, + }); + }; +} + + + class KaldbQuery extends SceneObjectBase { static Component = KaldbQueryRenderer; @@ -474,10 +578,10 @@ class KaldbQuery extends SceneObjectBase { } const histogramNodeStats = new NodeStats(); -const logsNodeStats = new NodeStats(); const resultsCounter = new ResultStats(); const queryComponent = new KaldbQuery(); const fieldComponent = new FieldStats(); +const logsComponent = new KalDBLogs(); const getExploreScene = () => { return new EmbeddedScene({ @@ -559,7 +663,7 @@ const getExploreScene = () => { new SceneFlexItem({ height: '100%', minHeight: 300, - body: logsPanel.build(), + body: logsComponent, }), ], }), @@ -572,101 +676,74 @@ const getExploreScene = () => { }); }; -const logsPanel = PanelBuilders.logs() - .setOption('showTime', true) - .setOption('wrapLogMessage', true) - .setHoverHeader(true) - .setHeaderActions( - new SceneFlexLayout({ - children: [logsNodeStats], - }) - ) - .setTitle('Logs'); - - /** - * This custom transform operation is used to rewrite the _source field to an ansi log line, as - * well as initialize the meta information used for debugging purposes. + * Parse the log data returned from KalDB and extract the relevant information + * to our various SceneObject's */ -const logsResultTransformation: CustomTransformOperator = () => (source: Observable) => { - return source.pipe( - map((data: DataFrame[]) => { - // Set log count - if (data.length > 0 && data[0].meta['shards']) { - logsNodeStats.setCount(data[0].meta['shards'].total, data[0].meta['shards'].failed); +const parseAndExtractLogData = (data: DataFrame[]) => { + // Set log count + if (data.length > 0 && data[0].meta['shards']) { + logsComponent.setTotalCount(data[0].meta['shards'].total); + logsComponent.setTotalFailed(data[0].meta['shards'].failed); + } + + // Set field names, the most popular fields, and calculates the frequency of the most common values + if (data.length > 0 && data[0].fields.length > 0) { + let fieldCounts: Map = new Map(); + + let mappedFields: Map = new Map(); + let reconstructedLogs: Log[] = []; + let timestamps: number[] = []; + + for (let unmappedField of data[0].fields) { + let unmappedFieldValuesArray = unmappedField.values.toArray(); + let logsWithDefinedValue = unmappedFieldValuesArray.filter((value) => value !== undefined).length; + + // TODO: This should be user configurable + if (unmappedField.name === "_timesinceepoch") { + timestamps = [ ...unmappedField.values.toArray() ]; + } + if (unmappedField.name === "_source") { + reconstructedLogs = unmappedField.values.toArray().map( + (value: object) => ( + new Map(Object.entries(value)) + )); } - // Set field names, the most popular fields, and calculates the frequency of the most common values - if (data.length > 0 && data[0].fields.length > 0) { - let fieldCounts: Map = new Map(); - - let mappedFields: Map = new Map(); - data[0].fields.map((unmappedField) => { - let unmappedFieldValuesArray = unmappedField.values.toArray(); - let logsWithDefinedValue = unmappedFieldValuesArray.filter((value) => value !== undefined).length; - - let mapped_field: Field = { - name: unmappedField.name, - type: unmappedField.type.toString(), - numberOfLogsFieldIsIn: logsWithDefinedValue, - unmappedFieldValuesArray: unmappedFieldValuesArray - }; - - fieldCounts.set(unmappedField.name, logsWithDefinedValue); - mappedFields.set(unmappedField.name, mapped_field); - }); - - let sortedFieldCounts: Map = new Map([...fieldCounts].sort((a, b) => (a[1] >= b[1] ? -1 : 0))); - let topTenMostPopularFields: Field[] = []; - let i = 0; - for (let [name, _count] of sortedFieldCounts) { - if (i === 10) { - break; - } - - // Add the name to the top ten field and remove it from the mapped fields, which is used to display all the - // others. This way we don't double-display - topTenMostPopularFields.push(mappedFields.get(name)); - mappedFields.delete(name); - i++; - } + let mapped_field: Field = { + name: unmappedField.name, + type: unmappedField.type.toString(), + numberOfLogsFieldIsIn: logsWithDefinedValue, + unmappedFieldValuesArray: unmappedFieldValuesArray + }; + + fieldCounts.set(unmappedField.name, logsWithDefinedValue); + mappedFields.set(unmappedField.name, mapped_field); + } + + logsComponent.setLogs(reconstructedLogs); + logsComponent.setTimestamps(timestamps); - fieldComponent.setFields([...mappedFields.values()]); - fieldComponent.setTopTenMostPopularFields(topTenMostPopularFields); + let sortedFieldCounts: Map = new Map([...fieldCounts].sort((a, b) => (a[1] >= b[1] ? -1 : 0))); + let topTenMostPopularFields: Field[] = []; + let i = 0; + for (let [name, _count] of sortedFieldCounts) { + if (i === 10) { + break; } - return data.map((frame: DataFrame) => { - return { - ...frame, - fields: frame.fields.map((field) => { - // todo - this should use the config value "message field name" - if (field.name === '_source') { - return { - ...field, - values: new ArrayVector( - field.values.toArray().map((v) => { - let str = ''; - for (const [key, value] of Object.entries(v)) { - // we specifically choose style code "2" here (dim) because it is the only style - // that has custom logic specific to Grafana to allow it to look good in dark and light themes - // https://github.com/grafana/grafana/blob/701c6b6f074d4bc515f0824ed4de1997db035b69/public/app/features/logs/components/LogMessageAnsi.tsx#L19-L24 - str = str + key + ': ' + '\u001b[2m' + value + '\u001b[0m' + ' '; - } - return str; - }) - ), - }; - } - return { - ...field, - keys: ['Line'], - }; - }), - }; - }); - }) - ); -}; + // Add the name to the top ten field and remove it from the mapped fields, which is used to display all the + // others. This way we don't double-display + topTenMostPopularFields.push(mappedFields.get(name)); + mappedFields.delete(name); + i++; + } + + fieldComponent.setFields([...mappedFields.values()]); + fieldComponent.setTopTenMostPopularFields(topTenMostPopularFields); + } + return data; +} const queryRunner = new SceneQueryRunner({ datasource: { @@ -706,6 +783,8 @@ queryRunner.subscribeToEvent(SceneObjectStateChangedEvent, (event) => { if (event.payload.newState['data'].state === 'Done') { queryComponent.setLoading(false); fieldComponent.setLoading(false); + logsComponent.setLoading(false); + parseAndExtractLogData(event.payload.newState['data'].series); } else if (event.payload.newState['data'].state === 'Loading') { resultsCounter.setResults(-1); queryComponent.setLoading(true); @@ -715,35 +794,14 @@ queryRunner.subscribeToEvent(SceneObjectStateChangedEvent, (event) => { fieldComponent.setFields([]); fieldComponent.setTopTenMostPopularFields([]) fieldComponent.setLoading(false); - logsNodeStats.setCount(-1, -1); + logsComponent.setTotalCount(-1); + logsComponent.setTotalFailed(-1); resultsCounter.setResults(-1); histogramNodeStats.setCount(-1, -1); } } }); -logsPanel.setData( - new SceneDataTransformer({ - $data: queryRunner, - transformations: [ - logsResultTransformation, - { - id: 'organize', - options: { - excludeByName: {}, - indexByName: { - // todo - this should use the config value for timestamp - _timesinceepoch: 0, - // todo - this should use the config value "message field name" - _source: 1, - }, - renameByName: {}, - }, - }, - ], - }) -); - const histogramPanel = PanelBuilders.timeseries() .setCustomFieldConfig('drawStyle', DrawStyle.Bars) .setCustomFieldConfig('fillOpacity', 100) diff --git a/yarn.lock b/yarn.lock index 016a46a..3c6c1bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2824,6 +2824,13 @@ dependencies: "@types/react" "*" +"@types/react-window@^1.8.8": + version "1.8.8" + resolved "https://registry.yarnpkg.com/@types/react-window/-/react-window-1.8.8.tgz#c20645414d142364fbe735818e1c1e0a145696e3" + integrity sha512-8Ls660bHR1AUA2kuRvVG9D/4XpRC6wjAaPT9dil7Ckc76eP9TKWZwwmgfq8Q1LANX3QNDnoU4Zp48A3w+zK69Q== + dependencies: + "@types/react" "*" + "@types/react@*", "@types/react@17.0.14", "@types/react@^17": version "17.0.14" resolved "https://registry.npmjs.org/@types/react/-/react-17.0.14.tgz"