From 1b1b847d01f2ff41392415fbcfbf1a400214abfc Mon Sep 17 00:00:00 2001 From: Kyle Sammons <1023070+kyle-sammons@users.noreply.github.com> Date: Fri, 1 Dec 2023 11:05:55 -0800 Subject: [PATCH] Improve performance of fields list rendering and application as a whole (#42) Optimized the fields list rendering and the app as a whole --- .config/webpack/webpack.config.ts | 14 ++ docker-compose.yaml | 1 + package.json | 4 +- .../components/FieldValueFrequency.tsx | 54 +++++- src/datasource/components/Toggletip.tsx | 8 +- src/datasource/types.ts | 3 +- src/pages/explore.tsx | 170 +++++++----------- yarn.lock | 8 + 8 files changed, 140 insertions(+), 122 deletions(-) diff --git a/.config/webpack/webpack.config.ts b/.config/webpack/webpack.config.ts index 22cb86c..d0743c4 100644 --- a/.config/webpack/webpack.config.ts +++ b/.config/webpack/webpack.config.ts @@ -13,10 +13,13 @@ import path from 'path'; import ReplaceInFileWebpackPlugin from 'replace-in-file-webpack-plugin'; import { Configuration } from 'webpack'; + import { getPackageJson, getPluginJson, hasReadme, getEntries } from './utils'; import { SOURCE_DIR, DIST_DIR } from './constants'; const pluginJson = getPluginJson(); +const TerserPlugin = require("terser-webpack-plugin"); + const config = async (env): Promise => ({ cache: { @@ -196,6 +199,17 @@ const config = async (env): Promise => ({ modules: [path.resolve(process.cwd(), 'src'), 'node_modules'], unsafeCache: true, }, + + optimization: { + minimize: true, + usedExports: true, + minimizer: [new TerserPlugin({ + terserOptions: { + sourceMap: true, + compress: true + } + })] + } }); export default config; diff --git a/docker-compose.yaml b/docker-compose.yaml index 3639d25..3841968 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,3 +20,4 @@ services: GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" GF_PATHS_PLUGINS: "/var/lib/grafana/plugins" GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "slack-kaldb-app,slack-kaldb-app-backend-datasource" + GF_SERVER_ENABLE_GZIP: "true" diff --git a/package.json b/package.json index cd727e8..10a874b 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "react-dom": "17.0.2", "react-use": "17.4.0", "react-virtualized-auto-sizer": "1.0.2", + "react-window": "^1.8.10", "rxjs": "7.8.0", "semver": "7.3.7", "tslib": "2.5.3", @@ -100,5 +101,6 @@ "engines": { "node": ">=16" }, - "packageManager": "yarn@1.22.21" + "packageManager": "yarn@1.22.21", + "sideEffects": false } diff --git a/src/datasource/components/FieldValueFrequency.tsx b/src/datasource/components/FieldValueFrequency.tsx index 27caaa7..01bf488 100644 --- a/src/datasource/components/FieldValueFrequency.tsx +++ b/src/datasource/components/FieldValueFrequency.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Field } from 'datasource/types'; +import { Field, ValueFrequency } from 'datasource/types'; import { Toggletip } from './Toggletip'; import { HorizontalGroup, VerticalGroup, Button } from '@grafana/ui'; @@ -10,6 +10,45 @@ interface Props { onMinusClick?: (field: Field, value: string) => void; } + +/* + * Calculates the frequency map for a list of values. + * The map returned is in sorted descending order + */ +function getFrequencyMap(values: T[]): Map { + let frequencyMap = new Map(); + for (let value of values) { + if (value === undefined) { + continue; + } + + let stringValue = JSON.stringify(value); + + let currentCount = frequencyMap.has(stringValue) ? frequencyMap.get(stringValue) : 0; + frequencyMap.set(stringValue, currentCount + 1); + } + return new Map([...frequencyMap].sort((a, b) => (a[1] >= b[1] ? -1 : 0))); +} + +function getValueCountsForField(unmappedFieldValuesArray: any): ValueFrequency[] { + let frequencyMapForField = getFrequencyMap(unmappedFieldValuesArray); + let topFiveMostPopularValues: ValueFrequency[] = []; + let i = 0; + for (let [value, _count] of frequencyMapForField) { + if (i === 5) { + break; + } + let definedCount = unmappedFieldValuesArray.filter((value) => value !== undefined).length; + let valueFreq: ValueFrequency = { + value: value, + frequency: _count / definedCount, + }; + topFiveMostPopularValues.push(valueFreq); + i++; + } + return topFiveMostPopularValues; +} + const InnerTitle = (field: Field) => { return (
@@ -31,14 +70,15 @@ const InnerContent = ( onPlusClick?: (field: Field, value: string) => void, onMinusClick?: (field: Field, value: string) => void ) => { + let mostCommonValues = getValueCountsForField(field.unmappedFieldValuesArray) return (
- Top {field.mostCommonValues.length} {field.mostCommonValues.length > 1 ? 'values' : 'value'} + Top {mostCommonValues.length} {mostCommonValues.length > 1 ? 'values' : 'value'} - {field.mostCommonValues.map((valueFreq) => { + {mostCommonValues.map((valueFreq) => { return ( @@ -84,7 +124,7 @@ const InnerContent = ( const InnerFooter = (field: Field) => { return ( - Exists in {field.numberOfLogsFieldIsIn} / {field.totalNumberOfLogs} records + Exists in {field.numberOfLogsFieldIsIn} / {field.unmappedFieldValuesArray.values.length} records ); }; @@ -92,7 +132,7 @@ const InnerFooter = (field: Field) => { /** * A component to show the FieldValueFrequency for a given field value in the app UI. */ -export const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick }: Props) => { +const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick }: Props) => { // This doesn't make sense for this field if (field.name === '_source') { return
; @@ -101,7 +141,7 @@ export const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick return ( InnerContent(field, onPlusClick, onMinusClick)} footer={InnerFooter(field)} closeButton={false} placement={'right'} @@ -110,3 +150,5 @@ export const FieldValueFrequency = ({ field, children, onMinusClick, onPlusClick ); }; + +export default FieldValueFrequency; diff --git a/src/datasource/components/Toggletip.tsx b/src/datasource/components/Toggletip.tsx index b1b9e17..ad8bc47 100644 --- a/src/datasource/components/Toggletip.tsx +++ b/src/datasource/components/Toggletip.tsx @@ -183,7 +183,7 @@ export interface ToggletipContentProps { update?: () => void; } -export type ToggletipContent = string | React.ReactElement | ((props: ToggletipContentProps) => JSX.Element); +export type ToggletipContent = string | React.ReactElement | (() => React.ReactElement); export interface ToggletipProps { /** The theme used to display the toggletip */ @@ -243,7 +243,7 @@ export const Toggletip = React.memo( return undefined; }, [controlledVisible, closeToggletip]); - const { getArrowProps, getTooltipProps, setTooltipRef, setTriggerRef, visible, update } = usePopperTooltip({ + const { getArrowProps, getTooltipProps, setTooltipRef, setTriggerRef, visible } = usePopperTooltip({ visible: controlledVisible, placement: placement, interactive: true, @@ -257,6 +257,7 @@ export const Toggletip = React.memo( }, }); + return ( <> {React.cloneElement(children, { @@ -283,8 +284,7 @@ export const Toggletip = React.memo( )}
- {(typeof content === 'string' || React.isValidElement(content)) && content} - {typeof content === 'function' && update && content({ update })} + {(typeof content === 'function' && React.isValidElement(content())) && content()}
{Boolean(footer) &&
{footer}
}
diff --git a/src/datasource/types.ts b/src/datasource/types.ts index ef0eed4..85091a9 100644 --- a/src/datasource/types.ts +++ b/src/datasource/types.ts @@ -103,7 +103,6 @@ export interface ValueFrequency { export interface Field { name: string; type: string; - mostCommonValues: ValueFrequency[]; numberOfLogsFieldIsIn: number; - totalNumberOfLogs: number; + unmappedFieldValuesArray: any[]; } diff --git a/src/pages/explore.tsx b/src/pages/explore.tsx index 8179359..43b0bb2 100644 --- a/src/pages/explore.tsx +++ b/src/pages/explore.tsx @@ -39,8 +39,9 @@ import { import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; import { VariableHide } from '@grafana/schema'; -import { Field, ValueFrequency } from 'datasource/types'; -import { FieldValueFrequency } from '../datasource/components/FieldValueFrequency'; +import { Field } from 'datasource/types'; +import FieldValueFrequency from '../datasource/components/FieldValueFrequency'; +import { FixedSizeList as List } from 'react-window' /** * The main explore component for KalDB, using the new Grafana scenes implementation. @@ -246,12 +247,43 @@ const KalDBFieldsList = (fields: Field[], topTenMostPopularFields: Field[]) => { return 'Unknown field'; }; + const ListItem = ({ index, data }) => { + const field = data[index]; + + return (
+ queryComponent.appendToQuery(`${field.name}: ${value}`)} + onMinusClick={(field: Field, value: string) => + queryComponent.appendToQuery(`NOT ${field.name}: ${value}`) + } + > +
+ + + + {field.name} + + +
+ +
); + } + return (
{ > Popular -
    - {topTenMostPopularFields.map((field) => ( -
    -
  • - queryComponent.appendToQuery(`${field.name}: ${value}`)} - onMinusClick={(field: Field, value: string) => - queryComponent.appendToQuery(`NOT ${field.name}: ${value}`) - } - > -
    - - - - {field.name} - - -
    -     -
    • -
    - ))} -
+ {ListItem} +
-
    - {fields.map((field) => ( -
    -
  • - queryComponent.appendToQuery(`${field.name}: ${value}`)} - onMinusClick={(field: Field, value: string) => - queryComponent.appendToQuery(`NOT ${field.name}: ${value}`) - } - > -
    - - - - {field.name} - - -
    -     -
    • -
    - ))} -
+ + {ListItem} + +
); }; @@ -596,24 +583,6 @@ const logsPanel = PanelBuilders.logs() ) .setTitle('Logs'); -/* - * Calculates the frequency map for a list of values. - * The map returned is in sorted descending order - */ -function getFrequencyMap(values: T[]): Map { - let frequencyMap = new Map(); - for (let value of values) { - if (value === undefined) { - continue; - } - - let stringValue = JSON.stringify(value); - - let currentCount = frequencyMap.has(stringValue) ? frequencyMap.get(stringValue) : 0; - frequencyMap.set(stringValue, currentCount + 1); - } - return new Map([...frequencyMap].sort((a, b) => (a[1] >= b[1] ? -1 : 0))); -} /** * This custom transform operation is used to rewrite the _source field to an ansi log line, as @@ -634,30 +603,13 @@ const logsResultTransformation: CustomTransformOperator = () => (source: Observa let mappedFields: Map = new Map(); data[0].fields.map((unmappedField) => { let unmappedFieldValuesArray = unmappedField.values.toArray(); - let frequencyMapForField = getFrequencyMap(unmappedFieldValuesArray); - let topFiveMostPopularValues: ValueFrequency[] = []; - let i = 0; - for (let [value, _count] of frequencyMapForField) { - if (i === 5) { - break; - } - let definedCount = unmappedFieldValuesArray.filter((value) => value !== undefined).length; - let valueFreq: ValueFrequency = { - value: value, - frequency: _count / definedCount, - }; - topFiveMostPopularValues.push(valueFreq); - i++; - } - let logsWithDefinedValue = unmappedFieldValuesArray.filter((value) => value !== undefined).length; let mapped_field: Field = { name: unmappedField.name, type: unmappedField.type.toString(), - mostCommonValues: topFiveMostPopularValues, numberOfLogsFieldIsIn: logsWithDefinedValue, - totalNumberOfLogs: unmappedField.values.length, + unmappedFieldValuesArray: unmappedFieldValuesArray }; fieldCounts.set(unmappedField.name, logsWithDefinedValue); diff --git a/yarn.lock b/yarn.lock index 14e9393..016a46a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8879,6 +8879,14 @@ react-window@1.8.8: "@babel/runtime" "^7.0.0" memoize-one ">=3.1.1 <6" +react-window@^1.8.10: + version "1.8.10" + resolved "https://registry.yarnpkg.com/react-window/-/react-window-1.8.10.tgz#9e6b08548316814b443f7002b1cf8fd3a1bdde03" + integrity sha512-Y0Cx+dnU6NLa5/EvoHukUD0BklJ8qITCtVEPY1C/nL8wwoZ0b5aEw8Ff1dOVHw7fCzMt55XfJDd8S8W8LCaUCg== + dependencies: + "@babel/runtime" "^7.0.0" + memoize-one ">=3.1.1 <6" + react@17.0.2: version "17.0.2" resolved "https://registry.npmjs.org/react/-/react-17.0.2.tgz"