Skip to content

Commit

Permalink
attempt to improve performance with web workers and lodash throttling
Browse files Browse the repository at this point in the history
  • Loading branch information
molti-tasking committed Nov 14, 2024
1 parent 43e31e5 commit e6ce2e6
Show file tree
Hide file tree
Showing 7 changed files with 205 additions and 34 deletions.
76 changes: 76 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
"babylonjs-hook": "^0.1.1",
"class-variance-authority": "^0.7.0",
"clsx": "^2.1.1",
"comlink": "^4.4.2",
"d3": "^7.9.0",
"lodash": "^4.17.21",
"lucide-react": "^0.452.0",
"react": "^18.3.1",
"react-dom": "18.3.1",
Expand All @@ -48,6 +50,7 @@
"@babylonjs/materials": "^7.30.0",
"@nabla/vite-plugin-eslint": "2.0.4",
"@types/d3": "^7.4.3",
"@types/lodash": "^4.17.13",
"@types/node": "^20.16.11",
"@types/react": "18.3.3",
"@types/react-dom": "18.3.0",
Expand Down
30 changes: 19 additions & 11 deletions src/components/MultiAggregatedLineChart.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { type ChartPresentationSettings, aggregatorB } from "@/lib/clusteringB";
import { type ChartPresentationSettings } from "@/lib/clusteringB";
import { cn, deepMerge } from "@/lib/utils";
import { useRawDataStore } from "@/store/useRawDataStore";
import { type ClassValue } from "clsx";
Expand All @@ -11,6 +11,8 @@ import {
TooltipTrigger,
} from "./ui/tooltip";
import { useViewSettingsStore } from "@/store/useViewSettingsStore";
import { useViewModelStore } from "@/store/useViewModelStore";
import { useEffect } from "react";

const chartModeSpecs: Record<
ChartPresentationSettings["mode"],
Expand Down Expand Up @@ -117,8 +119,8 @@ const chartModeSpecs: Record<
// TODO: Make this chart mostly recursive in a way that a user sees a subset whenever he selects one of those charts

export const MultiAggregatedLineChart = () => {
const dimensions = useRawDataStore((state) => state.dimensions);
const values = useRawDataStore((state) => state.values);
// const dimensions = useRawDataStore((state) => state.dimensions);

const colors: ClassValue[] = [
"bg-red-200",
Expand All @@ -136,16 +138,23 @@ export const MultiAggregatedLineChart = () => {
"bg-sky-200",
];
const presentationSettings = useViewSettingsStore();
console.time("Aggregator duration");

const { aggregated, colsAccordingToAggregation, yDomain } = aggregatorB(
values,
dimensions,
presentationSettings
);
console.timeEnd("Aggregator duration");
const { aggregated, colsAccordingToAggregation, yDomain, processData } =
useViewModelStore();

// console.time("Rendering process duration");
// const { aggregated, colsAccordingToAggregation, yDomain } = aggregatorB(
// values,
// dimensions,
// presentationSettings
// );
// console.timeEnd("Rendering process duration");

const boringDataCount = values.length - aggregated[0].length;
useEffect(() => {
processData();
}, [presentationSettings, values]);

const boringDataCount = values.length - aggregated?.[0]?.length;

return (
<div className="container w-full my-2 flex flex-col flex-wrap gap-2">
Expand All @@ -157,7 +166,6 @@ export const MultiAggregatedLineChart = () => {
<ClusterChartPreferencesPopover />
{/* <pre>{JSON.stringify(presentationSettings)}</pre> */}
</div>

<div className="flex flex-row flex-shrink items-center rounded-sm overflow-hidden">
{colsAccordingToAggregation.map(([name, styleGroup], index) => (
<TooltipProvider key={`${name}-${index}`}>
Expand Down
12 changes: 12 additions & 0 deletions src/lib/clustering.worker.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import * as Comlink from "comlink";
import { aggregatorB } from "./clusteringB";

// Define the functions that will be available in the worker
const workerFunctions = {
aggregatorB,
};

export type ClusteringWorker = typeof workerFunctions;

// Expose the worker functions to the main thread
Comlink.expose(workerFunctions);
44 changes: 23 additions & 21 deletions src/store/useRawDataStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,29 @@ interface DataStore {
) => void;
}

export const useRawDataStore = create<DataStore>((set, get) => ({
mode: "random",
dimensions: [],
values: [],
streamingInterval: null,
intervalId: null,

updateData: (mode, columnCount, rowCount, streamingInterval) => {
const { intervalId } = get();
if (intervalId) clearInterval(intervalId);
set({ mode, streamingInterval, values: [], dimensions: [] });
generateData(columnCount, rowCount);
if (streamingInterval) {
const newIntervalId = setInterval(() => {
console.log("Stream update");
streamDataUpdate();
}, streamingInterval);
set({ intervalId: newIntervalId });
}
},
}));
export const useRawDataStore = create<DataStore>((set, get) => {
return {
mode: "peaks",
dimensions: [],
values: [],
streamingInterval: null,
intervalId: null,

updateData: (mode, columnCount, rowCount, streamingInterval) => {
const { intervalId } = get();
if (intervalId) clearInterval(intervalId);
set({ mode, streamingInterval, values: [], dimensions: [] });
generateData(columnCount, rowCount);
if (streamingInterval) {
const newIntervalId = setInterval(() => {
console.log("Stream update");
streamDataUpdate();
}, streamingInterval);
set({ intervalId: newIntervalId });
}
},
};
});

// ----------------
// DATA GENERATION
Expand Down
55 changes: 55 additions & 0 deletions src/store/useViewModelStore.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { create } from "zustand";
import { useViewSettingsStore } from "./useViewSettingsStore";
import { useRawDataStore } from "./useRawDataStore";
import _ from "lodash";
import { ClusteringWorker } from "@/lib/clustering.worker";
import * as Comlink from "comlink";

import Worker from "../lib/clustering.worker?worker";

interface DataStore {
aggregated: Record<string, number>[][];
yDomain: [number, number];
colsAccordingToAggregation: [string, number][];

processData: () => void;
}

const workerInstance = new Worker({ name: "aggregator" });
const workerApi = Comlink.wrap<ClusteringWorker>(workerInstance);

export const useViewModelStore = create<DataStore>((set) => {
console.log("init view model store");

const throttledDataProcess = _.throttle(async () => {
console.count("Throttled process data");
const timerName = Date.now();

console.time("ViewModel process duration " + String(timerName));
const dimensions = useRawDataStore.getState().dimensions;
const values = useRawDataStore.getState().values;
const { updateSettings, ...presentationSettings } =
useViewSettingsStore.getState();
console.log("Settings: ", presentationSettings);

const aggregated = await workerApi.aggregatorB(
values,
dimensions,
presentationSettings
);
console.timeEnd("ViewModel process duration " + String(timerName));

set(aggregated);
}, 2000);

return {
aggregated: [],
yDomain: [0, 10],
colsAccordingToAggregation: [],

processData: async () => {
console.count("Process data with worker");
throttledDataProcess();
},
};
});
19 changes: 17 additions & 2 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -609,7 +609,7 @@

"@radix-ui/react-checkbox@^1.1.2":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@radix-ui/react-checkbox/-/react-checkbox-1.1.2.tgz#6465b800420923ecc39cbeaa8f357b5f09dbfd52"
resolved "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.1.2.tgz"
integrity sha512-/i0fl686zaJbDQLNKrkCbMyDm6FQMt4jg323k7HuqitoANm9sE23Ql8yOK3Wusk34HSLKDChhMux05FnP6KUkw==
dependencies:
"@radix-ui/primitive" "1.1.0"
Expand Down Expand Up @@ -791,7 +791,7 @@

"@radix-ui/react-switch@^1.1.1":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@radix-ui/react-switch/-/react-switch-1.1.1.tgz#1401658c24d66a18610f18793afbaa7fedf5429a"
resolved "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.1.1.tgz"
integrity sha512-diPqDDoBcZPSicYoMWdWx+bCPuTRH4QSp9J+65IvtdS0Kuzt67bI6n32vCj8q6NZmYW/ah+2orOtMwcX5eQwIg==
dependencies:
"@radix-ui/primitive" "1.1.0"
Expand Down Expand Up @@ -1331,6 +1331,11 @@
resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz"
integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==

"@types/lodash@^4.17.13":
version "4.17.13"
resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz"
integrity sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==

"@types/node@^20.16.11":
version "20.16.14"
resolved "https://registry.npmjs.org/@types/node/-/node-20.16.14.tgz"
Expand Down Expand Up @@ -1910,6 +1915,11 @@ color-name@~1.1.4:
resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz"
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==

comlink@^4.4.2:
version "4.4.2"
resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.4.2.tgz#cbbcd82742fbebc06489c28a183eedc5c60a2bca"
integrity sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==

commander@2:
version "2.20.3"
resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz"
Expand Down Expand Up @@ -3408,6 +3418,11 @@ lodash.merge@^4.6.2:
resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz"
integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==

lodash@^4.17.21:
version "4.17.21"
resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==

loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
version "1.4.0"
resolved "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz"
Expand Down

0 comments on commit e6ce2e6

Please sign in to comment.