imiplement more view modes for performance optimization but so far it…

… is still not very fast

package.json

@@ -12,6 +12,9 @@
   "dependencies": {
     "@babylonjs/loaders": "^7.30.0",
     "@babylonjs/serializers": "^7.30.0",
+    "@deck.gl/core": "^9.0.35",
+    "@deck.gl/layers": "^9.0.35",
+    "@deck.gl/react": "^9.0.35",
     "@hookform/resolvers": "^3.9.0",
     "@jpmorganchase/anu": "^0.1.4",
     "@radix-ui/react-checkbox": "^1.1.2",
@@ -28,11 +31,14 @@
     "clsx": "^2.1.1",
     "comlink": "^4.4.2",
     "d3": "^7.9.0",
+    "deck.gl": "^9.0.35",
     "lodash": "^4.17.21",
     "lucide-react": "^0.452.0",
+    "plotly.js": "^2.35.2",
     "react": "^18.3.1",
     "react-dom": "18.3.1",
     "react-hook-form": "^7.53.1",
+    "react-plotly.js": "^2.6.0",
     "react-router-dom": "^6.27.0",
     "react-vega": "^7.6.0",
     "tailwind-merge": "^2.5.3",
@@ -52,8 +58,10 @@
     "@types/d3": "^7.4.3",
     "@types/lodash": "^4.17.13",
     "@types/node": "^20.16.11",
+    "@types/plotly.js": "^2.35.0",
     "@types/react": "18.3.3",
     "@types/react-dom": "18.3.0",
+    "@types/react-plotly.js": "^2.6.3",
     "@typescript-eslint/eslint-plugin": "6.13.1",
     "@typescript-eslint/parser": "6.13.1",
     "@vitejs/plugin-react-swc": "3.7.0",

src/components/ClusterChartPreferencesPopover.tsx

@@ -11,9 +11,9 @@ import { ChevronDownIcon } from "lucide-react";
 import { useState } from "react";
 import { Input } from "./ui/input";
 import { Label } from "./ui/label";
-import { ChartPresentationSettings } from "@/lib/clusteringB";
 import { Checkbox } from "./ui/checkbox";
 import { useViewSettingsStore } from "@/store/useViewSettingsStore";
+import { ChartPresentationSettings } from "@/lib/ChartPresentationSettings";
 
 export function ClusterChartPreferencesPopover() {
   const [open, setOpen] = useState(false);
@@ -34,6 +34,7 @@ export function ClusterChartPreferencesPopover() {
 const chartViewOptions: Record<ChartPresentationSettings["mode"], string> = {
   multiline: "Multiline",
   envelope: "Envelope",
+  plotly: "Plotly (GL)",
 };
 
 const ignoreBoringDataModeOptions: Record<

src/components/MultiAggregatedLineChart.tsx

@@ -1,8 +1,6 @@
-import { type ChartPresentationSettings } from "@/lib/clusteringB";
-import { cn, deepMerge } from "@/lib/utils";
+import { cn } from "@/lib/utils";
 import { useRawDataStore } from "@/store/useRawDataStore";
 import { type ClassValue } from "clsx";
-import { VegaLite, type VisualizationSpec } from "react-vega";
 import { ClusterChartPreferencesPopover } from "./ClusterChartPreferencesPopover";
 import {
   Tooltip,
@@ -12,134 +10,20 @@ import {
 } from "./ui/tooltip";
 import { useViewSettingsStore } from "@/store/useViewSettingsStore";
 import { useViewModelStore } from "@/store/useViewModelStore";
-import { useEffect } from "react";
-
-const chartModeSpecs: Record<
-  ChartPresentationSettings["mode"],
-  Partial<VisualizationSpec>
-> = {
-  multiline: {
-    encoding: {
-      color: {
-        field: "variable",
-        type: "nominal",
-        title: "Variables",
-      },
-    },
-  },
-  envelope: {
-    transform: [
-      {
-        calculate: "toNumber(datum.value)",
-        as: "value",
-      },
-      {
-        calculate: "toNumber(datum.timestamp)",
-        as: "timestamp",
-      },
-      {
-        aggregate: [
-          { op: "mean", field: "value", as: "mean_value" },
-          { op: "max", field: "value", as: "max_value" },
-          { op: "min", field: "value", as: "min_value" },
-          { op: "q1", field: "value", as: "q1_value" },
-          { op: "q3", field: "value", as: "q3_value" },
-        ],
-        groupby: ["timestamp"],
-      },
-      {
-        fold: ["mean_value", "max_value", "min_value", "q1_value", "q3_value"],
-        as: ["aggregation", "value"],
-      },
-    ],
-    layer: [
-      {
-        mark: {
-          type: "area",
-          color: "steelblue",
-          opacity: 0.3,
-        },
-        encoding: {
-          x: {
-            field: "timestamp",
-            type: "temporal",
-            title: "Time",
-          },
-          y: {
-            field: "q1_value",
-            type: "quantitative",
-            title: "Value",
-          },
-          y2: {
-            field: "q3_value",
-          },
-        },
-      },
-      {
-        // Mean line (solid)
-        mark: {
-          type: "line",
-          tooltip: true,
-        },
-        encoding: {
-          x: {
-            field: "timestamp",
-            type: "temporal",
-            title: "Time",
-          },
-          y: {
-            field: "value",
-            type: "quantitative",
-          },
-          color: {
-            field: "aggregation",
-            type: "nominal",
-            scale: {
-              domain: ["mean_value", "max_value", "min_value"],
-              range: ["steelblue", "steelblue", "steelblue"],
-            },
-          },
-          strokeDash: {
-            field: "aggregation",
-            type: "nominal",
-            scale: {
-              domain: ["max_value", "min_value"],
-              range: [
-                [4, 4],
-                [4, 4],
-              ],
-            },
-          },
-        },
-      },
-    ],
-  },
-};
+import { memo, useEffect } from "react";
+import { PlotlyChart } from "./charts/PlotlyChart";
+import { VegaLiteChart } from "./charts/VegaLiteChart";
+import { AlertCircleIcon } from "lucide-react";
 
 // TODO: Make this chart mostly recursive in a way that a user sees a subset whenever he selects one of those charts
 
 export const MultiAggregatedLineChart = () => {
   const values = useRawDataStore((state) => state.values);
   // const dimensions = useRawDataStore((state) => state.dimensions);
 
-  const colors: ClassValue[] = [
-    "bg-red-200",
-    "bg-green-200",
-    "bg-blue-200",
-    "bg-purple-200",
-    "bg-emerald-200",
-    "bg-orange-200",
-    "bg-teal-200",
-    "bg-rose-200",
-    "bg-lime-200",
-    "bg-cyan-200",
-    "bg-pink-200",
-    "bg-amber-200",
-    "bg-sky-200",
-  ];
   const presentationSettings = useViewSettingsStore();
 
-  const { aggregated, colsAccordingToAggregation, yDomain, processData } =
+  const { aggregated, colsAccordingToAggregation, processData } =
     useViewModelStore();
 
   // console.time("Rendering process duration");
@@ -154,14 +38,11 @@ export const MultiAggregatedLineChart = () => {
     processData();
   }, [presentationSettings, values]);
 
-  const boringDataCount = values.length - aggregated?.[0]?.length;
-
   return (
     <div className="container w-full my-2 flex flex-col flex-wrap gap-2">
       <div className="flex flex-row justify-between gap-4 items-center">
-        <div>
-          Ignored {boringDataCount} entries. Showing {aggregated.length}{" "}
-          clusters.
+        <div className="text-muted-foreground">
+          Detected {aggregated.length} clusters.
         </div>
         <ClusterChartPreferencesPopover />
         {/* <pre>{JSON.stringify(presentationSettings)}</pre> */}
@@ -184,69 +65,113 @@ export const MultiAggregatedLineChart = () => {
           </TooltipProvider>
         ))}
       </div>
+
       <div className="flex-1 flex flex-col gap-2 overflow-scroll">
-        {aggregated.map((val, index) => (
+        <Charts />
+        {/* <PlotlyChart /> */}
+        {/* {aggregated.map((val, index) => (
           <AggregatedLineChart
             values={val}
             key={index}
             className={colors[index % colors.length]}
             yDomain={yDomain}
             presentationSettings={presentationSettings}
           />
-        ))}
+        ))} */}
       </div>
     </div>
   );
 };
-const AggregatedLineChart = ({
-  values,
-  className,
-  yDomain,
-  presentationSettings,
-}: {
-  values: Record<string, number>[];
-  className: ClassValue;
-  yDomain: [number, number];
-  presentationSettings: ChartPresentationSettings;
-}) => {
-  const dimensions = values.length
-    ? Object.keys(values[0]).filter((e) => e !== "timestamp")
-    : [];
-  const dataSpec: Partial<VisualizationSpec> = {
-    $schema: "https://vega.github.io/schema/vega-lite/v5.json",
-    width: "container",
-    height: "container",
-    background: "transparent",
-    data: { values },
-    transform: [{ fold: dimensions, as: ["variable", "value"] }],
-    mark: "line",
-    encoding: {
-      x: {
-        field: "timestamp",
-        type:
-          "saveScreenSpace" in presentationSettings &&
-          !!presentationSettings.saveScreenSpace
-            ? "ordinal"
-            : "temporal",
-        title: "Time",
-      },
-      y: {
-        field: "value",
-        type: "quantitative",
-        title: "Value",
-        scale: { domain: yDomain },
-      },
-    },
-  };
 
-  // I don't 100% know why, but as of now it was very important to keep this order of the specs how they are getting passed into the merge function. Otherwise, the vizualisation breaks.
-  const spec = deepMerge(dataSpec, chartModeSpecs[presentationSettings.mode]);
+const colors: ClassValue[] = [
+  "bg-red-200",
+  "bg-green-200",
+  "bg-blue-200",
+  "bg-purple-200",
+  "bg-emerald-200",
+  "bg-orange-200",
+  "bg-teal-200",
+  "bg-rose-200",
+  "bg-lime-200",
+  "bg-cyan-200",
+  "bg-pink-200",
+  "bg-amber-200",
+  "bg-sky-200",
+];
+const Charts = memo(() => {
+  /**
+   * This is a list of entries that again are a list of multiple values representing a multiline chart. So this data should be used to render multiple Muli-Line-Charts
+   *
+   * It can be more than 30 charts that are getting updated every second.
+   */
+  const aggregated = useViewModelStore((state) => state.aggregated);
+  const yDomain = useViewModelStore((state) => state.yDomain);
+
+  const mode = useViewSettingsStore((state) => state.mode);
+  const saveScreenSpace = useViewSettingsStore(
+    (state) => "saveScreenSpace" in state && state.saveScreenSpace
+  );
+  if (mode === "plotly") {
+    // From plotly docs: https://plotly.com/python/webgl-vs-svg/
+
+    //// Context limits: browsers impose a strict limit on the number of WebGL "contexts" that any given web document can access.
+    //// WebGL-powered traces in plotly can use multiple contexts in some cases but as a general rule, it may not be possible to
+    //// render more than 8 WebGL-involving figures on the same page at the same time.
+
+    // So we are going to only render the first 8 ones :-(
+    const plotlyMaxNumberOfPlots = 8;
+
+    if (aggregated.length > plotlyMaxNumberOfPlots) {
+      return (
+        <>
+          {aggregated.slice(0, plotlyMaxNumberOfPlots).map((val, index) => (
+            <PlotlyChart
+              values={val}
+              key={index}
+              className={colors[index % colors.length]}
+              yDomain={yDomain}
+              mode={mode}
+              saveScreenSpace={saveScreenSpace}
+            />
+          ))}
+          <div className="bg-orange-400 p-4 mt-2 rounded-md flex flex-row items-center gap-4 text-white">
+            <AlertCircleIcon />
+            <span>
+              There are {aggregated.length - plotlyMaxNumberOfPlots} more
+              clusters, that are not shown.
+            </span>
+          </div>
+        </>
+      );
+    }
+    return (
+      <>
+        {aggregated.map((val, index) => (
+          <PlotlyChart
+            values={val}
+            key={index}
+            className={colors[index % colors.length]}
+            yDomain={yDomain}
+            mode={mode}
+            saveScreenSpace={saveScreenSpace}
+          />
+        ))}
+      </>
+    );
+  }
 
   return (
-    <VegaLite
-      spec={spec}
-      style={{ cursor: "pointer" }}
-      className={cn("flex-1", className, "rounded-sm overflow-hidden min-h-40")}
-    />
+    <>
+      {aggregated.map((val, index) => (
+        <VegaLiteChart
+          values={val}
+          key={index}
+          className={colors[index % colors.length]}
+          yDomain={yDomain}
+          mode={mode}
+          saveScreenSpace={saveScreenSpace}
+        />
+      ))}
+    </>
   );
-};
+});

src/components/charts/ChartProps.ts

@@ -0,0 +1,10 @@
+import { ChartPresentationSettings } from "@/lib/ChartPresentationSettings";
+import { ClassValue } from "clsx";
+
+export type ChartProps = {
+  values: Record<string, number>[];
+  className: ClassValue;
+  yDomain: [number, number];
+  saveScreenSpace?: boolean;
+  mode: ChartPresentationSettings["mode"];
+};