diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
index 551bb638c3..911b9cd923 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx
@@ -19,7 +19,9 @@ import {
   Box,
   Button,
   Card,
+  Checkbox,
   CircularProgress,
+  FormControlLabel,
   IconButton,
   MenuItem,
   Select,
@@ -44,6 +46,7 @@ import { formatValue, getNumericDifference, useTriggerEvent } from "./utils";
 
 const KEY_COLOR = "#ff6d04";
 const COMPARE_KEY_COLOR = "#03a9f4";
+const DEFAULT_BAR_CONFIG = { sortBy: "az" };
 
 export default function Evaluation(props: EvaluationProps) {
   const {
@@ -64,15 +67,14 @@ export default function Evaluation(props: EvaluationProps) {
   const [expanded, setExpanded] = React.useState("summary");
   const [mode, setMode] = useState("chart");
   const [editNoteState, setEditNoteState] = useState({ open: false, note: "" });
-  const [barConfigState, setBarConfigState] = useState({
-    sortBy: "",
-    limit: 0,
-  });
-  const [barConfigDialogState, setBarConfigDialogState] = useState({
-    open: false,
-    sortBy: "best",
-    limit: 20,
-  });
+  const [classPerformanceConfig, setClassPerformanceConfig] =
+    useState<PLOT_CONFIG_TYPE>({});
+  const [classPerformanceDialogConfig, setClassPerformanceDialogConfig] =
+    useState<PLOT_CONFIG_DIALOG_TYPE>(DEFAULT_BAR_CONFIG);
+  const [confusionMatrixConfig, setConfusionMatrixConfig] =
+    useState<PLOT_CONFIG_TYPE>({ log: true });
+  const [confusionMatrixDialogConfig, setConfusionMatrixDialogConfig] =
+    useState<PLOT_CONFIG_DIALOG_TYPE>(DEFAULT_BAR_CONFIG);
   const [metricMode, setMetricMode] = useState("chart");
   const [classMode, setClassMode] = useState("chart");
   const [performanceClass, setPerformanceClass] = useState("precision");
@@ -87,11 +89,14 @@ export default function Evaluation(props: EvaluationProps) {
     return evaluation;
   }, [data]);
   const confusionMatrix = useMemo(() => {
-    return evaluation?.confusion_matrix;
-  }, [evaluation]);
+    return getMatrix(evaluation?.confusion_matrices, confusionMatrixConfig);
+  }, [evaluation, confusionMatrixConfig]);
   const compareConfusionMatrix = useMemo(() => {
-    return compareEvaluation?.confusion_matrix;
-  }, [compareEvaluation]);
+    return getMatrix(
+      compareEvaluation?.confusion_matrices,
+      confusionMatrixConfig
+    );
+  }, [compareEvaluation, confusionMatrixConfig]);
   const compareKeys = useMemo(() => {
     const keys: string[] = [];
     const evaluations = data?.evaluations || [];
@@ -129,8 +134,11 @@ export default function Evaluation(props: EvaluationProps) {
   const closeNoteDialog = () => {
     setEditNoteState((note) => ({ ...note, open: false }));
   };
-  const closeBarConfigDialog = () => {
-    setBarConfigDialogState((state) => ({ ...state, open: false }));
+  const closeClassPerformanceConfigDialog = () => {
+    setClassPerformanceDialogConfig((state) => ({ ...state, open: false }));
+  };
+  const closeConfusionMatrixConfigDialog = () => {
+    setConfusionMatrixDialogConfig((state) => ({ ...state, open: false }));
   };
 
   if (!evaluation) {
@@ -400,7 +408,7 @@ export default function Evaluation(props: EvaluationProps) {
   const performanceClasses = Object.keys(perClassPerformance);
   const classPerformance = formatPerClassPerformance(
     perClassPerformance[performanceClass],
-    barConfigState
+    classPerformanceConfig
   );
 
   return (
@@ -608,7 +616,13 @@ export default function Evaluation(props: EvaluationProps) {
                       1
                     );
                     const positiveRatio = ratio > 0;
-                    const ratioColor = positiveRatio ? "#8BC18D" : "#FF6464";
+                    const zeroRatio = ratio === 0;
+                    const negativeRatio = ratio < 0;
+                    const ratioColor = positiveRatio
+                      ? "#8BC18D"
+                      : negativeRatio
+                      ? "#FF6464"
+                      : theme.palette.text.tertiary;
                     const showTrophy = lesserIsBetter
                       ? difference < 0
                       : difference > 0;
@@ -635,13 +649,13 @@ export default function Evaluation(props: EvaluationProps) {
                               )}
                               {filterable && (
                                 <IconButton
-                                  sx={{ p: 0 }}
+                                  sx={{ p: 0.25, borderRadius: 0.5 }}
                                   onClick={() => {
                                     loadView("field", { field: rowId });
                                   }}
                                   title="Load view"
                                 >
-                                  <GridView sx={{ fontSize: 16 }} />
+                                  <GridView sx={{ fontSize: 14 }} />
                                 </IconButton>
                               )}
                             </Stack>
@@ -661,7 +675,10 @@ export default function Evaluation(props: EvaluationProps) {
 
                                 {filterable && (
                                   <IconButton
-                                    sx={{ p: 0 }}
+                                    sx={{
+                                      p: 0.25,
+                                      borderRadius: 0.5,
+                                    }}
                                     onClick={() => {
                                       loadView("field", {
                                         field: rowId,
@@ -686,13 +703,22 @@ export default function Evaluation(props: EvaluationProps) {
                                     direction="row"
                                     sx={{ alignItems: "center" }}
                                   >
-                                    {positiveRatio ? (
+                                    {positiveRatio && (
                                       <ArrowDropUp sx={{ color: ratioColor }} />
-                                    ) : (
+                                    )}
+                                    {negativeRatio && (
                                       <ArrowDropDown
                                         sx={{ color: ratioColor }}
                                       />
                                     )}
+                                    {zeroRatio && (
+                                      <Typography
+                                        pr={1}
+                                        sx={{ color: ratioColor }}
+                                      >
+                                        —
+                                      </Typography>
+                                    )}
                                     <Typography
                                       sx={{
                                         fontSize: "12px",
@@ -854,6 +880,11 @@ export default function Evaluation(props: EvaluationProps) {
                 <Stack direction="row" sx={{ justifyContent: "space-between" }}>
                   <Typography color="secondary">
                     {CLASS_LABELS[performanceClass]} Per Class
+                    {getConfigLabel({
+                      config: classPerformanceConfig,
+                      type: "classPerformance",
+                      dashed: true,
+                    })}
                   </Typography>
                   <Stack
                     alignItems="flex-end"
@@ -895,7 +926,7 @@ export default function Evaluation(props: EvaluationProps) {
                     {classMode === "chart" && (
                       <IconButton
                         onClick={() => {
-                          setBarConfigDialogState((state) => ({
+                          setClassPerformanceDialogConfig((state) => ({
                             ...state,
                             open: true,
                           }));
@@ -1039,6 +1070,23 @@ export default function Evaluation(props: EvaluationProps) {
               Confusion Matrices
             </AccordionSummary>
             <AccordionDetails>
+              <Stack direction="row" sx={{ justifyContent: "space-between" }}>
+                <Typography color="secondary">
+                  {getConfigLabel({ config: confusionMatrixConfig })}
+                </Typography>
+                <Box>
+                  <IconButton
+                    onClick={() => {
+                      setConfusionMatrixDialogConfig((state) => ({
+                        ...state,
+                        open: true,
+                      }));
+                    }}
+                  >
+                    <Settings />
+                  </IconButton>
+                </Box>
+              </Stack>
               <Stack direction={"row"} key={compareKey}>
                 <Stack sx={{ width: "100%" }}>
                   <Stack
@@ -1056,7 +1104,9 @@ export default function Evaluation(props: EvaluationProps) {
                         x: confusionMatrix?.labels,
                         y: confusionMatrix?.labels,
                         type: "heatmap",
-                        colorscale: "viridis",
+                        colorscale: confusionMatrixConfig.log
+                          ? confusionMatrix?.colorscale || "viridis"
+                          : "viridis",
                       },
                     ]}
                     onClick={({ points }) => {
@@ -1082,6 +1132,9 @@ export default function Evaluation(props: EvaluationProps) {
                           x: compareConfusionMatrix?.labels,
                           y: compareConfusionMatrix?.labels,
                           type: "heatmap",
+                          colorscale: confusionMatrixConfig.log
+                            ? compareConfusionMatrix?.colorscale || "viridis"
+                            : "viridis",
                         },
                       ]}
                     />
@@ -1163,7 +1216,7 @@ export default function Evaluation(props: EvaluationProps) {
           </Stack>
           <TextField
             multiline
-            rows={3}
+            rows={10}
             defaultValue={evaluationNotes}
             placeholder="Note (markdown) for the evaluation..."
             onChange={(e) => {
@@ -1194,9 +1247,9 @@ export default function Evaluation(props: EvaluationProps) {
         </Stack>
       </Dialog>
       <Dialog
-        open={barConfigDialogState.open}
+        open={Boolean(classPerformanceDialogConfig.open)}
         fullWidth
-        onClose={closeBarConfigDialog}
+        onClose={closeClassPerformanceConfigDialog}
         PaperProps={{
           sx: { background: (theme) => theme.palette.background.level2 },
         }}
@@ -1211,50 +1264,141 @@ export default function Evaluation(props: EvaluationProps) {
             <Select
               size="small"
               onChange={(e) => {
-                setBarConfigDialogState((state) => ({
+                setClassPerformanceDialogConfig((state) => ({
                   ...state,
                   sortBy: e.target.value as string,
                 }));
               }}
-              defaultValue={barConfigDialogState.sortBy}
+              defaultValue={classPerformanceDialogConfig.sortBy}
             >
-              <MenuItem value="best">Best Performing</MenuItem>
-              <MenuItem value="worst">Worst Performing</MenuItem>
+              {CLASS_PERFORMANCE_SORT_OPTIONS.map((option) => {
+                return (
+                  <MenuItem key={option.value} value={option.value}>
+                    {option.label}
+                  </MenuItem>
+                );
+              })}
             </Select>
           </Stack>
           <Stack spacing={1} pt={1}>
             <Typography color="secondary">Limit bars:</Typography>
             <TextField
-              defaultValue={barConfigDialogState.limit}
+              defaultValue={classPerformanceDialogConfig.limit}
               size="small"
               type="number"
               onChange={(e) => {
-                setBarConfigDialogState((state) => ({
+                const newLimit = parseInt(e.target.value);
+                setClassPerformanceDialogConfig((state) => {
+                  return {
+                    ...state,
+                    limit: isNaN(newLimit) ? undefined : newLimit,
+                  };
+                });
+              }}
+            />
+          </Stack>
+          <Stack direction="row" spacing={1} pt={2}>
+            <Button
+              variant="outlined"
+              color="secondary"
+              sx={{ width: "100%" }}
+              onClick={closeConfusionMatrixConfigDialog}
+            >
+              Cancel
+            </Button>
+            <Button
+              variant="contained"
+              sx={{ width: "100%" }}
+              onClick={() => {
+                setClassPerformanceConfig(classPerformanceDialogConfig);
+                closeClassPerformanceConfigDialog();
+              }}
+            >
+              Save
+            </Button>
+          </Stack>
+        </Stack>
+      </Dialog>
+      <Dialog
+        open={Boolean(confusionMatrixDialogConfig.open)}
+        fullWidth
+        onClose={closeConfusionMatrixConfigDialog}
+        PaperProps={{
+          sx: { background: (theme) => theme.palette.background.level2 },
+        }}
+      >
+        <Stack spacing={2} sx={{ p: 2 }}>
+          <Stack direction="row" spacing={1}>
+            <Settings />
+            <Typography>Display Options: Confusion Matrix</Typography>
+          </Stack>
+          <Stack spacing={1} pt={1}>
+            <Typography color="secondary">Sort by:</Typography>
+            <Select
+              size="small"
+              onChange={(e) => {
+                setConfusionMatrixDialogConfig((state) => ({
                   ...state,
-                  limit: parseInt(e.target.value),
+                  sortBy: e.target.value as string,
                 }));
               }}
+              defaultValue={confusionMatrixDialogConfig.sortBy}
+            >
+              {CONFUSION_MATRIX_SORT_OPTIONS.map((option) => {
+                return (
+                  <MenuItem key={option.value} value={option.value}>
+                    {option.label}
+                  </MenuItem>
+                );
+              })}
+            </Select>
+          </Stack>
+          <Stack spacing={1} pt={1}>
+            <Typography color="secondary">Limit classes:</Typography>
+            <TextField
+              defaultValue={confusionMatrixConfig.limit}
+              size="small"
+              type="number"
+              onChange={(e) => {
+                const newLimit = parseInt(e.target.value);
+                setConfusionMatrixDialogConfig((state) => {
+                  return {
+                    ...state,
+                    limit: isNaN(newLimit) ? undefined : newLimit,
+                  };
+                });
+              }}
             />
           </Stack>
+          <FormControlLabel
+            label="Use logarithmic colorscale"
+            control={
+              <Checkbox
+                defaultChecked={confusionMatrixConfig.log}
+                onChange={(e, checked) => {
+                  setConfusionMatrixDialogConfig((state) => ({
+                    ...state,
+                    log: checked,
+                  }));
+                }}
+              />
+            }
+          />
           <Stack direction="row" spacing={1} pt={2}>
             <Button
               variant="outlined"
               color="secondary"
               sx={{ width: "100%" }}
-              onClick={closeBarConfigDialog}
+              onClick={closeConfusionMatrixConfigDialog}
             >
               Cancel
             </Button>
             <Button
               variant="contained"
               sx={{ width: "100%" }}
-              disabled={
-                barConfigDialogState.sortBy === "" ||
-                barConfigDialogState.limit === 0
-              }
               onClick={() => {
-                setBarConfigState(barConfigDialogState);
-                closeBarConfigDialog();
+                setConfusionMatrixConfig(confusionMatrixDialogConfig);
+                closeConfusionMatrixConfigDialog();
               }}
             >
               Save
@@ -1314,14 +1458,70 @@ const CLASSES = Object.keys(CLASS_LABELS);
 const EXCLUDED_CLASSES = ["macro avg", "micro avg", "weighted avg"];
 
 function formatPerClassPerformance(perClassPerformance, barConfig) {
-  if (!barConfig.sortBy || !barConfig.limit) return perClassPerformance;
+  const { limit, sortBy } = barConfig;
+  if (!sortBy && !limit) return perClassPerformance;
 
-  const sorted = perClassPerformance.sort((a, b) => {
-    if (barConfig.sortBy === "best") {
-      return b.value - a.value;
-    } else {
-      return a.value - b.value;
-    }
-  });
-  return sorted.slice(0, barConfig.limit);
+  let computedPerClassPerformance = perClassPerformance;
+  if (sortBy && sortBy !== DEFAULT_BAR_CONFIG.sortBy) {
+    computedPerClassPerformance = perClassPerformance.sort((a, b) => {
+      if (sortBy === "best") {
+        return b.value - a.value;
+      } else if (sortBy === "worst") {
+        return a.value - b.value;
+      } else {
+        return b.property.localeCompare(a.property);
+      }
+    });
+  }
+
+  if (typeof limit === "number") {
+    return computedPerClassPerformance.slice(0, limit);
+  }
+  return computedPerClassPerformance;
+}
+
+function getMatrix(matrices, config) {
+  if (!matrices) return;
+  const { sortBy = "az", limit } = config;
+  const parsedLimit = typeof limit === "number" ? limit : undefined;
+  const classes = matrices[`${sortBy}_classes`].slice(parsedLimit);
+  const matrix = matrices[`${sortBy}_matrix`].slice(parsedLimit);
+  const colorscale = matrices[`${sortBy}_colorscale`];
+  return { labels: classes, matrix, colorscale };
+}
+
+function getConfigLabel({ config, type, dashed }) {
+  const { sortBy } = config;
+  if (!sortBy) return "";
+  const sortByLabels =
+    type === "classPerformance"
+      ? CLASS_PERFORMANCE_SORT_OPTIONS
+      : CONFUSION_MATRIX_SORT_OPTIONS;
+  const sortByLabel = sortByLabels.find(
+    (option) => option.value === sortBy
+  )?.label;
+  return dashed ? ` - ${sortByLabel}` : sortByLabel;
 }
+
+const CLASS_PERFORMANCE_SORT_OPTIONS = [
+  { value: "az", label: "Alphabetical (A-Z)" },
+  { value: "za", label: "Alphabetical (Z-A)" },
+  { value: "best", label: "Best performing" },
+  { value: "worst", label: "Worst performing" },
+];
+const CONFUSION_MATRIX_SORT_OPTIONS = [
+  { value: "az", label: "Alphabetical (A-Z)" },
+  { value: "za", label: "Alphabetical (Z-A)" },
+  { value: "mc", label: "Most common classes" },
+  { value: "lc", label: "Least common classes" },
+];
+
+type PLOT_CONFIG_TYPE = {
+  sortBy?: string;
+  limit?: number;
+  log?: boolean;
+};
+
+type PLOT_CONFIG_DIALOG_TYPE = PLOT_CONFIG_TYPE & {
+  open?: boolean;
+};
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationNotes.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationNotes.tsx
index c0257ffae3..f281813486 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationNotes.tsx
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationNotes.tsx
@@ -5,7 +5,7 @@ import React, { useCallback, useEffect, useLayoutEffect } from "react";
 
 const DEFAULT_NOTE =
   "No evaluation notes added yet. Click the edit button to add evaluation notes.";
-const MAX_NOTES_HEIGHT = 72;
+const MAX_NOTES_HEIGHT = 128;
 
 export default function EvaluationNotes(props: EvaluationNotesProps) {
   const { notes, variant = "overview" } = props;
@@ -35,11 +35,8 @@ export default function EvaluationNotes(props: EvaluationNotesProps) {
 
   if (!hasNotes && !showDefault) return null;
 
-  const color = hasNotes
-    ? variant === "overview"
-      ? "secondary"
-      : "primary"
-    : "tertiary";
+  const isOverview = variant === "overview";
+  const color = hasNotes ? (isOverview ? "secondary" : "primary") : "tertiary";
 
   return (
     <Box>
@@ -57,7 +54,7 @@ export default function EvaluationNotes(props: EvaluationNotesProps) {
           <Markdown>{notes || DEFAULT_NOTE}</Markdown>
         </Typography>
       </Box>
-      {isTruncated && (
+      {isTruncated && !isOverview && (
         <Box
           sx={{
             background: `linear-gradient(to bottom, hsla(200, 0%, ${
@@ -67,7 +64,7 @@ export default function EvaluationNotes(props: EvaluationNotesProps) {
             }) 100%)`,
             position: "relative",
             top: expanded ? 0 : "-12px",
-            p: "12px 4px 4px 4px",
+            p: 0.5,
           }}
         >
           <Button
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationPlot.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationPlot.tsx
index e5963dfd81..81c8dee348 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationPlot.tsx
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationPlot.tsx
@@ -50,12 +50,14 @@ export default function EvaluationPlot(props: EvaluationPlotProps) {
       scrollZoom: false,
       modeBarButtonsToRemove: [
         "autoScale2d",
+        "select2d",
         "lasso2d",
         "pan2d",
         "resetScale2d",
         "zoom2d",
         "zoomIn2d",
         "zoomOut2d",
+        "toImage",
       ],
     };
   }, []);
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx
index 191189e567..056e5e92f6 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx
@@ -40,7 +40,8 @@ export default function NativeModelEvaluationView(props) {
     }, {});
   }, [computedEvaluations]);
   const { page = "overview", key, id, compareKey } = viewState;
-  const showEmptyOverview = computedEvaluations.length === 0;
+  const showEmptyOverview =
+    computedEvaluations.length === 0 && pending_evaluations.length === 0;
   const triggerEvent = useTriggerEvent();
 
   return (
diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/utils.ts b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/utils.ts
index 1d63ac6dc7..44e7ffec33 100644
--- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/utils.ts
+++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/utils.ts
@@ -17,16 +17,18 @@ export function useTriggerEvent() {
 }
 
 export function getNumericDifference(
-  a,
-  b,
+  value,
+  compareValue,
   percentage = false,
   fractionDigits?: number
 ) {
-  if (typeof a === "number" && typeof b === "number") {
+  if (typeof value === "number" && typeof compareValue === "number") {
+    const difference = value - compareValue;
     if (percentage) {
-      return formatValue(((a - b) / a) * 100, fractionDigits);
+      const percentageDifference = (difference / compareValue) * 100;
+      return formatValue(percentageDifference, fractionDigits);
     }
-    return formatValue(a - b, fractionDigits);
+    return formatValue(difference, fractionDigits);
   }
 }
 
diff --git a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
index 5054c00612..709a00615a 100644
--- a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
+++ b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py
@@ -6,7 +6,7 @@
 |
 """
 
-from collections import defaultdict
+from collections import defaultdict, Counter
 import traceback
 
 from fiftyone import ViewField as F
@@ -15,6 +15,8 @@
 from fiftyone.operators.utils import is_new
 import fiftyone.operators.types as types
 
+from fiftyone.core.plots.plotly import _to_log_colorscale
+
 
 STORE_NAME = "model_evaluation_panel_builtin"
 STATUS_LABELS = {
@@ -24,6 +26,7 @@
 }
 EVALUATION_TYPES_WITH_CONFIDENCE = ["detection", "classification"]
 EVALUATION_TYPES_WITH_IOU = ["detection"]
+ENABLE_CACHING = False
 
 
 class EvaluationPanel(Panel):
@@ -35,7 +38,6 @@ def config(self):
             icon="ssid_chart",
             category=Categories.ANALYZE,
             beta=True,
-            is_new=is_new("2024-11-07"),
         )
 
     def get_dataset_id(self, ctx):
@@ -231,10 +233,42 @@ def get_per_class_metrics(self, info, results):
                 per_class_metrics[c]["support"] = c_report["support"]
         return per_class_metrics
 
-    def get_confusion_matrix(self, results):
-        matrix = results.confusion_matrix()
+    def get_confusion_matrix_colorscale(self, matrix):
+        maxval = matrix.max()
+        colorscale = _to_log_colorscale("oranges", maxval)
+        return colorscale
+
+    def get_confusion_matrices(self, results):
         classes = results.classes
-        return matrix, classes
+        freq = Counter(results.ytrue)
+        if results.missing in freq:
+            freq.pop(results.missing)
+        az_classes = sorted(classes)
+        za_classes = sorted(classes, reverse=True)
+        mc_classes = sorted(freq, key=freq.get, reverse=True)
+        lc_classes = sorted(freq, key=freq.get)
+        az_matrix = results.confusion_matrix(classes=az_classes)
+        za_matrix = results.confusion_matrix(classes=za_classes)
+        mc_matrix = results.confusion_matrix(classes=mc_classes)
+        lc_matrix = results.confusion_matrix(classes=lc_classes)
+        az_colorscale = self.get_confusion_matrix_colorscale(az_matrix)
+        za_colorscale = self.get_confusion_matrix_colorscale(za_matrix)
+        mc_colorscale = self.get_confusion_matrix_colorscale(mc_matrix)
+        lc_colorscale = self.get_confusion_matrix_colorscale(lc_matrix)
+        return {
+            "az_classes": az_classes,
+            "za_classes": za_classes,
+            "mc_classes": mc_classes,
+            "lc_classes": lc_classes,
+            "az_matrix": az_matrix.tolist(),
+            "za_matrix": za_matrix.tolist(),
+            "mc_matrix": mc_matrix.tolist(),
+            "lc_matrix": lc_matrix.tolist(),
+            "az_colorscale": az_colorscale,
+            "za_colorscale": za_colorscale,
+            "mc_colorscale": mc_colorscale,
+            "lc_colorscale": lc_colorscale,
+        }
 
     def load_evaluation(self, ctx):
         view_state = ctx.panel.get_state("view") or {}
@@ -243,7 +277,9 @@ def load_evaluation(self, ctx):
         eval_id = view_state.get("id")
         computed_eval_id = ctx.params.get("id", eval_id)
         store = self.get_store(ctx)
-        evaluation_data = store.get(computed_eval_id)
+        evaluation_data = (
+            store.get(computed_eval_id) if ENABLE_CACHING else None
+        )
         if evaluation_data is None:
             info = ctx.dataset.get_evaluation_info(computed_eval_key)
             results = ctx.dataset.load_evaluation_results(computed_eval_key)
@@ -256,21 +292,18 @@ def load_evaluation(self, ctx):
                 ctx
             )
             metrics["mAP"] = self.get_map(results)
-            matrix, labels = self.get_confusion_matrix(results)
             evaluation_data = {
                 "metrics": metrics,
                 "info": info.serialize(),
-                "confusion_matrix": {
-                    "matrix": matrix.tolist(),
-                    "labels": labels.tolist(),
-                },
+                "confusion_matrices": self.get_confusion_matrices(results),
                 "per_class_metrics": per_class_metrics,
             }
-            # Cache the evaluation data
-            try:
-                store.set(computed_eval_id, evaluation_data)
-            except Exception:
-                traceback.print_exc()
+            if ENABLE_CACHING:
+                # Cache the evaluation data
+                try:
+                    store.set(computed_eval_id, evaluation_data)
+                except Exception:
+                    traceback.print_exc()
 
         ctx.panel.set_data(f"evaluation_{computed_eval_key}", evaluation_data)
 
@@ -278,26 +311,41 @@ def on_change_view(self, ctx):
         # Used only for triggering re-renders when the view changes
         pass
 
+    def has_evaluation_results(self, dataset, eval_key):
+        try:
+            return bool(dataset._doc.evaluations[eval_key].results)
+        except Exception:
+            return False
+
     def load_pending_evaluations(self, ctx, skip_update=False):
+        pending_evaluations = []
+        eval_keys = ctx.dataset.list_evaluations()
         store = self.get_store(ctx)
         dataset_id = self.get_dataset_id(ctx)
-        pending_evaluations = store.get("pending_evaluations") or {}
-        pending = pending_evaluations.get(dataset_id, [])
-        if not skip_update:
-            eval_keys = ctx.dataset.list_evaluations()
-            updated_pending = []
-            update = False
-            for item in pending:
-                pending_eval_key = item.get("eval_key")
-                if pending_eval_key in eval_keys:
-                    update = True
-                else:
-                    updated_pending.append(item)
-            if update:
-                pending_evaluations[dataset_id] = updated_pending
-                store.set("pending_evaluations", pending_evaluations)
-                pending = updated_pending
-        ctx.panel.set_data("pending_evaluations", pending)
+        pending_evaluations_in_store = store.get("pending_evaluations") or {}
+        pending_evaluations_for_dataset_in_store = (
+            pending_evaluations_in_store.get(dataset_id, [])
+        )
+        updated_pending_evaluations_for_dataset_in_stored = []
+        update_store = False
+        for pending in pending_evaluations_for_dataset_in_store:
+            pending_eval_key = pending.get("eval_key")
+            if pending_eval_key in eval_keys:
+                update_store = True
+            else:
+                pending_evaluations.append(pending)
+                updated_pending_evaluations_for_dataset_in_stored.append(
+                    pending
+                )
+        for key in eval_keys:
+            if not self.has_evaluation_results(ctx.dataset, key):
+                pending_evaluations.append({"eval_key": key})
+        if update_store:
+            pending_evaluations_in_store[
+                dataset_id
+            ] = updated_pending_evaluations_for_dataset_in_stored
+            store.set("pending_evaluations", pending_evaluations)
+        ctx.panel.set_data("pending_evaluations", pending_evaluations)
 
     def on_evaluate_model_success(self, ctx):
         dataset_id = self.get_dataset_id(ctx)
@@ -320,7 +368,7 @@ def on_evaluate_model_success(self, ctx):
             pending_evaluations[dataset_id] = []
         pending_evaluations[dataset_id].append(pending)
         store.set("pending_evaluations", pending_evaluations)
-        self.load_pending_evaluations(ctx, True)
+        self.load_pending_evaluations(ctx)
 
     def on_evaluate_model(self, ctx):
         if not self.can_evaluate(ctx):