Normal curve adornment (#1298)

* [#186948671] Feature: The user can cause a normal curve to be drawn on top of a dot plot

This is the first portion of work on a normal curve. It covers display of normal curve on dot plots. Remaining is display of normal curve on histograms and gaussian fit curves on histograms when there exists a URL parameter for gaussianFit and code for fitting to histograms brought in from V2.

* Start by modifying standard error adornment
  * The normal-curve-adornment-component.tsx file contains code that is redundant with the standard-error-adornment-component.tsx file. Presumably there is a way to decrease the redundancy, perhaps by making `useAdornmentAttributes` or some similar hook. But we're not making that attempt now.
* The `NormalCurveAdornmentModel` has no real state of its own. Its purpose is to compute mean, standard deviation and standard error for each cell in which we draw the curve. I decided to put `getCaseCount` in `UnivariateMeasureAdornmentModel` even though it's only so far used for the normal curve.
* It proved necessary to store the point overlap in `GraphContentModel` so that the normal curve adornment could access it and use it for computing coordinates.
* `en-US.json5` was out of date compared to the most recent V2 counterpart
* Comment out portions of code that refer to standard error since we won't be displaying that until we have histograms that display a Gaussian Fit curve instead of a normal curve
* In cleaning up existing elements of measure adornments, test for undefined
* Modified `v2-adornment-importer.test.ts` to expect 10 adornments

* chore: code review tweaks

* * Code review changes

---------

Co-authored-by: Kirk Swenson <[email protected]>

v3/src/components/graph/adornments/adornment-types.ts

@@ -12,15 +12,13 @@ import { IStandardDeviationAdornmentModel, StandardDeviationAdornmentModel }
   from "./univariate-measures/standard-deviation/standard-deviation-adornment-model"
 import { IStandardErrorAdornmentModel, StandardErrorAdornmentModel }
   from "./univariate-measures/standard-error/standard-error-adornment-model"
+import { NormalCurveAdornmentModel } from "./univariate-measures/normal-curve/normal-curve-adornment-model"
 import { IMeanAbsoluteDeviationAdornmentModel, MeanAbsoluteDeviationAdornmentModel }
   from "./univariate-measures/mean-absolute-deviation/mean-absolute-deviation-adornment-model"
 import { BoxPlotAdornmentModel, IBoxPlotAdornmentModel } from "./univariate-measures/box-plot/box-plot-adornment-model"
 import { PlottedFunctionAdornmentModel, IPlottedFunctionAdornmentModel }
   from "./plotted-function/plotted-function-adornment-model"
 import { ILSRLAdornmentModel, LSRLAdornmentModel } from "./lsrl/lsrl-adornment-model"
-
-export const kGraphAdornmentsClass = "graph-adornments-grid"
-export const kGraphAdornmentsClassSelector = `.${kGraphAdornmentsClass}`
 export const kDefaultFontSize = 12
 export const kGraphAdornmentsBannerHeight = 22
 
@@ -39,6 +37,7 @@ const adornmentTypeDispatcher = (adornmentSnap: IAdornmentModel) => {
     case "Plotted Value": return PlottedValueAdornmentModel
     case "Standard Deviation": return StandardDeviationAdornmentModel
     case "Standard Error": return StandardErrorAdornmentModel
+    case "Normal Curve": return NormalCurveAdornmentModel
     default: {
       console.warn(`Unknown adornment type: ${adornmentSnap.type}`)
       return UnknownAdornmentModel
@@ -49,7 +48,7 @@ const adornmentTypeDispatcher = (adornmentSnap: IAdornmentModel) => {
 export const AdornmentModelUnion = types.union({ dispatcher: adornmentTypeDispatcher },
   BoxPlotAdornmentModel, CountAdornmentModel, LSRLAdornmentModel, MeanAdornmentModel,
   MeanAbsoluteDeviationAdornmentModel, MedianAdornmentModel, MovableValueAdornmentModel, MovableLineAdornmentModel,
-  MovablePointAdornmentModel, PlottedFunctionAdornmentModel, PlottedValueAdornmentModel,
+  MovablePointAdornmentModel, NormalCurveAdornmentModel, PlottedFunctionAdornmentModel, PlottedValueAdornmentModel,
   StandardDeviationAdornmentModel, StandardErrorAdornmentModel, UnknownAdornmentModel)
 export type IAdornmentModelUnion = IBoxPlotAdornmentModel | ICountAdornmentModel | ILSRLAdornmentModel |
   IMeanAdornmentModel | IMeanAbsoluteDeviationAdornmentModel | IMedianAdornmentModel | IMovableValueAdornmentModel |

v3/src/components/graph/adornments/adornment-ui-types.ts

@@ -38,6 +38,7 @@ export const measures: IMeasures = {
       title: "DG.Inspector.graphBoxPlotNormalCurveOptions", type: "Group", rulerStateKey: 'boxPlotAndNormalCurve',
       items: [
         {title: "DG.Inspector.graphPlottedBoxPlot", type: "Box Plot"},
+        {title: "DG.Inspector.graphPlottedNormal", type: "Normal Curve"},
       ]
     },
     {

v3/src/components/graph/adornments/univariate-measures/normal-curve/normal-curve-adornment-model.test.ts

@@ -0,0 +1,30 @@
+import { NormalCurveAdornmentModel } from "./normal-curve-adornment-model"
+import { kNormalCurveType } from "./normal-curve-adornment-types"
+
+describe("NormalCurveModel", () => {
+  it("can be created", () => {
+    const mean = NormalCurveAdornmentModel.create()
+    expect(mean).toBeDefined()
+    expect(mean.type).toEqual(kNormalCurveType)
+  })
+  it("can have its showLabels property set", () => {
+    const mean = NormalCurveAdornmentModel.create()
+    expect(mean.showMeasureLabels).toBe(false)
+    mean.setShowMeasureLabels(true)
+    expect(mean.showMeasureLabels).toBe(true)
+  })
+  it("can have a new normal curve added to its measures map", () => {
+    const adornment = NormalCurveAdornmentModel.create()
+    expect(adornment.measures.size).toBe(0)
+    adornment.addMeasure(10)
+    expect(adornment.measures.size).toBe(1)
+  })
+  it("can have an existing normal curve removed from its measures map", () => {
+    const adornment = NormalCurveAdornmentModel.create()
+    expect(adornment.measures.size).toBe(0)
+    adornment.addMeasure(10)
+    expect(adornment.measures.size).toBe(1)
+    adornment.removeMeasure("{}")
+    expect(adornment.measures.size).toBe(0)
+  })
+})

v3/src/components/graph/adornments/univariate-measures/normal-curve/normal-curve-adornment-model.ts

@@ -0,0 +1,36 @@
+import { Instance, types } from "mobx-state-tree"
+import { mean, std } from "mathjs"
+import {IGraphDataConfigurationModel} from "../../../models/graph-data-configuration-model"
+import { UnivariateMeasureAdornmentModel, IUnivariateMeasureAdornmentModel }
+  from "../univariate-measure-adornment-model"
+import { kNormalCurveValueTitleKey, kNormalCurveType } from "./normal-curve-adornment-types"
+
+export const NormalCurveAdornmentModel = UnivariateMeasureAdornmentModel
+  .named("NormalCurveAdornmentModel")
+  .props({
+    type: types.optional(types.literal(kNormalCurveType), kNormalCurveType),
+    labelTitle: types.optional(types.literal(kNormalCurveValueTitleKey), kNormalCurveValueTitleKey),
+  })
+  .views(self => ({
+    computeMean(attrId: string, cellKey: Record<string, string>, dataConfig: IGraphDataConfigurationModel) {
+      return mean(self.getCaseValues(attrId, cellKey, dataConfig))
+    },
+    computeStandardDeviation(attrId: string, cellKey: Record<string, string>,
+                             dataConfig: IGraphDataConfigurationModel) {
+      // Cast to Number should not be necessary, but there appears to be an issue with the mathjs type signature.
+      // See https://github.com/josdejong/mathjs/issues/2429 for some history, although that bug is supposedly
+      // fixed, but a variant of it seems to be re-occurring.
+      return Number(std(self.getCaseValues(attrId, cellKey, dataConfig)))
+    },
+    computeStandardError(attrId: string, cellKey: Record<string, string>,
+                             dataConfig: IGraphDataConfigurationModel) {
+      return this.computeStandardDeviation(attrId, cellKey, dataConfig) /
+        Math.sqrt(self.getCaseCount(attrId, cellKey, dataConfig))
+    },
+  }))
+
+export interface INormalCurveAdornmentModel extends Instance<typeof NormalCurveAdornmentModel> {}
+export function isNormalCurveAdornment(adornment: IUnivariateMeasureAdornmentModel):
+  adornment is INormalCurveAdornmentModel {
+  return adornment.type === kNormalCurveType
+}

v3/src/components/graph/adornments/univariate-measures/normal-curve/normal-curve-adornment-registration.test.ts

@@ -0,0 +1,20 @@
+import { getAdornmentComponentInfo } from "../../adornment-component-info"
+import { getAdornmentContentInfo } from "../../adornment-content-info"
+import { kNormalCurveClass, kNormalCurvePrefix,
+         kNormalCurveType } from "./normal-curve-adornment-types"
+import "./normal-curve-adornment-registration"
+
+describe("NormalCurveRegistration", () => {
+  it("registers content and component info", () => {
+    const standardErrorInfo = getAdornmentContentInfo(kNormalCurveType)
+    expect(standardErrorInfo).toBeDefined()
+    expect(standardErrorInfo?.type).toBe(kNormalCurveType)
+    expect(standardErrorInfo?.modelClass).toBeDefined()
+    expect(standardErrorInfo?.prefix).toBe(kNormalCurvePrefix)
+    const standardErrorComponentInfo = getAdornmentComponentInfo(kNormalCurveType)
+    expect(standardErrorComponentInfo).toBeDefined()
+    expect(standardErrorComponentInfo?.adornmentEltClass).toBe(kNormalCurveClass)
+    expect(standardErrorComponentInfo?.Component).toBeDefined()
+    expect(standardErrorComponentInfo?.type).toBe(kNormalCurveType)
+  })
+})

v3/src/components/graph/adornments/univariate-measures/normal-curve/normal-curve-adornment-registration.tsx

@@ -0,0 +1,44 @@
+import React from "react"
+import { AdornmentCheckbox } from "../../adornment-checkbox"
+import { registerAdornmentComponentInfo } from "../../adornment-component-info"
+import { registerAdornmentContentInfo } from "../../adornment-content-info"
+import {
+  kNormalCurveClass, kNormalCurveLabelKey, kNormalCurveType, kNormalCurvePrefix,
+  kNormalCurveUndoAddKey, kNormalCurveRedoAddKey, kNormalCurveRedoRemoveKey,
+  kNormalCurveUndoRemoveKey
+} from "./normal-curve-adornment-types"
+import { NormalCurveAdornmentModel } from "./normal-curve-adornment-model"
+import { NormalCurveAdornmentComponent } from "./normal-curve-adornment-component"
+
+const Controls = () => {
+  return (
+    <AdornmentCheckbox
+      classNameValue={kNormalCurveClass}
+      labelKey={kNormalCurveLabelKey}
+      type={kNormalCurveType}
+    />
+  )
+}
+
+registerAdornmentContentInfo({
+  type: kNormalCurveType,
+  parentType: "Univariate Measure",
+  plots: ["dotPlot"],
+  prefix: kNormalCurvePrefix,
+  modelClass: NormalCurveAdornmentModel,
+  undoRedoKeys: {
+    undoAdd: kNormalCurveUndoAddKey,
+    redoAdd: kNormalCurveRedoAddKey,
+    undoRemove: kNormalCurveUndoRemoveKey,
+    redoRemove: kNormalCurveRedoRemoveKey,
+  }
+})
+
+registerAdornmentComponentInfo({
+  adornmentEltClass: kNormalCurveClass,
+  Component: NormalCurveAdornmentComponent,
+  Controls,
+  labelKey: kNormalCurveLabelKey,
+  order: 10,
+  type: kNormalCurveType
+})

v3/src/components/graph/adornments/univariate-measures/normal-curve/normal-curve-adornment-types.ts

@@ -0,0 +1,11 @@
+export const kNormalCurveClass = "normal-curve"
+export const kNormalCurveType = "Normal Curve"
+export const kNormalCurvePrefix = "ADRN"
+export const kNormalCurveLabelKey = "DG.Inspector.graphPlottedNormal"
+export const kNormalCurveUndoAddKey = "DG.Undo.graph.showPlottedNormal"
+export const kNormalCurveRedoAddKey = "DG.Redo.graph.showPlottedNormal"
+export const kNormalCurveUndoRemoveKey = "DG.Undo.graph.hidePlottedNormal"
+export const kNormalCurveRedoRemoveKey = "DG.Redo.graph.hidePlottedNormal"
+export const kNormalCurveValueTitleKey = "" // We don't have a single value title because we display mean and sd
+export const kNormalCurveMeanValueTitleKey = "DG.PlottedAverageAdornment.meanValueTitle"
+export const kNormalCurveStdDevValueTitleKey = "DG.PlottedAverageAdornment.stDevValueTitle"

v3/src/components/graph/adornments/univariate-measures/univariate-measure-adornment-base-component.scss

@@ -67,6 +67,23 @@
     }
   }
 
+  .normal-curve {
+    fill: none;
+    stroke: #027d34;
+    stroke-width: 1px;
+  }
+
+  .normal-curve-hover-cover {
+    stroke: #027d34;
+    stroke-width: 6px;
+    opacity: 0.001;
+    pointer-events: all;
+
+    &:hover, &.highlighted {
+      opacity: .2;
+    }
+  }
+
   .measure-tip {
     display: none;
     fill: #000;

v3/src/components/graph/adornments/univariate-measures/univariate-measure-adornment-model.ts

@@ -50,6 +50,9 @@ export const UnivariateMeasureAdornmentModel = AdornmentModel
       })
       return caseValues
     },
+    getCaseCount(attrId: string, cellKey: Record<string, string>, dataConfig: IGraphDataConfigurationModel) {
+      return this.getCaseValues(attrId, cellKey, dataConfig).length
+    },
     get isUnivariateMeasure() {
       return true
     },

v3/src/components/graph/adornments/univariate-measures/univariate-measure-adornment-simple-component.tsx

@@ -237,7 +237,7 @@ export const UnivariateMeasureAdornmentSimpleComponent = observer(
     useEffect(() => {
       // Clean up any existing elements
       return () => {
-        Object.values(valueObjRef.current).forEach((aSelection) => aSelection.remove())
+        Object.values(valueObjRef.current).forEach((aSelection) => aSelection?.remove())
       }
     }, [])
 

v3/src/components/graph/adornments/v2-adornment-importer.test.ts

@@ -88,7 +88,7 @@ describe("V2AdornmentImporter", () => {
     })
     expect(adornmentStore).toBeDefined()
     expect(adornmentStore.showMeasureLabels).toBe(true)
-    expect(adornmentStore.adornments.length).toBe(9)
+    expect(adornmentStore.adornments.length).toBe(10)
     const meanAdornment = adornmentStore.adornments.find(a=> a.type === "Mean") as IMeanAdornmentModel
     expect(meanAdornment).toBeDefined()
     expect(meanAdornment.id).toBeDefined()

v3/src/components/graph/adornments/v2-adornment-importer.ts

@@ -17,6 +17,7 @@ import { kMedianType } from "./univariate-measures/median/median-adornment-types
 import { kPlottedValueType } from "./univariate-measures/plotted-value/plotted-value-adornment-types"
 import { kStandardDeviationType } from "./univariate-measures/standard-deviation/standard-deviation-adornment-types"
 import { kStandardErrorType } from "./univariate-measures/standard-error/standard-error-adornment-types"
+import { kNormalCurveType } from "./univariate-measures/normal-curve/normal-curve-adornment-types"
 
 interface IProps {
   data?: Record<string, any>
@@ -307,6 +308,19 @@ export const v2AdornmentImporter = ({data, plotModels, attributeDescriptions, yA
     v3Adornments.push(boxPlotAdornmentImport)
   }
 
+  // NORMAL CURVE
+  const normalCurveAdornment = v2Adornments.plottedNormal
+  if (normalCurveAdornment) {
+    const measures = univariateMeasureInstances(normalCurveAdornment, instanceKeys)
+    const normalCurveAdornmentImport = {
+      id: typedId("ADRN"),
+      isVisible: normalCurveAdornment.isVisible,
+      measures,
+      type: kNormalCurveType
+    }
+    v3Adornments.push(normalCurveAdornmentImport)
+  }
+
   // MOVABLE VALUES
   const movableValuesAdornment = v2Adornments.multipleMovableValues
   if (movableValuesAdornment) {

v3/src/components/graph/components/freedotplotdots.tsx

@@ -54,6 +54,7 @@ export const FreeDotPlotDots = observer(function FreeDotPlotDots(props: PlotProp
         pointDiameter, primaryAttrID, primaryAxisScale, primaryPlace, secondaryAttrID, secondaryBandwidth
       }
       const { binMap, overlap } = computeBinPlacements(binPlacementProps)
+      graphModel.setPointOverlap(overlap) // So that if we draw a normal curve, it can use the overlap
 
       interface ISubPlotDetails {
         cases: string[]

v3/src/components/graph/models/graph-content-model.ts

@@ -88,7 +88,8 @@ export const GraphContentModel = DataDisplayContentModel
     dragBinIndex: -1,
     dynamicBinAlignment: undefined as number | undefined,
     dynamicBinWidth: undefined as number | undefined,
-    prevDataSetId: ""
+    prevDataSetId: "",
+    pointOverlap: 0,  // Set by plots so that it is accessible to adornments
   }))
   .preProcessSnapshot(snap => {
     // some properties were historically written out as null because NaN => null in JSON
@@ -108,7 +109,10 @@ export const GraphContentModel = DataDisplayContentModel
     },
     setDragBinIndex(index: number) {
       self.dragBinIndex = index
-    }
+    },
+    setPointOverlap(overlap: number) {
+      self.pointOverlap = overlap
+    },
   }))
   .views(self => ({
     get graphPointLayerModel(): IGraphPointLayerModel {

v3/src/components/graph/register-adornment-types.ts

@@ -4,6 +4,7 @@ import "./adornments/movable-line/movable-line-adornment-registration"
 import "./adornments/movable-point/movable-point-adornment-registration"
 import "./adornments/movable-value/movable-value-adornment-registration"
 import "./adornments/univariate-measures/box-plot/box-plot-adornment-registration"
+import "./adornments/univariate-measures/normal-curve/normal-curve-adornment-registration"
 import "./adornments/plotted-function/plotted-function-adornment-registration"
 import "./adornments/univariate-measures/mean/mean-adornment-registration"
 import "./adornments/univariate-measures/mean-absolute-deviation/mean-absolute-deviation-adornment-registration"

v3/src/components/graph/utilities/graph-utils.ts

@@ -660,11 +660,12 @@ export const pathBasis = (p0: Point, p1: Point, p2: Point, p3: Point) => {
    * Returns the point that is the weighted sum of the specified control points,
    * using the specified weights. This method requires that there are four
    * weights and four control points.
+   * We round to 2 decimal places to keep the length of path strings relatively short.
    */
   const weight = (w: number[]) => {
     return {
-      x: w[0] * p0.x + w[1] * p1.x + w[2] * p2.x + w[3] * p3.x,
-      y: w[0] * p0.y  + w[1] * p1.y  + w[2] * p2.y  + w[3] * p3.y
+      x: Math.round((w[0] * p0.x + w[1] * p1.x + w[2] * p2.x + w[3] * p3.x) * 100) / 100,
+      y: Math.round((w[0] * p0.y  + w[1] * p1.y  + w[2] * p2.y  + w[3] * p3.y) * 100) / 100
     }
   }
 

v3/src/utilities/math-utils.ts

@@ -127,3 +127,13 @@ export function goodTickValue(iMin: number, iMax: number) {
   return Math.max(power * base, Number.MIN_VALUE)
 }
 
+// The normal distribution function. The amplitude is the height of the curve at the mean.
+// The mean is the center of the curve. The sigma is the standard deviation of the curve.
+// The formula for the normal distribution is:
+// f(x) = amplitude * exp(-(x - mean)^2 / (2 * sigma^2))
+export function normal(x: number, amp: number, mu: number, sigma: number) {
+  const exponent = -(Math.pow(x - mu, 2) / (2 * Math.pow(sigma, 2)))
+  return amp * Math.exp(exponent)
+}
+
+

v3/src/utilities/translation/lang/en-US.json5

@@ -904,7 +904,7 @@
     // DG.PlottedAverageAdornment
     "DG.PlottedAverageAdornment.meanValueTitle": "mean=%@", // "mean=123.456"
     "DG.PlottedAverageAdornment.medianValueTitle": "median=%@", // "median=123.456"
-    "DG.PlottedAverageAdornment.stDevValueTitle": "±1 SD, %@", // "±1 SD, 123.456"
+    "DG.PlottedAverageAdornment.stDevValueTitle": "SD=%@", // "±1 SD, 123.456"
     "DG.PlottedAverageAdornment.stErrValueTitle": "%@ SE%@mean%@=%@", // "2 SE=123.456" 2nd & 3rd args for subscripting
     "DG.PlottedAverageAdornment.madValueTitle": "±1 MAD, %@", // "±1 MAD, 123.456"
     "DG.PlottedAverageAdornment.iqrValueTitle": "IQR=%@", // "iqr=123.456"