Merge pull request #153 from concord-consortium/187867019-optimize-ti…

…meseries-data-format

feat: Optimize time series data format [PT-187867019]

src/lara-app/utils/download-csv.test.ts

@@ -1,3 +1,4 @@
+import { TimeSeriesDataKey, TimeSeriesMetadataKey } from "../../shared/utils/time-series";
 import { IExperiment, IExperimentData } from "../../shared/experiment-types";
 import { downloadCSV, getFilename, getRows, getTimeKey, sortNaturally } from "./download-csv";
 
@@ -54,7 +55,7 @@ const timeSeriesExperiment: IExperiment = {
         experimentData: {
           items: {
             properties: {
-              timeSeries: {
+              [TimeSeriesDataKey]: {
                 title: "Results"
               },
               label: {
@@ -68,7 +69,7 @@ const timeSeriesExperiment: IExperiment = {
     formUiSchema: {
       experimentData: {
         "ui:dataTableOptions": {
-          sensorFields: ["timeSeries"]
+          sensorFields: [TimeSeriesDataKey]
         }
       }
     }
@@ -78,12 +79,11 @@ const timeSeriesExperiment: IExperiment = {
 const timeSeriesData: IExperimentData = {
   timestamp: 987654321,
   experimentData: [
-    {timeSeries: [{time: 0, value: 1}, {time: 1, value: 2}], label: "Label #1"},
-    {timeSeries: [{time: 1, value: 3}, {time: 2, value: 4}], label: "Label #2"}
+    {[TimeSeriesDataKey]: [1, 2], [TimeSeriesMetadataKey]: { measurementPeriod: 1000}, label: "Label #1"},
+    {[TimeSeriesDataKey]: [3, 4], [TimeSeriesMetadataKey]: { measurementPeriod: 2000}, label: "Label #2"}
   ]
 };
 
-
 describe("download csv functions", () => {
 
   describe("getTimeKey()", () => {
@@ -126,22 +126,22 @@ describe("download csv functions", () => {
       expect(getRows(timeSeriesExperiment, timeSeriesData)).toStrictEqual([
         {
           "Time": "0",
-          "Row 1 Results": 1,
-          "Row 2 Results": "",
+          "Row 1 Results": "1",
+          "Row 2 Results": "3",
           "Row 1 Label": "Label #1",
           "Row 2 Label": "Label #2",
         },
         {
           "Time": "1",
-          "Row 1 Results": 2,
-          "Row 2 Results": 3,
+          "Row 1 Results": "2",
+          "Row 2 Results": "",
           "Row 1 Label": "Label #1",
           "Row 2 Label": "Label #2",
         },
         {
           "Time": "2",
           "Row 1 Results": "",
-          "Row 2 Results": 4,
+          "Row 2 Results": "4",
           "Row 1 Label": "Label #1",
           "Row 2 Label": "Label #2",
         },

src/lara-app/utils/download-csv.ts

@@ -1,10 +1,11 @@
 import {unparse} from "papaparse";
 
-import { IDataTableRow, IDataTableTimeData } from "../../shared/components/data-table-field";
+import { IDataTableRow } from "../../shared/components/data-table-field";
 import { IExperiment, IExperimentData } from "../../shared/experiment-types";
 import { isFunctionSymbol, handleSpecialValue } from "../../shared/utils/handle-special-value";
+import { ITimeSeriesMetadata, TimeSeriesDataKey, TimeSeriesMetadataKey } from "../../shared/utils/time-series";
 
-type TimeValues = Record<string, number|undefined>;
+type TimeValues = Record<string, string|undefined>;
 type OtherValues = Record<string, any>;
 type TimeSeriesRow = {timeValues: TimeValues, otherValues: OtherValues};
 
@@ -36,10 +37,10 @@ export const getRows = (experiment: IExperiment, data: IExperimentData) => {
   }, {});
 
   const rawRows: IDataTableRow[] = data.experimentData;
-  const isTimeSeries = (experiment.schema.formUiSchema?.experimentData?.["ui:dataTableOptions"]?.sensorFields || []).indexOf("timeSeries") !== -1;
+  const isTimeSeries = (experiment.schema.formUiSchema?.experimentData?.["ui:dataTableOptions"]?.sensorFields || []).indexOf(TimeSeriesDataKey) !== -1;
 
   if (isTimeSeries) {
-    const nonTimeSeriesTitles = Object.keys(titleMap).filter(key => key !== "timeSeries");
+    const nonTimeSeriesTitles = Object.keys(titleMap).filter(key => key !== TimeSeriesDataKey);
     const timeKeys = new Set<string>();
     const timeSeriesRows: TimeSeriesRow[] = [];
 
@@ -51,10 +52,13 @@ export const getRows = (experiment: IExperiment, data: IExperimentData) => {
         otherValues[titleMap[key]] = getRowValue(key, rawRow, rawRows);
       });
 
-      const timeSeries = (rawRow.timeSeries ?? []) as IDataTableTimeData[];
-      timeSeries.forEach(({time, value}) => {
+      const timeSeries = (rawRow[TimeSeriesDataKey] ?? []) as number[];
+      const {measurementPeriod} = (rawRow[TimeSeriesMetadataKey] ?? {measurementPeriod: 0}) as ITimeSeriesMetadata;
+      const timeDelta = measurementPeriod / 1000;
+      timeSeries.forEach((value, index) => {
+        const time = index * timeDelta;
         const timeKey = getTimeKey(time);
-        timeValues[timeKey] = value;
+        timeValues[timeKey] = String(value);
         timeKeys.add(timeKey);
       });
 
@@ -64,7 +68,7 @@ export const getRows = (experiment: IExperiment, data: IExperimentData) => {
 
     // unroll each time/value reading into its own row
     const unrolledRows: Record<string,any>[] = [];
-    const timeSeriesKey = titleMap.timeSeries ?? "Time Series Value";
+    const timeSeriesKey = titleMap[TimeSeriesDataKey] ?? "Time Series Value";
     sortedTimeKeys.forEach(timeKey => {
       const row: Record<string, any> = {Time: timeKey};
       timeSeriesRows.forEach(({timeValues}, rowIndex) => {

src/mobile-app/components/sensor.tsx

@@ -1,13 +1,13 @@
 import React, { useState, useRef, useMemo, useEffect } from "react";
 import classNames from "classnames";
 import { SensorValue } from "./sensor-value";
-import { Sensor, IConnectDevice, SelectDeviceFn, ITimeSeriesCapabilities, MaxNumberOfTimeSeriesValues, ISensorValues } from "../../sensors/sensor";
+import { Sensor, IConnectDevice, SelectDeviceFn, ISensorValues } from "../../sensors/sensor";
+import { ITimeSeriesCapabilities, ITimeSeriesMetadata, MaxNumberOfTimeSeriesValues, getTimeSeriesMetadata } from "../../shared/utils/time-series";
 import { useSensor } from "../hooks/use-sensor";
 import { MenuComponent, MenuItemComponent } from "../../shared/components/menu";
 import { inCordova } from "../../shared/utils/in-cordova";
 import { SensorStrength } from "./sensor-strength";
 import { Icon } from "../../shared/components/icon";
-import { IDataTableTimeData } from "../../shared/components/data-table-field";
 import DataTableSparkGraph from "../../shared/components/data-table-sparkgraph";
 
 import css from "./sensor.module.scss";
@@ -80,29 +80,33 @@ export const SensorComponent: React.FC<ISensorComponentProps> = ({sensor, manual
   const cancelSelectDevice = useRef<CancelDeviceFn|undefined>();
 
   // generate a sliding window of time series values
-  const latestValuesRef = useRef<IDataTableTimeData[]>([]);
+  const timeSeriesValuesRef = useRef<number[]>([]);
   useEffect(() => {
     if (!sensor.connected || !isTimeSeries || !timeSeriesCapabilities) {
-      latestValuesRef.current = [];
+      timeSeriesValuesRef.current = [];
       return;
     }
 
     const valueKey = timeSeriesCapabilities.valueKey as keyof ISensorValues;
-    const timeDelta = sensor.pollInterval / 1000;
-    const hasValues = latestValuesRef.current.length > 0;
-    let latestTime = hasValues ? latestValuesRef.current[latestValuesRef.current.length - 1].time : 0;
-    let nextTime = hasValues ? latestTime + timeDelta : 0;
-    if (latestTime >= maxTime) {
-      nextTime = maxTime;
+    const measurementPeriod = sensor.pollInterval / 1000;
+    let latestTime = timeSeriesValuesRef.current.length * measurementPeriod;
+    if (latestTime > maxTime) {
       latestTime = maxTime;
-      latestValuesRef.current = latestValuesRef.current.slice(1).map(({value}, index) => {
-        return {value, time: index * timeDelta};
-      });
+      timeSeriesValuesRef.current = timeSeriesValuesRef.current.slice(1);
     }
-    latestValuesRef.current.push({value: values[valueKey] ?? 0, time: nextTime});
-    latestValuesRef.current[0].capabilities = timeSeriesCapabilities;
+    timeSeriesValuesRef.current.push(values[valueKey] ?? 0);
   }, [isTimeSeries, sensor, timeSeriesCapabilities, values]);
 
+  const timeSeriesMetadataRef = useRef<ITimeSeriesMetadata|undefined>(undefined);
+  useEffect(() => {
+    if (!isTimeSeries || !timeSeriesCapabilities) {
+      timeSeriesMetadataRef.current = undefined;
+      return;
+    }
+
+    timeSeriesMetadataRef.current = {...getTimeSeriesMetadata(timeSeriesCapabilities), measurementPeriod: sensor.pollInterval};
+  }, [isTimeSeries, timeSeriesCapabilities]);
+
   const timeSeriesPeriods = useMemo(() => {
     if (!timeSeriesCapabilities) {
       return [];
@@ -222,7 +226,7 @@ export const SensorComponent: React.FC<ISensorComponentProps> = ({sensor, manual
   };
 
   const renderTimeSeries = () => {
-    if (!timeSeriesCapabilities) {
+    if (!timeSeriesCapabilities || !timeSeriesMetadataRef.current) {
       return <div className={css.timeSeriesValue} />;
     }
 
@@ -240,7 +244,8 @@ export const SensorComponent: React.FC<ISensorComponentProps> = ({sensor, manual
             <DataTableSparkGraph
               width={25}
               height={50}
-              values={latestValuesRef.current}
+              values={timeSeriesValuesRef.current}
+              metadata={timeSeriesMetadataRef.current}
               minTime={0}
               maxTime={maxTime}
               showAxes={true}

src/sensors/device-sensor.ts

@@ -1,13 +1,11 @@
-import { Sensor, ISensorValues, ISensorOptions, ISensorCapabilities, ISetConnectedOptions, IPollOptions, IConnectOptions, ITimeSeriesCapabilities } from "./sensor";
-
+import { Sensor, ISensorValues, ISensorOptions, ISetConnectedOptions, IPollOptions, IConnectOptions } from "./sensor";
 import { Device, ISelectableSensorInfo } from "./devices/device";
 import { SensorTag2Device } from "./devices/sensor-tag-cc2650";
 import { SensorTagCC1350Device } from "./devices/sensor-tag-cc1350";
 import { MultiSensorDevice } from "./devices/multi-sensor";
 import { GDXSensorDevice } from "./devices/gdx-sensor";
 import { logInfo } from "../shared/utils/log";
 import { inCordova } from "../shared/utils/in-cordova";
-import { IDataTableTimeData } from "../shared/components/data-table-field";
 
 declare global {
   // tslint:disable-next-line:interface-name
@@ -123,7 +121,7 @@ export class DeviceSensor extends Sensor {
     });
   }
 
-  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: IDataTableTimeData[]) => void): () => void {
+  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: number[]) => void): () => void {
     if (this.device) {
       return this.device.collectTimeSeries(measurementPeriod, selectableSensorId, callback);
     }

src/sensors/devices/device.ts

@@ -1,5 +1,5 @@
-import { IDataTableTimeData } from "../../shared/components/data-table-field";
-import { ISensorCapabilities, ISensorValues, SensorCapabilityKey, AllCapabilityKeys, ITimeSeriesCapabilities } from "../sensor";
+import { ISensorCapabilities, ISensorValues, SensorCapabilityKey, AllCapabilityKeys } from "../sensor";
+import { ITimeSeriesCapabilities } from "../../shared/utils/time-series";
 
 export interface IDeviceOptions {
   name: string;
@@ -51,7 +51,7 @@ export class Device {
     return []; // set in each device
   }
 
-  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: IDataTableTimeData[]) => void): () => void {
+  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: number[]) => void): () => void {
     throw new Error("collectTimeSeries() method not overridden!");
   }
 

src/sensors/devices/gdx-sensor.ts

@@ -1,7 +1,7 @@
 import { Device, ISelectableSensorInfo } from "./device";
 import godirect from "@vernier/godirect/dist/godirect.min.cjs";
-import { ISensorCapabilities, ISensorValues, ITimeSeriesCapabilities } from "../sensor";
-import { IDataTableTimeData } from "../../shared/components/data-table-field";
+import { ISensorCapabilities, ISensorValues } from "../sensor";
+import { ITimeSeriesCapabilities } from "../../shared/utils/time-series";
 
 const goDirectServiceUUID = "d91714ef-28b9-4f91-ba16-f0d9a604f112";
 
@@ -76,10 +76,8 @@ export class GDXSensorDevice extends Device {
     return this.gdxDevice.sensors.map((s: any) => ({name: s.name, internalId: s.number}));
   }
 
-  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: IDataTableTimeData[]) => void): () => void {
-    let capabilities = this.timeSeriesCapabilities(selectableSensorId);
-
-    if (!this.gdxDevice || !capabilities) {
+  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: number[]) => void): () => void {
+    if (!this.gdxDevice) {
       return () => {
         // noop
       };
@@ -89,16 +87,11 @@ export class GDXSensorDevice extends Device {
 
     let time = 0;
     const delta = measurementPeriod / 1000;
-    const values: IDataTableTimeData[] = [];
-    capabilities = {...capabilities, measurementPeriod};
+    const values: number[] = [];
 
     const handleChange = (sensor: any) => {
       if (sensor.number === selectableSensorId) {
-        if (values.length === 0) {
-          values.push({time, value: sensor.value, capabilities});
-        } else {
-          values.push({time, value: sensor.value});
-        }
+        values.push(sensor.value);
         callback(values);
         time += delta;
       }

src/sensors/mock-sensor.ts

@@ -1,6 +1,7 @@
-import { IDataTableTimeData } from "../shared/components/data-table-field";
+import { TimeSeriesDataKey, getTimeSeriesMetadata } from "../shared/utils/time-series";
 import { ISelectableSensorInfo } from "./devices/device";
-import { Sensor, ISensorOptions, ISensorValues, IPollOptions, IConnectOptions, ITimeSeriesCapabilities } from "./sensor";
+import { Sensor, ISensorOptions, ISensorValues, IPollOptions, IConnectOptions } from "./sensor";
+import { ITimeSeriesCapabilities } from "../shared/utils/time-series";
 
 type MockValueDirection = "up" | "down";
 interface IStartingSensorValues extends Required<ISensorValues> {
@@ -46,19 +47,19 @@ export class MockSensor extends Sensor {
       illuminance: options.minValues?.illuminance || 0,
       temperature: options.minValues?.temperature || -18,
       humidity: options.minValues?.humidity || 0,
-      timeSeries: options.minValues?.timeSeries || -5,
+      timeSeries: options.minValues?.[TimeSeriesDataKey] || -5,
     };
     this.maxMockValues = {
       illuminance: options.maxValues?.illuminance || 10000,
       temperature: options.maxValues?.temperature || 38,
       humidity: options.maxValues?.humidity || 90,
-      timeSeries: options.maxValues?.timeSeries || 5,
+      timeSeries: options.maxValues?.[TimeSeriesDataKey] || 5,
     };
     this.mockValues = {
       illuminance: this.randomInRange("illuminance"),
       temperature: this.randomInRange("temperature"),
       humidity: this.randomInRange("humidity"),
-      timeSeries: this.randomInRange("timeSeries"),
+      timeSeries: this.randomInRange(TimeSeriesDataKey),
       ...options.startingValues,
     };
     this.mockValueDirections = {
@@ -127,8 +128,8 @@ export class MockSensor extends Sensor {
       this.setNextRandomMockValue({measurement: "temperature", increment: 0.2});
     }
 
-    values.timeSeries = this.mockValues.timeSeries;
-    this.setNextRandomMockValue({measurement: "timeSeries", increment: 0.2});
+    values[TimeSeriesDataKey] = this.mockValues[TimeSeriesDataKey];
+    this.setNextRandomMockValue({measurement: TimeSeriesDataKey, increment: 0.2});
 
     return Promise.resolve(values);
   }
@@ -140,7 +141,7 @@ export class MockSensor extends Sensor {
       minMeasurementPeriod: 10,
       defaultMeasurementPeriod,
       measurement: "Fake Value",
-      valueKey: "timeSeries",
+      valueKey: TimeSeriesDataKey,
       units: "N/A",
       minValue: -5,
       maxValue: 5,
@@ -154,22 +155,14 @@ export class MockSensor extends Sensor {
     ];
   }
 
-  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: IDataTableTimeData[]) => void): () => void {
-    let time = 0;
-    const delta = measurementPeriod / 1000;
-    const values: IDataTableTimeData[] = [];
-    const capabilities = {...this.timeSeriesCapabilities(selectableSensorId), measurementPeriod};
+  public collectTimeSeries(measurementPeriod: number, selectableSensorId: any, callback: (values: number[]) => void): () => void {
+    const values: number[] = [];
 
     const callCallback = () => {
-      const value = this.mockValues.timeSeries;
-      this.setNextRandomMockValue({measurement: "timeSeries", increment: 0.5});
-      if (values.length === 0) {
-        values.push({time, value, capabilities});
-      } else {
-        values.push({time, value});
-      }
+      const value = this.mockValues[TimeSeriesDataKey];
+      this.setNextRandomMockValue({measurement: TimeSeriesDataKey, increment: 0.5});
+      values.push(value);
       callback(values);
-      time += delta;
     };
 
     callCallback();