Merge pull request #147 from concord-consortium/187816321-add-csv-dow…

…nload

feat: Add CSV download [PT-187816321]

package.json

@@ -79,6 +79,7 @@
     "@types/jest": "^25.1.0",
     "@types/js-base64": "^2.3.1",
     "@types/jsonwebtoken": "^8.3.7",
+    "@types/papaparse": "^5.3.14",
     "@types/react": "^16.9.19",
     "@types/react-dom": "^16.9.5",
     "@types/react-jsonschema-form": "^1.6.8",
@@ -135,6 +136,7 @@
     "jsonwebtoken": "^8.5.1",
     "jsqr": "^1.2.0",
     "lines-and-columns": "^1.1.6",
+    "papaparse": "^5.4.1",
     "qrcode-svg": "^1.1.0",
     "react": "^16.12.0",
     "react-ace": "^8.0.0",

src/lara-app/components/runtime.module.scss

@@ -76,10 +76,11 @@
   flex-wrap: nowrap;
   flex-direction: row;
   justify-content: flex-end;
+  margin: 10px;
+  gap: 10px;
 
   .button {
     display: inline-block;
-    margin: 10px;
     border: 1px solid $ccTealDark2;
     background-color: #fff;
     color: $ccTealDark2;

src/lara-app/components/runtime.tsx

@@ -1,6 +1,7 @@
 import React, { useState, useEffect, useRef } from "react";
 import * as firebase from "firebase/app";
 import "firebase/firestore";
+
 import { Experiment } from "../../shared/components/experiment";
 import { IExperiment, IExperimentData, IExperimentConfig } from "../../shared/experiment-types";
 import { IRun } from "../../mobile-app/hooks/use-runs";
@@ -13,6 +14,7 @@ const QRCode = require("qrcode-svg");
 import { generateDataset } from "../utils/generate-dataset";
 import css from "./runtime.module.scss";
 import { IJwtClaims } from "@concord-consortium/lara-plugin-api";
+import { downloadCSV } from "../utils/download-csv";
 
 const UPDATE_QR_INTERVAL = 1000 * 60 * 60;  // 60 minutes
 
@@ -151,6 +153,13 @@ export const RuntimeComponent = ({
     displayingCode.current = true;
   };
 
+  const handleDownloadCSV = () => {
+    // not stuttering here...
+    if (experimentData?.experimentData) {
+      downloadCSV(experiment, experimentData);
+    }
+  };
+
   const toggleDisplayQr = () => setDisplayQrAndMaybeRegenerateQR(!displayQr);
 
   const handleSaveData = (data: IExperimentData) => {
@@ -215,6 +224,7 @@ export const RuntimeComponent = ({
         {reportOrPreviewMode ? undefined :
           <div className={css.topBar}>
             <div className={css.button} onClick={handleUploadAgain}>Import</div>
+            {experimentData && <div className={css.button} onClick={handleDownloadCSV}>Download CSV</div>}
           </div>}
         <div className={css.runtimeExperiment}>
           <Experiment

src/lara-app/utils/download-csv.test.ts

@@ -0,0 +1,170 @@
+import { IExperiment, IExperimentData } from "../../shared/experiment-types";
+import { downloadCSV, getFilename, getRows, getTimeKey, sortNaturally } from "./download-csv";
+
+const tabularExperiment: IExperiment = {
+  version: "1.0.0",
+  metadata: {
+    uuid: "59f660d6-0593-4f6a-9d54-35caed536095",
+    name: "Tabular Experiment",
+    initials: "TE",
+  },
+  schema: {
+    dataSchema: {
+      properties: {
+        experimentData: {
+          items: {
+            properties: {
+              foo: {
+                title: "Foo"
+              },
+              bar: {
+                title: "Bar"
+              }
+            }
+          }
+        }
+      }
+    }
+  } as any,
+  data: {}
+};
+const tabularData: IExperimentData = {
+  timestamp: 123456789,
+  experimentData: [
+    {foo: 1, bar: 2},
+    {foo: 2, bar: 3},
+    {foo: "<AVG>", bar: "<AVG>"},
+  ]
+};
+
+const timeSeriesExperiment: IExperiment = {
+  version: "1.0.0",
+  metadata: {
+    uuid: "bfca7fb5-39ea-4238-8515-e469c44aa872",
+    name: "Time Series Experiment",
+    initials: "TS",
+  },
+  schema: {
+    dataSchema: {
+      properties: {
+        experimentData: {
+          items: {
+            properties: {
+              timeSeries: {
+                title: "Results"
+              },
+              label: {
+                title: "Label"
+              }
+            }
+          }
+        }
+      }
+    },
+    formUiSchema: {
+      experimentData: {
+        "ui:dataTableOptions": {
+          sensorFields: ["timeSeries"]
+        }
+      }
+    }
+  } as any,
+  data: {}
+};
+const timeSeriesData: IExperimentData = {
+  timestamp: 987654321,
+  experimentData: [
+    {timeSeries: [{time: 0, value: 1}, {time: 1, value: 2}], label: "Label #1"},
+    {timeSeries: [{time: 1, value: 3}, {time: 2, value: 4}], label: "Label #2"}
+  ]
+};
+
+
+describe("download csv functions", () => {
+
+  describe("getTimeKey()", () => {
+    it("works for integers", () => {
+      expect(getTimeKey(0)).toBe("0");
+    });
+    it("works for floats, up to 3 digits", () => {
+      expect(getTimeKey(0.1)).toBe("0.1");
+      expect(getTimeKey(0.10)).toBe("0.1");
+      expect(getTimeKey(0.100)).toBe("0.1");
+      expect(getTimeKey(0.101)).toBe("0.101");
+      expect(getTimeKey(0.1011)).toBe("0.101");
+      expect(getTimeKey(0.1000000001)).toBe("0.1");
+    });
+  });
+
+  describe("sortNaturally()", () => {
+    it("sorts", () => {
+      expect(sortNaturally(["20", "2", "1", "100", "10", "20.1"])).toStrictEqual(["1", "2", "10", "20", "20.1", "100"]);
+    });
+  });
+
+  describe("getFileName()", () => {
+    it("generates filename", () => {
+      expect(getFilename(tabularExperiment, tabularData)).toBe("tabular-experiment-123456789.csv");
+      expect(getFilename(timeSeriesExperiment, timeSeriesData)).toBe("time-series-experiment-987654321.csv");
+    });
+  });
+
+  describe("getRows()", () => {
+    it("works for tabular data with function symbols", () => {
+      expect(getRows(tabularExperiment, tabularData)).toStrictEqual([
+        {Foo: 1, Bar: 2},
+        {Foo: 2, Bar: 3},
+        {Foo: 1.5, Bar: 2.5},
+      ]);
+    });
+
+    it("works for time series data", () => {
+      expect(getRows(timeSeriesExperiment, timeSeriesData)).toStrictEqual([
+        {
+          "Time": "0",
+          "Row 1 Results": 1,
+          "Row 2 Results": "",
+          "Row 1 Label": "Label #1",
+          "Row 2 Label": "Label #2",
+        },
+        {
+          "Time": "1",
+          "Row 1 Results": 2,
+          "Row 2 Results": 3,
+          "Row 1 Label": "Label #1",
+          "Row 2 Label": "Label #2",
+        },
+        {
+          "Time": "2",
+          "Row 1 Results": "",
+          "Row 2 Results": 4,
+          "Row 1 Label": "Label #1",
+          "Row 2 Label": "Label #2",
+        },
+      ]);
+    });
+  });
+
+  describe("downloadCSV", () => {
+    it("downloads", () => {
+      const createObjectURL = jest.fn();
+      const appendChild = jest.fn();
+      const removeChild = jest.fn();
+
+      const saveUrl = window.URL;
+      const saveBody = document.body;
+      window.URL = { createObjectURL } as any;
+      document.body.appendChild = appendChild;
+      document.body.removeChild = removeChild;
+
+      downloadCSV(tabularExperiment, tabularData);
+
+      expect(createObjectURL).toBeCalledTimes(1);
+      expect(appendChild).toBeCalledTimes(1);
+      expect(removeChild).toBeCalledTimes(1);
+
+      window.URL = saveUrl;
+      document.body = saveBody;
+    });
+  });
+});

src/lara-app/utils/download-csv.ts

@@ -0,0 +1,113 @@
+import {unparse} from "papaparse";
+
+import { IDataTableRow, IDataTableTimeData } from "../../shared/components/data-table-field";
+import { IExperiment, IExperimentData } from "../../shared/experiment-types";
+import { isFunctionSymbol, handleSpecialValue } from "../../shared/utils/handle-special-value";
+
+type TimeValues = Record<string, number|undefined>;
+type OtherValues = Record<string, any>;
+type TimeSeriesRow = {timeValues: TimeValues, otherValues: OtherValues};
+
+export const getTimeKey = (time: number) => time.toFixed(3).replace(/\.?0+$/, "");
+
+const naturalCollator = Intl.Collator(undefined, { numeric: true });
+export const sortNaturally = (arr: string[]): string[] => {
+  arr.sort((a, b) => naturalCollator.compare(a, b));
+  return arr;
+};
+
+export const getRowValue = (key: string, rawRow: IDataTableRow, rawRows: IDataTableRow[]) => {
+  let value: any = rawRow[key] ?? "";
+  if (isFunctionSymbol(value)) {
+    value = handleSpecialValue(value, key, rawRows);
+    if (value === undefined) {
+      value = "";
+    }
+  }
+  return value;
+};
+
+export const getRows = (experiment: IExperiment, data: IExperimentData) => {
+  // get all columns
+  const properties = experiment.schema.dataSchema.properties.experimentData?.items?.properties ?? {};
+  const titleMap = Object.keys(properties).reduce<Record<string,string>>((acc, key) => {
+    acc[key] = properties[key].title ?? key;
+    return acc;
+  }, {});
+
+  const rawRows: IDataTableRow[] = data.experimentData;
+  const isTimeSeries = (experiment.schema.formUiSchema?.experimentData?.["ui:dataTableOptions"]?.sensorFields || []).indexOf("timeSeries") !== -1;
+
+  if (isTimeSeries) {
+    const nonTimeSeriesTitles = Object.keys(titleMap).filter(key => key !== "timeSeries");
+    const timeKeys = new Set<string>();
+    const timeSeriesRows: TimeSeriesRow[] = [];
+
+    rawRows.forEach(rawRow => {
+      const timeValues: TimeValues = {};
+      const otherValues: OtherValues = {};
+
+      nonTimeSeriesTitles.forEach(key => {
+        otherValues[titleMap[key]] = getRowValue(key, rawRow, rawRows);
+      });
+
+      const timeSeries = (rawRow.timeSeries ?? []) as IDataTableTimeData[];
+      timeSeries.forEach(({time, value}) => {
+        const timeKey = getTimeKey(time);
+        timeValues[timeKey] = value;
+        timeKeys.add(timeKey);
+      });
+
+      timeSeriesRows.push({timeValues, otherValues});
+    }, []);
+    const sortedTimeKeys = sortNaturally(Array.from(timeKeys));
+
+    // unroll each time/value reading into its own row
+    const unrolledRows: Record<string,any>[] = [];
+    const timeSeriesKey = titleMap.timeSeries ?? "Time Series Value";
+    sortedTimeKeys.forEach(timeKey => {
+      const row: Record<string, any> = {Time: timeKey};
+      timeSeriesRows.forEach(({timeValues}, rowIndex) => {
+        row[`Row ${rowIndex + 1} ${timeSeriesKey}`] = timeValues[timeKey] ?? "";
+      });
+      timeSeriesRows.forEach(({otherValues}, rowIndex) => {
+        nonTimeSeriesTitles.forEach(key => {
+          const title = titleMap[key];
+          row[`Row ${rowIndex + 1} ${title}`] = otherValues[title] ?? "";
+        });
+      });
+      unrolledRows.push(row);
+    });
+
+    return unrolledRows;
+  }
+
+  // create an empty row with all the experiment columns to ensure it in the same order
+  const emptyRow = Object.values(titleMap).reduce<Record<string,any>>((acc, value) => {
+    acc[value] = "";
+    return acc;
+  }, {});
+
+  return rawRows.map(rawRow => {
+    return Object.keys(rawRow).reduce<Record<string,any>>((acc, key) => {
+      acc[titleMap[key] ?? key] = getRowValue(key, rawRow, rawRows);
+      return acc;
+    }, {...emptyRow});
+  });
+};
+
+export const getFilename = (experiment: IExperiment, data: IExperimentData) => {
+  return `${experiment.metadata.name.trim().toLowerCase().replace(/[^a-zA-Z]/g, "-")}-${data.timestamp}.csv`;
+};
+
+export const downloadCSV = (experiment: IExperiment, data: IExperimentData) => {
+  const rows = getRows(experiment, data);
+  const blob = new Blob([unparse(rows)], { type: 'text/csv' });
+  const link = document.createElement('a');
+
+  link.href = URL.createObjectURL(blob);
+  link.download = getFilename(experiment, data);
+  document.body.appendChild(link);
+  link.click();
+  document.body.removeChild(link);
+};