From 40415a9122872a40bf2ec445d2668d6f9c6a803b Mon Sep 17 00:00:00 2001 From: Doug Martin Date: Fri, 21 Jun 2024 11:09:46 -0400 Subject: [PATCH] feat: Add CSV download [PT-187816321] This adds a csv download button to the LARA/AP app. --- package-lock.json | 14 ++ package.json | 2 + src/lara-app/components/runtime.module.scss | 3 +- src/lara-app/components/runtime.tsx | 10 ++ src/lara-app/utils/download-csv.test.ts | 170 ++++++++++++++++++++ src/lara-app/utils/download-csv.ts | 113 +++++++++++++ 6 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 src/lara-app/utils/download-csv.test.ts create mode 100644 src/lara-app/utils/download-csv.ts diff --git a/package-lock.json b/package-lock.json index db3ff42..fc1749f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4840,6 +4840,15 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-13.1.8.tgz", "integrity": "sha512-6XzyyNM9EKQW4HKuzbo/CkOIjn/evtCmsU+MUM1xDfJ+3/rNjBttM1NgN7AOQvN6tP1Sl1D1PIKMreTArnxM9A==" }, + "@types/papaparse": { + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", + "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", @@ -17043,6 +17052,11 @@ "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==", "dev": true }, + "papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "parallel-transform": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz", diff --git a/package.json b/package.json index 6211625..e004781 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "@types/jest": "^25.1.0", "@types/js-base64": "^2.3.1", "@types/jsonwebtoken": "^8.3.7", + "@types/papaparse": "^5.3.14", "@types/react": "^16.9.19", "@types/react-dom": "^16.9.5", "@types/react-jsonschema-form": "^1.6.8", @@ -135,6 +136,7 @@ "jsonwebtoken": "^8.5.1", "jsqr": "^1.2.0", "lines-and-columns": "^1.1.6", + "papaparse": "^5.4.1", "qrcode-svg": "^1.1.0", "react": "^16.12.0", "react-ace": "^8.0.0", diff --git a/src/lara-app/components/runtime.module.scss b/src/lara-app/components/runtime.module.scss index 6122874..a4f81a5 100644 --- a/src/lara-app/components/runtime.module.scss +++ b/src/lara-app/components/runtime.module.scss @@ -76,10 +76,11 @@ flex-wrap: nowrap; flex-direction: row; justify-content: flex-end; + margin: 10px; + gap: 10px; .button { display: inline-block; - margin: 10px; border: 1px solid $ccTealDark2; background-color: #fff; color: $ccTealDark2; diff --git a/src/lara-app/components/runtime.tsx b/src/lara-app/components/runtime.tsx index a20d254..c80ae3b 100644 --- a/src/lara-app/components/runtime.tsx +++ b/src/lara-app/components/runtime.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect, useRef } from "react"; import * as firebase from "firebase/app"; import "firebase/firestore"; + import { Experiment } from "../../shared/components/experiment"; import { IExperiment, IExperimentData, IExperimentConfig } from "../../shared/experiment-types"; import { IRun } from "../../mobile-app/hooks/use-runs"; @@ -13,6 +14,7 @@ const QRCode = require("qrcode-svg"); import { generateDataset } from "../utils/generate-dataset"; import css from "./runtime.module.scss"; import { IJwtClaims } from "@concord-consortium/lara-plugin-api"; +import { downloadCSV } from "../utils/download-csv"; const UPDATE_QR_INTERVAL = 1000 * 60 * 60; // 60 minutes @@ -151,6 +153,13 @@ export const RuntimeComponent = ({ displayingCode.current = true; }; + const handleDownloadCSV = () => { + // not stuttering here... + if (experimentData?.experimentData) { + downloadCSV(experiment, experimentData); + } + }; + const toggleDisplayQr = () => setDisplayQrAndMaybeRegenerateQR(!displayQr); const handleSaveData = (data: IExperimentData) => { @@ -215,6 +224,7 @@ export const RuntimeComponent = ({ {reportOrPreviewMode ? undefined :
Import
+ {experimentData &&
Download CSV
}
}
", bar: ""}, + ] +}; + +const timeSeriesExperiment: IExperiment = { + version: "1.0.0", + metadata: { + uuid: "bfca7fb5-39ea-4238-8515-e469c44aa872", + name: "Time Series Experiment", + initials: "TS", + }, + schema: { + dataSchema: { + properties: { + experimentData: { + items: { + properties: { + timeSeries: { + title: "Results" + }, + label: { + title: "Label" + } + } + } + } + } + }, + formUiSchema: { + experimentData: { + "ui:dataTableOptions": { + sensorFields: ["timeSeries"] + } + } + } + } as any, + data: {} +}; +const timeSeriesData: IExperimentData = { + timestamp: 987654321, + experimentData: [ + {timeSeries: [{time: 0, value: 1}, {time: 1, value: 2}], label: "Label #1"}, + {timeSeries: [{time: 1, value: 3}, {time: 2, value: 4}], label: "Label #2"} + ] +}; + + +describe("download csv functions", () => { + + describe("getTimeKey()", () => { + it("works for integers", () => { + expect(getTimeKey(0)).toBe("0"); + }); + it("works for floats, up to 3 digits", () => { + expect(getTimeKey(0.1)).toBe("0.1"); + expect(getTimeKey(0.10)).toBe("0.1"); + expect(getTimeKey(0.100)).toBe("0.1"); + expect(getTimeKey(0.101)).toBe("0.101"); + expect(getTimeKey(0.1011)).toBe("0.101"); + expect(getTimeKey(0.1000000001)).toBe("0.1"); + }); + }); + + describe("sortNaturally()", () => { + it("sorts", () => { + expect(sortNaturally(["20", "2", "1", "100", "10", "20.1"])).toStrictEqual(["1", "2", "10", "20", "20.1", "100"]); + }); + }); + + describe("getFileName()", () => { + it("generates filename", () => { + expect(getFilename(tabularExperiment, tabularData)).toBe("tabular-experiment-123456789.csv"); + expect(getFilename(timeSeriesExperiment, timeSeriesData)).toBe("time-series-experiment-987654321.csv"); + }); + }); + + describe("getRows()", () => { + it("works for tabular data with function symbols", () => { + expect(getRows(tabularExperiment, tabularData)).toStrictEqual([ + {Foo: 1, Bar: 2}, + {Foo: 2, Bar: 3}, + {Foo: 1.5, Bar: 2.5}, + ]); + }); + + it("works for time series data", () => { + expect(getRows(timeSeriesExperiment, timeSeriesData)).toStrictEqual([ + { + "Time": "0", + "Row 1 Results": 1, + "Row 2 Results": "", + "Row 1 Label": "Label #1", + "Row 2 Label": "Label #2", + }, + { + "Time": "1", + "Row 1 Results": 2, + "Row 2 Results": 3, + "Row 1 Label": "Label #1", + "Row 2 Label": "Label #2", + }, + { + "Time": "2", + "Row 1 Results": "", + "Row 2 Results": 4, + "Row 1 Label": "Label #1", + "Row 2 Label": "Label #2", + }, + ]); + }); + }); + + describe("downloadCSV", () => { + it("downloads", () => { + const createObjectURL = jest.fn(); + const appendChild = jest.fn(); + const removeChild = jest.fn(); + + const saveUrl = window.URL; + const saveBody = document.body; + window.URL = { createObjectURL } as any; + document.body.appendChild = appendChild; + document.body.removeChild = removeChild; + + downloadCSV(tabularExperiment, tabularData); + + expect(createObjectURL).toBeCalledTimes(1); + expect(appendChild).toBeCalledTimes(1); + expect(removeChild).toBeCalledTimes(1); + + window.URL = saveUrl; + document.body = saveBody; + }); + }); +}); \ No newline at end of file diff --git a/src/lara-app/utils/download-csv.ts b/src/lara-app/utils/download-csv.ts new file mode 100644 index 0000000..f259868 --- /dev/null +++ b/src/lara-app/utils/download-csv.ts @@ -0,0 +1,113 @@ +import {unparse} from "papaparse"; + +import { IDataTableRow, IDataTableTimeData } from "../../shared/components/data-table-field"; +import { IExperiment, IExperimentData } from "../../shared/experiment-types"; +import { isFunctionSymbol, handleSpecialValue } from "../../shared/utils/handle-special-value"; + +type TimeValues = Record; +type OtherValues = Record; +type TimeSeriesRow = {timeValues: TimeValues, otherValues: OtherValues}; + +export const getTimeKey = (time: number) => time.toFixed(3).replace(/\.?0+$/, ""); + +const naturalCollator = Intl.Collator(undefined, { numeric: true }); +export const sortNaturally = (arr: string[]): string[] => { + arr.sort((a, b) => naturalCollator.compare(a, b)); + return arr; +}; + +export const getRowValue = (key: string, rawRow: IDataTableRow, rawRows: IDataTableRow[]) => { + let value: any = rawRow[key] ?? ""; + if (isFunctionSymbol(value)) { + value = handleSpecialValue(value, key, rawRows); + if (value === undefined) { + value = ""; + } + } + return value; +}; + +export const getRows = (experiment: IExperiment, data: IExperimentData) => { + // get all columns + const properties = experiment.schema.dataSchema.properties.experimentData?.items?.properties ?? {}; + const titleMap = Object.keys(properties).reduce>((acc, key) => { + acc[key] = properties[key].title ?? key; + return acc; + }, {}); + + const rawRows: IDataTableRow[] = data.experimentData; + const isTimeSeries = (experiment.schema.formUiSchema?.experimentData?.["ui:dataTableOptions"]?.sensorFields || []).indexOf("timeSeries") !== -1; + + if (isTimeSeries) { + const nonTimeSeriesTitles = Object.keys(titleMap).filter(key => key !== "timeSeries"); + const timeKeys = new Set(); + const timeSeriesRows: TimeSeriesRow[] = []; + + rawRows.forEach(rawRow => { + const timeValues: TimeValues = {}; + const otherValues: OtherValues = {}; + + nonTimeSeriesTitles.forEach(key => { + otherValues[titleMap[key]] = getRowValue(key, rawRow, rawRows); + }); + + const timeSeries = (rawRow.timeSeries ?? []) as IDataTableTimeData[]; + timeSeries.forEach(({time, value}) => { + const timeKey = getTimeKey(time); + timeValues[timeKey] = value; + timeKeys.add(timeKey); + }); + + timeSeriesRows.push({timeValues, otherValues}); + }, []); + const sortedTimeKeys = sortNaturally(Array.from(timeKeys)); + + // unroll each time/value reading into its own row + const unrolledRows: Record[] = []; + const timeSeriesKey = titleMap.timeSeries ?? "Time Series Value"; + sortedTimeKeys.forEach(timeKey => { + const row: Record = {Time: timeKey}; + timeSeriesRows.forEach(({timeValues}, rowIndex) => { + row[`Row ${rowIndex + 1} ${timeSeriesKey}`] = timeValues[timeKey] ?? ""; + }); + timeSeriesRows.forEach(({otherValues}, rowIndex) => { + nonTimeSeriesTitles.forEach(key => { + const title = titleMap[key]; + row[`Row ${rowIndex + 1} ${title}`] = otherValues[title] ?? ""; + }); + }); + unrolledRows.push(row); + }); + + return unrolledRows; + } + + // create an empty row with all the experiment columns to ensure it in the same order + const emptyRow = Object.values(titleMap).reduce>((acc, value) => { + acc[value] = ""; + return acc; + }, {}); + + return rawRows.map(rawRow => { + return Object.keys(rawRow).reduce>((acc, key) => { + acc[titleMap[key] ?? key] = getRowValue(key, rawRow, rawRows); + return acc; + }, {...emptyRow}); + }); +}; + +export const getFilename = (experiment: IExperiment, data: IExperimentData) => { + return `${experiment.metadata.name.trim().toLowerCase().replace(/[^a-zA-Z]/g, "-")}-${data.timestamp}.csv`; +}; + +export const downloadCSV = (experiment: IExperiment, data: IExperimentData) => { + const rows = getRows(experiment, data); + const blob = new Blob([unparse(rows)], { type: 'text/csv' }); + const link = document.createElement('a'); + + link.href = URL.createObjectURL(blob); + link.download = getFilename(experiment, data); + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); +}; \ No newline at end of file