From 5149f9590339ff75470c7915a2a0029fc6fe4e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 14:04:54 -0300 Subject: [PATCH 1/7] Adds some CLI tests --- app/cli/deno.jsonc | 2 +- app/cli/src/handlers/defaults/mod.ts | 6 +- app/cli/src/handlers/defaults/utils.ts | 8 +- .../src/handlers/plugins/default-parser.ts | 36 ++++- app/cli/src/handlers/plugins/handler/utils.ts | 2 + app/cli/src/handlers/plugins/registry.ts | 4 +- app/cli/test/fixtures/example_data.csv | 21 +++ app/cli/test/main.test.ts | 130 ++++++++++++++++-- app/cli/test/supports/mod.ts | 17 +++ .../standard/src/manifest/data/tabular/row.ts | 3 +- .../src/manifest/data/tabular/value.ts | 3 +- 11 files changed, 202 insertions(+), 30 deletions(-) create mode 100644 app/cli/test/fixtures/example_data.csv create mode 100644 app/cli/test/supports/mod.ts diff --git a/app/cli/deno.jsonc b/app/cli/deno.jsonc index ef42bb43..3d6170ec 100644 --- a/app/cli/deno.jsonc +++ b/app/cli/deno.jsonc @@ -4,7 +4,7 @@ "compile:deb": "deno task bundle && bash ./compile.sh && rm -rf ./tmp ./bin", "bundle": "mkdir -p bin && deno bundle src/mod.ts bin/run", "fsml": "deno run --allow-net --allow-read --allow-env --allow-write --allow-run src/mod.ts", - "test": "deno test --coverage=./cov_profile/ --allow-read --allow-env", + "test": "deno test --coverage=./cov_profile/ --allow-read --allow-env --allow-net --allow-write", "coverage": "deno coverage ./cov_profile --lcov --output=cov_profile.lcov" }, "importMap": "../../import_map.json", diff --git a/app/cli/src/handlers/defaults/mod.ts b/app/cli/src/handlers/defaults/mod.ts index aece7122..18e48d49 100644 --- a/app/cli/src/handlers/defaults/mod.ts +++ b/app/cli/src/handlers/defaults/mod.ts @@ -8,7 +8,7 @@ import { saveConfigs, } from "./utils.ts"; -/** CLI "defaults" commmand handlers **/ +/** CLI "defaults" command handlers **/ async function edit(args: Arguments) { const { section } = args; const configs = await getConfigs(); @@ -22,11 +22,11 @@ async function list(args: Arguments) { const configs = await getConfigs(); const _format = format || configs?.defaults?.format; - const stdout_text: string = await jsonToText({ + const stdout_text: string = jsonToText({ format: _format, content: configs, }); - await toStdOut(stdout_text); + toStdOut(stdout_text); } async function set(args: Arguments) { diff --git a/app/cli/src/handlers/defaults/utils.ts b/app/cli/src/handlers/defaults/utils.ts index 8eae9d41..ce9c8011 100644 --- a/app/cli/src/handlers/defaults/utils.ts +++ b/app/cli/src/handlers/defaults/utils.ts @@ -9,7 +9,7 @@ import { } from "@fsml/packages/utils/mod.ts"; import { Configs, TConfigs, TConfigValue } from "@fsml/cli/types/configs.ts"; -const DEFAULT_CONFIGS = { +export const DEFAULT_CONFIGS = { "defaults": { "filepath": "./configs.yaml", "format": "yaml" }, "manifest": { "author": null, @@ -37,7 +37,7 @@ function getDefaultConfigs() { async function getConfigs({ section }: { section?: string } = {}) { const defaultConfigs = getDefaultConfigs(); fs.ensureFileSync(USER_CONFIG_FILEPATH); - const configsText = await read(USER_CONFIG_FILEPATH); + const configsText = read(USER_CONFIG_FILEPATH); const configs = await yaml.parse(configsText); @@ -48,12 +48,12 @@ async function getConfigs({ section }: { section?: string } = {}) { return finalConfigs; } -async function saveConfigs( +function saveConfigs( newConfigs: Partial, ) { if (validateConfigs(newConfigs)) { const newConfigTextFile = yaml.stringify(newConfigs); - await toFile({ + toFile({ filepath: USER_CONFIG_FILEPATH, content: newConfigTextFile, }); diff --git a/app/cli/src/handlers/plugins/default-parser.ts b/app/cli/src/handlers/plugins/default-parser.ts index 39f60568..08cdbdcf 100644 --- a/app/cli/src/handlers/plugins/default-parser.ts +++ b/app/cli/src/handlers/plugins/default-parser.ts @@ -2,7 +2,7 @@ * whatever interface we end up designing for data parsers. **/ import { IParser, PluginTypes } from "@fsml/packages/plugins/types.ts"; import { set } from "@fsml/packages/utils/deps/lodash.ts"; -import { createTemplateForType } from "@fsml/packages/utils/mod.ts"; +import { createTemplateForType, read } from "@fsml/packages/utils/mod.ts"; import { TabularData, TTabularData, @@ -15,6 +15,8 @@ import { TColumn, TKind, } from "@fsml/packages/standard/manifest/data/tabular/column/mod.ts"; +import papaparse from "https://esm.sh/papaparse@5.3.2"; +import { TValue } from "../../../../../packages/standard/src/manifest/data/tabular/value.ts"; // TODO: This one shall be extended so that it becomes // an actually useful default parser by somehow auto-detecting @@ -24,8 +26,9 @@ import { const DefaultDataParser: IParser = { name: "defaultDataParser", type: PluginTypes.PARSER, - run: async (filepath) => { - console.info(`Parsing file '${filepath}'...`); + run: async (file) => { + console.info(`Parsing file '${file}'...`); + const tabularDataObject = createTemplateForType(TabularData); const columnObject = createTemplateForType(Column); const kindObject = createTemplateForType(Kind); @@ -36,13 +39,32 @@ const DefaultDataParser: IParser = { set(columnObject, "kind", kindObject); set(tabularDataObject, "column", columnObject); + const data = typeof file === "string" ? read(file) : file.toString(); + const dataRows = papaparse.parse(data).data; + dataRows.forEach((csvRow, rowIndex) => { + const values: TValue[] = []; + csvRow.forEach((value, columnIndex) => { + values.push({ + index: columnIndex, + value, + }); + tabularDataObject.rows.push({ + index: rowIndex, + values, + }); + }); + }); + // TODO: write data object to file. return await Promise.resolve({ data: tabularDataObject }); }, - - isApplicable: async (filepath) => { - console.info(`Checking if ${filepath} can be parsed...`); - return await Promise.resolve(true); + isApplicable: async (file) => { + console.info(`Checking if file can be parsed...`); + if (typeof file === "string" && file.endsWith(".csv")) { + console.info(`File suitable for the defaultDataParser...`); + return await Promise.resolve(true); + } + return await Promise.resolve(false); }, }; diff --git a/app/cli/src/handlers/plugins/handler/utils.ts b/app/cli/src/handlers/plugins/handler/utils.ts index b4201856..4034a5ce 100644 --- a/app/cli/src/handlers/plugins/handler/utils.ts +++ b/app/cli/src/handlers/plugins/handler/utils.ts @@ -33,6 +33,8 @@ const defaultVersionResolver = async ( version = match[1]; } } + // Just close response read stream. We don't need to read its body. + await resp.body?.cancel(); return version; }; diff --git a/app/cli/src/handlers/plugins/registry.ts b/app/cli/src/handlers/plugins/registry.ts index 5e0ae2f4..c5dd0787 100644 --- a/app/cli/src/handlers/plugins/registry.ts +++ b/app/cli/src/handlers/plugins/registry.ts @@ -37,7 +37,7 @@ async function addModuleToRegistry( pluginsRegistry, ); - await toFile({ + toFile({ filepath: PLUGINS_REGISTRY_FILEPATH, content: pluginsRegistryString_updated, }); @@ -75,7 +75,7 @@ async function removeModuleFromRegistry( async function getPluginRegistry(): Promise { fs.ensureFileSync(PLUGINS_REGISTRY_FILEPATH); - const pluginsRegistryString = await read(PLUGINS_REGISTRY_FILEPATH); + const pluginsRegistryString = read(PLUGINS_REGISTRY_FILEPATH); const pluginsRegistry = (await yaml.parse(pluginsRegistryString) as TPluginsRegistry) || PLUGIN_REGISTRY_TEMPLATE; diff --git a/app/cli/test/fixtures/example_data.csv b/app/cli/test/fixtures/example_data.csv new file mode 100644 index 00000000..099b4901 --- /dev/null +++ b/app/cli/test/fixtures/example_data.csv @@ -0,0 +1,21 @@ +Clone,Time,Time Units,Acetone Concentration ,Acetone Concentration Unit,D-Glucose ,D-Glucose Unit +C_1,1,hrs,0.49279369,g/L,0.60864757,g/L +C_2,2,hrs,2.46021607,g/L,1.75225928,g/L +C_3,3,hrs,1.887799053,g/L,1.06535886,g/L +C_4,4,hrs,0.36752897,g/L,0.75896524,g/L +C_5,5,hrs,2.287611982,g/L,1.54513438,g/L +C_6,6,hrs,1.145001585,g/L,0.1740019,g/L +C_7,7,hrs,1.249359273,g/L,0.29923113,g/L +C_8,8,hrs,0.770046661,g/L,0.27594401,g/L +C_9,9,hrs,1.444788622,g/L,0.53374635,g/L +C_10,10,hrs,1.599680583,g/L,0.7196167,g/L +C_11,11,hrs,1.163177704,g/L,0.19581324,g/L +C_12,12,hrs,1.51270074,g/L,0.61524089,g/L +C_13,13,hrs,1.183822988,g/L,0.22058759,g/L +C_14,14,hrs,1.397965591,g/L,0.47755871,g/L +C_15,15,hrs,1.342762838,g/L,0.41131541,g/L +C_16,16,hrs,0.715611165,g/L,0.3412666,g/L +C_17,17,hrs,0.681662316,g/L,0.38200522,g/L +C_18,18,hrs,2.287762838,g/L,1.54531541,g/L +C_19,19,hrs,0.669134432,g/L,0.39703868,g/L +C_20,20,hrs,1.476763052,g/L,0.57211566,g/L \ No newline at end of file diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index a5b66b9f..fc268d94 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -1,14 +1,122 @@ +import { + afterAll, + beforeAll, + describe, + it, +} from "https://deno.land/std@0.166.0/testing/bdd.ts"; import { assertEquals } from "https://deno.land/std@0.148.0/testing/asserts.ts"; +import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; +import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; +import get from "https://deno.land/x/lodash@4.17.15-es/get.js"; +import { generateManifest } from "../src/handlers/manifest/utils.ts"; +import { ManifestTypes } from "../src/types/enums.ts"; +import { assertExists } from "https://deno.land/std/testing/asserts.ts"; +import { fixturePath } from "./supports/mod.ts"; +import { install, uninstall, upgrade } from "../src/handlers/plugins/mod.ts"; +import { getRegisteredModule } from "../src/handlers/plugins/registry.ts"; +import { Arguments } from "../src/deps/yargs.ts"; -function helperFn(msg: string): string { - const full = "hello " + msg; - console.log(full); - return full; -} - -Deno.test("url test", () => { - const res = helperFn("world"); - assertEquals(res, "hello world"); - const url = new URL("./foo.js", "https://deno.land/"); - assertEquals(url.href, "https://deno.land/foo.js"); +describe("defaults list", async () => { + const allSections = await getConfigs(); + const manifestSection = await getConfigs({ section: "manifest" }); + + const expectedDefaultSections = Object.keys(DEFAULT_CONFIGS); + const defaultSections = Object.keys(allSections); + assertEquals(difference(defaultSections, expectedDefaultSections).length, 0); + + const expectedManifestDefaults = Object.keys(DEFAULT_CONFIGS.manifest); + const manifestDefaults = Object.keys(manifestSection); + assertEquals( + difference(manifestDefaults, expectedManifestDefaults).length, + 0, + ); +}); + +describe("manifest create", async () => { + const fixtureFilepath = fixturePath("example_data.csv"); + + const manifest = await generateManifest({ + parser: "", + type: ManifestTypes.DATA, + filepattern: fixtureFilepath, + author: "deno-test", + }); + + assertExists(manifest); + + const rows = get(manifest, "supplementalInfo.data[0].rows"); + + assertEquals(rows[0].values.length, 7); + assertEquals(rows.length, 147); +}); + +describe("plugin commands", () => { + const MODULE_NAME = "lodash"; + const MODULE_VERSION_3 = "3.9.0"; + const MODULE_VERSION_3_PATCH = "3.9.3"; + const MODULE_VERSION_3_MINOR = "3.10.1"; + const MODULE_VERSION_3_MAJOR = "4.17.21"; + + async function cleanTestPlugin() { + await uninstall({ module: MODULE_NAME, _: [] }); + } + + beforeAll(cleanTestPlugin); + afterAll(cleanTestPlugin); + + it("install", async () => { + const args = { + module: `${MODULE_NAME}@${MODULE_VERSION_3}`, + _: [], + }; + await install(args); + + const pluginModule = await getRegisteredModule({ name: MODULE_NAME }); + + assertExists(pluginModule); + assertEquals(pluginModule.name, MODULE_NAME); + assertEquals(pluginModule.version, MODULE_VERSION_3); + assertEquals(pluginModule.uriScheme, "https"); + }); + + it("upgrade patch", async () => { + const args: Arguments = { + module: MODULE_NAME, + _: [], + }; + // Patch + args.patch = true; + await upgrade(args); + let pluginModule = await getRegisteredModule({ name: MODULE_NAME }); + assertEquals(pluginModule.version, MODULE_VERSION_3_PATCH); + + // Minor + args.patch = false; + args.minor = true; + await upgrade(args); + pluginModule = await getRegisteredModule({ name: MODULE_NAME }); + assertEquals(pluginModule.version, MODULE_VERSION_3_MINOR); + + // Major + args.minor = false; + args.major = true; + await upgrade(args); + pluginModule = await getRegisteredModule({ name: MODULE_NAME }); + assertEquals( + pluginModule.version.split(".")[0], + MODULE_VERSION_3_MAJOR.split(".")[0], + ); + }); + + it("uninstall", async () => { + const args = { + module: MODULE_NAME, + _: [], + }; + await uninstall(args); + + const pluginModule = await getRegisteredModule({ name: MODULE_NAME }); + + assertEquals(pluginModule, undefined); + }); }); diff --git a/app/cli/test/supports/mod.ts b/app/cli/test/supports/mod.ts new file mode 100644 index 00000000..cdbe78d5 --- /dev/null +++ b/app/cli/test/supports/mod.ts @@ -0,0 +1,17 @@ +import { path } from "../../src/deps/mod.ts"; + +export * as path from "https://deno.land/std@0.149.0/path/mod.ts"; + +const TEST_DIR = "test"; +const TEST_FIXTURE_DIR = "fixtures"; + +export function fixturePath(filename: string) { + const cwd = Deno.cwd(); + const absolute_filepath = path.join( + cwd, + TEST_DIR, + TEST_FIXTURE_DIR, + filename, + ); + return absolute_filepath; +} diff --git a/packages/standard/src/manifest/data/tabular/row.ts b/packages/standard/src/manifest/data/tabular/row.ts index ec60c4d6..d662e7ea 100644 --- a/packages/standard/src/manifest/data/tabular/row.ts +++ b/packages/standard/src/manifest/data/tabular/row.ts @@ -1,4 +1,4 @@ -import { Type } from "@fsml/packages/utils/deps/typebox.ts"; +import { Static, Type } from "@fsml/packages/utils/deps/typebox.ts"; import Value from './value.ts'; // NOTE: we might want to add a property specific to the row @@ -23,4 +23,5 @@ const Row = Type.Object({ fileReference: Type.Optional(Type.String()), }); +export type TRow = Static; export default Row; diff --git a/packages/standard/src/manifest/data/tabular/value.ts b/packages/standard/src/manifest/data/tabular/value.ts index 69457160..9854da48 100644 --- a/packages/standard/src/manifest/data/tabular/value.ts +++ b/packages/standard/src/manifest/data/tabular/value.ts @@ -1,4 +1,4 @@ -import { Type } from "@fsml/packages/utils/deps/typebox.ts"; +import { Static, Type } from "@fsml/packages/utils/deps/typebox.ts"; const Value = Type.Object({ /** @@ -13,4 +13,5 @@ const Value = Type.Object({ // unit: Type.Optional(Unit) }); +export type TValue = Static; export default Value; From b577a271036e811cd1ca4678c26db0d3ec5a84e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 14:10:40 -0300 Subject: [PATCH 2/7] attempt to fix await test --- app/cli/test/main.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index fc268d94..9b2d050e 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -61,8 +61,8 @@ describe("plugin commands", () => { await uninstall({ module: MODULE_NAME, _: [] }); } - beforeAll(cleanTestPlugin); - afterAll(cleanTestPlugin); + beforeAll(async () => await cleanTestPlugin()); + afterAll(async () => await cleanTestPlugin()); it("install", async () => { const args = { From bfcea30573d15b3c38b050927a1155b1cc1bebbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 14:17:37 -0300 Subject: [PATCH 3/7] checking test --- app/cli/test/main.test.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index 9b2d050e..12e920ca 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -16,21 +16,21 @@ import { install, uninstall, upgrade } from "../src/handlers/plugins/mod.ts"; import { getRegisteredModule } from "../src/handlers/plugins/registry.ts"; import { Arguments } from "../src/deps/yargs.ts"; -describe("defaults list", async () => { - const allSections = await getConfigs(); - const manifestSection = await getConfigs({ section: "manifest" }); - - const expectedDefaultSections = Object.keys(DEFAULT_CONFIGS); - const defaultSections = Object.keys(allSections); - assertEquals(difference(defaultSections, expectedDefaultSections).length, 0); - - const expectedManifestDefaults = Object.keys(DEFAULT_CONFIGS.manifest); - const manifestDefaults = Object.keys(manifestSection); - assertEquals( - difference(manifestDefaults, expectedManifestDefaults).length, - 0, - ); -}); +// describe("defaults list", async () => { +// const allSections = await getConfigs(); +// const manifestSection = await getConfigs({ section: "manifest" }); + +// const expectedDefaultSections = Object.keys(DEFAULT_CONFIGS); +// const defaultSections = Object.keys(allSections); +// assertEquals(difference(defaultSections, expectedDefaultSections).length, 0); + +// const expectedManifestDefaults = Object.keys(DEFAULT_CONFIGS.manifest); +// const manifestDefaults = Object.keys(manifestSection); +// assertEquals( +// difference(manifestDefaults, expectedManifestDefaults).length, +// 0, +// ); +// }); describe("manifest create", async () => { const fixtureFilepath = fixturePath("example_data.csv"); From eb8ce4dc0755f4bd2b1141aecb5e121671ed9ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 14:18:55 -0300 Subject: [PATCH 4/7] checking test --- app/cli/test/main.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index 12e920ca..ad140443 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -5,8 +5,8 @@ import { it, } from "https://deno.land/std@0.166.0/testing/bdd.ts"; import { assertEquals } from "https://deno.land/std@0.148.0/testing/asserts.ts"; -import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; -import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; +// import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; +// import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; import get from "https://deno.land/x/lodash@4.17.15-es/get.js"; import { generateManifest } from "../src/handlers/manifest/utils.ts"; import { ManifestTypes } from "../src/types/enums.ts"; From 8543718adbc45801b8a0ad6ae41197b655c7e15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 14:21:04 -0300 Subject: [PATCH 5/7] checking test --- app/cli/test/main.test.ts | 67 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index ad140443..1a4b7aa6 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -5,8 +5,8 @@ import { it, } from "https://deno.land/std@0.166.0/testing/bdd.ts"; import { assertEquals } from "https://deno.land/std@0.148.0/testing/asserts.ts"; -// import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; -// import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; +import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; +import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; import get from "https://deno.land/x/lodash@4.17.15-es/get.js"; import { generateManifest } from "../src/handlers/manifest/utils.ts"; import { ManifestTypes } from "../src/types/enums.ts"; @@ -16,38 +16,45 @@ import { install, uninstall, upgrade } from "../src/handlers/plugins/mod.ts"; import { getRegisteredModule } from "../src/handlers/plugins/registry.ts"; import { Arguments } from "../src/deps/yargs.ts"; -// describe("defaults list", async () => { -// const allSections = await getConfigs(); -// const manifestSection = await getConfigs({ section: "manifest" }); - -// const expectedDefaultSections = Object.keys(DEFAULT_CONFIGS); -// const defaultSections = Object.keys(allSections); -// assertEquals(difference(defaultSections, expectedDefaultSections).length, 0); - -// const expectedManifestDefaults = Object.keys(DEFAULT_CONFIGS.manifest); -// const manifestDefaults = Object.keys(manifestSection); -// assertEquals( -// difference(manifestDefaults, expectedManifestDefaults).length, -// 0, -// ); -// }); - -describe("manifest create", async () => { - const fixtureFilepath = fixturePath("example_data.csv"); - - const manifest = await generateManifest({ - parser: "", - type: ManifestTypes.DATA, - filepattern: fixtureFilepath, - author: "deno-test", +describe("defaults commands", () => { + it("list", async () => { + const allSections = await getConfigs(); + const manifestSection = await getConfigs({ section: "manifest" }); + + const expectedDefaultSections = Object.keys(DEFAULT_CONFIGS); + const defaultSections = Object.keys(allSections); + assertEquals( + difference(defaultSections, expectedDefaultSections).length, + 0, + ); + + const expectedManifestDefaults = Object.keys(DEFAULT_CONFIGS.manifest); + const manifestDefaults = Object.keys(manifestSection); + assertEquals( + difference(manifestDefaults, expectedManifestDefaults).length, + 0, + ); }); +}); + +describe("manifest create", () => { + it("generate", async () => { + const fixtureFilepath = fixturePath("example_data.csv"); - assertExists(manifest); + const manifest = await generateManifest({ + parser: "", + type: ManifestTypes.DATA, + filepattern: fixtureFilepath, + author: "deno-test", + }); - const rows = get(manifest, "supplementalInfo.data[0].rows"); + assertExists(manifest); - assertEquals(rows[0].values.length, 7); - assertEquals(rows.length, 147); + const rows = get(manifest, "supplementalInfo.data[0].rows"); + + assertEquals(rows[0].values.length, 7); + assertEquals(rows.length, 147); + }); }); describe("plugin commands", () => { From 296bc09de5a2c0f07d43694058480c6f9ee2e2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Mon, 28 Nov 2022 17:33:26 -0300 Subject: [PATCH 6/7] cli 'manifest validate' implementation & test --- app/cli/src/commands/manifest/mod.ts | 3 +- app/cli/src/commands/manifest/validate.ts | 16 +++++ app/cli/src/handlers/defaults/mod.ts | 2 +- app/cli/src/handlers/defaults/utils.ts | 24 +++---- app/cli/src/handlers/manifest/generator.ts | 2 +- app/cli/src/handlers/manifest/mod.ts | 27 ++++++-- app/cli/src/handlers/manifest/utils.ts | 10 +-- app/cli/test/main.test.ts | 78 +++++++++++++++------- app/cli/test/supports/mod.ts | 9 +++ packages/utils/src/deps/lodash.ts | 2 + packages/utils/src/deps/typebox.ts | 7 +- packages/utils/src/mod.ts | 60 +++++++++++++++-- 12 files changed, 181 insertions(+), 59 deletions(-) create mode 100644 app/cli/src/commands/manifest/validate.ts diff --git a/app/cli/src/commands/manifest/mod.ts b/app/cli/src/commands/manifest/mod.ts index 92899224..0a84cd6b 100644 --- a/app/cli/src/commands/manifest/mod.ts +++ b/app/cli/src/commands/manifest/mod.ts @@ -1,9 +1,10 @@ import create from "./create.ts"; import _import from "./import.ts"; +import validate from "./validate.ts"; import { commandFactory } from "../utils.ts"; export default commandFactory({ command: "manifest ", describe: "Operates with the FSML manifest", - subCommands: [create, _import], + subCommands: [create, _import, validate], }); diff --git a/app/cli/src/commands/manifest/validate.ts b/app/cli/src/commands/manifest/validate.ts new file mode 100644 index 00000000..87e83741 --- /dev/null +++ b/app/cli/src/commands/manifest/validate.ts @@ -0,0 +1,16 @@ +import { Yargs } from "@fsml/cli/deps/yargs.ts"; +import { validate } from "@fsml/cli/handlers/manifest/mod.ts"; + +function builder(yargs: Yargs) { + yargs.positional("filepath", { + describe: "FSML Manifest absolute file path", + type: "string", + }); +} + +export default { + command: "validate ", + describe: "Validate an FSML manifest", + builder, + handler: validate, +}; diff --git a/app/cli/src/handlers/defaults/mod.ts b/app/cli/src/handlers/defaults/mod.ts index 18e48d49..ebfd608a 100644 --- a/app/cli/src/handlers/defaults/mod.ts +++ b/app/cli/src/handlers/defaults/mod.ts @@ -33,7 +33,7 @@ async function set(args: Arguments) { const { key, value } = args; const configs = await getConfigs(); _set(configs, key, parseConfigValue(value)); - await saveConfigs(configs); + saveConfigs(configs); } async function reset(args: Arguments) { diff --git a/app/cli/src/handlers/defaults/utils.ts b/app/cli/src/handlers/defaults/utils.ts index ce9c8011..58093bc7 100644 --- a/app/cli/src/handlers/defaults/utils.ts +++ b/app/cli/src/handlers/defaults/utils.ts @@ -1,11 +1,12 @@ -import { fs, path, yaml } from "@fsml/cli/deps/mod.ts"; +import { fs, path } from "@fsml/cli/deps/mod.ts"; import { merge, set } from "@fsml/packages/utils/deps/lodash.ts"; -import { TypeCompiler } from "@fsml/packages/utils/deps/typebox.ts"; import { jsonToText, read, + textToJson, toFile, toStdOut, + validateType, } from "@fsml/packages/utils/mod.ts"; import { Configs, TConfigs, TConfigValue } from "@fsml/cli/types/configs.ts"; @@ -39,7 +40,7 @@ async function getConfigs({ section }: { section?: string } = {}) { fs.ensureFileSync(USER_CONFIG_FILEPATH); const configsText = read(USER_CONFIG_FILEPATH); - const configs = await yaml.parse(configsText); + const configs = await textToJson({ format: "yaml", text: configsText }); const finalConfigs = merge(defaultConfigs, configs || {}); @@ -52,7 +53,10 @@ function saveConfigs( newConfigs: Partial, ) { if (validateConfigs(newConfigs)) { - const newConfigTextFile = yaml.stringify(newConfigs); + const newConfigTextFile = jsonToText({ + format: "yaml", + content: newConfigs, + }); toFile({ filepath: USER_CONFIG_FILEPATH, content: newConfigTextFile, @@ -101,16 +105,10 @@ function parseConfigValue(value: string | null) { } function validateConfigs(configs: Partial) { - //@ts-ignore:next-line : This seems like an issue with typebox types. - const ConfigsCompiler = TypeCompiler.Compile(Configs); - const isValid = ConfigsCompiler.Check(configs); - // NOTE: these config error messages may be too verbose - // typebox documentation also references ajv for validation: https://deno.land/x/typebox@0.24.27#validation - const configsErrors = [...ConfigsCompiler.Errors(configs)]; + const { isValid, errors } = validateType(Configs, configs); + if (!isValid) { - configsErrors.forEach((error) => - toStdOut(jsonToText({ format: "json", content: error })) - ); + toStdOut(JSON.stringify(errors, null, 2)); } return isValid; } diff --git a/app/cli/src/handlers/manifest/generator.ts b/app/cli/src/handlers/manifest/generator.ts index 19cd260e..fb0a14af 100644 --- a/app/cli/src/handlers/manifest/generator.ts +++ b/app/cli/src/handlers/manifest/generator.ts @@ -94,7 +94,7 @@ const ManifestGenerator = ( async function generate(args: { author: string; filepath: string; - parser: string; + parser?: string; }): Promise { const { author: _author, filepath, parser } = args; const provenanceObject = author(_author); diff --git a/app/cli/src/handlers/manifest/mod.ts b/app/cli/src/handlers/manifest/mod.ts index 4759e3c0..3ddf4c36 100644 --- a/app/cli/src/handlers/manifest/mod.ts +++ b/app/cli/src/handlers/manifest/mod.ts @@ -1,6 +1,17 @@ import { Arguments } from "@fsml/cli/deps/yargs.ts"; -import { jsonToText, remove, toStdOut } from "@fsml/packages/utils/mod.ts"; -import { generateManifest, packManifest, writeManifest } from "./utils.ts"; +import { + jsonToText, + read, + remove, + textToJson, + toStdOut, +} from "@fsml/packages/utils/mod.ts"; +import { + generateManifest, + packManifest, + validateManifest, + writeManifest, +} from "./utils.ts"; /** CLI "manifest" command handlers **/ @@ -49,7 +60,15 @@ async function _import() {} // async function _export(args: Arguments) {} -// async function validate(args: Arguments) {} +async function validate(args: Arguments) { + const { filepath } = args; + + const manifestString = read(filepath); + + const manifest = await textToJson({ format: "json", text: manifestString }); + + return validateManifest(manifest); +} // async function pack(args: Arguments) {} @@ -65,5 +84,5 @@ export { // score, // unpack, // update, - // validate, + validate, }; diff --git a/app/cli/src/handlers/manifest/utils.ts b/app/cli/src/handlers/manifest/utils.ts index 2afe7282..e3fa7bc6 100644 --- a/app/cli/src/handlers/manifest/utils.ts +++ b/app/cli/src/handlers/manifest/utils.ts @@ -119,16 +119,10 @@ async function getDataFilepath( return { filepath: datafilepath, isPack }; } -function validateManifest(manifest: TManifest): boolean { +export function validateManifest(manifest: TManifest): boolean { const { isValid, errors } = validateType(Manifest, manifest); if (!isValid) { - toStdOut("Error in Manifest: \n"); - // TypeBox's TypeCompiler errors are quite verbosy and the escalate upwards the JSON tree. - // so if the error is located at a given leaf in the JSON tree, additional errors upwards the tree - // will be generated. - - // For this reason, it might be best to throw the leaf error only, which is the first one. - console.info(errors[0]); + toStdOut(JSON.stringify(errors, null, 2)); } return isValid; } diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index 1a4b7aa6..0c86e078 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -1,20 +1,29 @@ +import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; +import get from "https://deno.land/x/lodash@4.17.15-es/get.js"; +import _set from "https://deno.land/x/lodash@4.17.15-es/set.js"; import { afterAll, beforeAll, describe, it, } from "https://deno.land/std@0.166.0/testing/bdd.ts"; -import { assertEquals } from "https://deno.land/std@0.148.0/testing/asserts.ts"; +import { + assertEquals, + assertExists, +} from "https://deno.land/std@0.166.0/testing/asserts.ts"; + +import { cliArgs, fixturePath } from "./supports/mod.ts"; + import { DEFAULT_CONFIGS, getConfigs } from "../src/handlers/defaults/utils.ts"; -import difference from "https://deno.land/x/lodash@4.17.15-es/difference.js"; -import get from "https://deno.land/x/lodash@4.17.15-es/get.js"; import { generateManifest } from "../src/handlers/manifest/utils.ts"; -import { ManifestTypes } from "../src/types/enums.ts"; -import { assertExists } from "https://deno.land/std/testing/asserts.ts"; -import { fixturePath } from "./supports/mod.ts"; import { install, uninstall, upgrade } from "../src/handlers/plugins/mod.ts"; +import { set } from "../src/handlers/defaults/mod.ts"; import { getRegisteredModule } from "../src/handlers/plugins/registry.ts"; -import { Arguments } from "../src/deps/yargs.ts"; + +import { ManifestTypes } from "../src/types/enums.ts"; +import { validateType } from "../../../packages/utils/src/mod.ts"; +import { Manifest } from "../../../packages/standard/src/manifest/manifest.ts"; +import ManifestGenerator from "../src/handlers/manifest/generator.ts"; describe("defaults commands", () => { it("list", async () => { @@ -35,10 +44,27 @@ describe("defaults commands", () => { 0, ); }); + + it("set", async () => { + const MANIFEST_DEFAULT_FORMAT = "json"; + await set( + cliArgs({ key: "manifest.format", value: MANIFEST_DEFAULT_FORMAT }), + ); + let manifestSection = await getConfigs({ section: "manifest" }); + assertEquals(manifestSection.format, MANIFEST_DEFAULT_FORMAT); + + await set( + cliArgs({ key: "manifest.format", value: "some-invalid-format" }), + ); + + manifestSection = await getConfigs({ section: "manifest" }); + // Shouldn't have changed to "some-invalid-format", because its not a valid one + assertEquals(manifestSection.format, MANIFEST_DEFAULT_FORMAT); + }); }); -describe("manifest create", () => { - it("generate", async () => { +describe("manifest commands", () => { + it("create", async () => { const fixtureFilepath = fixturePath("example_data.csv"); const manifest = await generateManifest({ @@ -55,6 +81,21 @@ describe("manifest create", () => { assertEquals(rows[0].values.length, 7); assertEquals(rows.length, 147); }); + + it("validate", async () => { + const manifestGenerator = ManifestGenerator(); + + const manifest = await manifestGenerator.generate({ + author: "Deno Tester", + filepath: fixturePath("example_data.csv"), + }); + + _set(manifest, "supplementalInfo.data[0].type", "invalid-type"); + + const { errors } = validateType(Manifest, manifest); + assertExists(errors); + assertEquals(errors.length, 2); + }); }); describe("plugin commands", () => { @@ -72,11 +113,7 @@ describe("plugin commands", () => { afterAll(async () => await cleanTestPlugin()); it("install", async () => { - const args = { - module: `${MODULE_NAME}@${MODULE_VERSION_3}`, - _: [], - }; - await install(args); + await install(cliArgs({ module: `${MODULE_NAME}@${MODULE_VERSION_3}` })); const pluginModule = await getRegisteredModule({ name: MODULE_NAME }); @@ -86,11 +123,8 @@ describe("plugin commands", () => { assertEquals(pluginModule.uriScheme, "https"); }); - it("upgrade patch", async () => { - const args: Arguments = { - module: MODULE_NAME, - _: [], - }; + it("upgrade", async () => { + const args = cliArgs({ module: MODULE_NAME }); // Patch args.patch = true; await upgrade(args); @@ -116,11 +150,7 @@ describe("plugin commands", () => { }); it("uninstall", async () => { - const args = { - module: MODULE_NAME, - _: [], - }; - await uninstall(args); + await uninstall(cliArgs({ module: MODULE_NAME })); const pluginModule = await getRegisteredModule({ name: MODULE_NAME }); diff --git a/app/cli/test/supports/mod.ts b/app/cli/test/supports/mod.ts index cdbe78d5..b3da4e87 100644 --- a/app/cli/test/supports/mod.ts +++ b/app/cli/test/supports/mod.ts @@ -1,4 +1,5 @@ import { path } from "../../src/deps/mod.ts"; +import { Arguments } from "../../src/deps/yargs.ts"; export * as path from "https://deno.land/std@0.149.0/path/mod.ts"; @@ -15,3 +16,11 @@ export function fixturePath(filename: string) { ); return absolute_filepath; } + +// deno-lint-ignore no-explicit-any +export function cliArgs(args: any): Arguments { + return { + ...args, + _: [], + }; +} diff --git a/packages/utils/src/deps/lodash.ts b/packages/utils/src/deps/lodash.ts index 3b895a87..322000b9 100644 --- a/packages/utils/src/deps/lodash.ts +++ b/packages/utils/src/deps/lodash.ts @@ -11,6 +11,7 @@ import uniqBy from "https://deno.land/x/lodash@4.17.15-es/uniqBy.js"; import find from "https://deno.land/x/lodash@4.17.15-es/find.js"; import sortBy from "https://deno.land/x/lodash@4.17.15-es/sortBy.js"; import every from "https://deno.land/x/lodash@4.17.15-es/every.js"; +import partition from "https://deno.land/x/lodash@4.17.15-es/partition.js"; export { every, find, @@ -20,6 +21,7 @@ export { isUndefined, merge, omitBy, + partition, set, sortBy, uniqBy, diff --git a/packages/utils/src/deps/typebox.ts b/packages/utils/src/deps/typebox.ts index 0ace2e4f..f86c5612 100644 --- a/packages/utils/src/deps/typebox.ts +++ b/packages/utils/src/deps/typebox.ts @@ -1,10 +1,13 @@ // NOTE: not all of typebox API can be directly imported in deno, one solution was // to use esm.sh. https://github.com/sinclairzx81/typebox/issues/216 import { Value } from "https://esm.sh/@sinclair/typebox@0.24.27/value"; -import { TypeCompiler } from "https://esm.sh/@sinclair/typebox@0.24.27/compiler"; +import { + TypeCompiler, + ValueError, +} from "https://esm.sh/@sinclair/typebox@0.24.27/compiler"; import { Static, Type } from "https://esm.sh/@sinclair/typebox@0.24.27/typebox"; -export type { Static }; +export type { Static, ValueError }; export { Type, TypeCompiler, Value }; export * from "https://deno.land/x/typebox@0.24.27/src/typebox.ts"; diff --git a/packages/utils/src/mod.ts b/packages/utils/src/mod.ts index c867a61f..3ba2b521 100644 --- a/packages/utils/src/mod.ts +++ b/packages/utils/src/mod.ts @@ -1,6 +1,9 @@ -import { TypeCompiler, Value } from "./deps/typebox.ts"; +import { TypeCompiler, Value, ValueError } from "./deps/typebox.ts"; import { conversion, fs, path, yaml } from "./deps/mod.ts"; import compress from "./deps/compress.ts"; +import { partition } from "./deps/lodash.ts"; + +const UNION_TYPE_ERROR_CODE = 34; export function remove(filepath: string, opts?: Deno.RemoveOptions) { return Deno.removeSync(filepath, opts); @@ -13,7 +16,7 @@ export function read(filepath: string) { export function toStdOut(str: string) { const text = new TextEncoder().encode(`${str}\n`); - conversion.writeAllSync(Deno.stdout, text); + return conversion.writeAllSync(Deno.stdout, text); } export function toFile(args: { filepath: string; content: string }) { @@ -54,6 +57,25 @@ export function jsonToText( return text; } +export async function textToJson( + args: { format?: string; text: string }, + // deno-lint-ignore no-explicit-any +): Promise { + const { format, text } = args; + switch (format || "json") { + case "yaml": + return await yaml.parse(text); + + case "json": + return JSON.parse(text); + + default: + throw new Error( + `output format '${format}' not supported.`, + ); + } +} + export function expandGlobPaths(filepattern: string) { const filepaths = []; for (const file of fs.expandGlobSync(filepattern)) { @@ -70,12 +92,40 @@ export function createTemplateForType(type: any) { return Value.Create(type); } -// deno-lint-ignore no-explicit-any -export function validateType(type: any, value: any) { +export function validateType( + // deno-lint-ignore no-explicit-any + type: any, + // deno-lint-ignore no-explicit-any + value: any, +): { isValid: boolean; errors?: ValueError[] } { const ValueCompiler = TypeCompiler.Compile(type); const isValid = ValueCompiler.Check(value); const valueErrors = [...ValueCompiler.Errors(value)]; - return { isValid, errors: valueErrors }; + + if (!valueErrors.length) return { isValid }; + + /** + * TypeBox's TypeCompiler errors are quite verbosy and the escalate upwards the JSON tree, specially for Union Type. + * So if the error is located at a given leaf in the JSON tree, additional errors upwards the tree + * will be generated. + * For this reason, a little formatting is done here to only keep the errors + * for one of the last leafs of the json tree, + */ + + // Only keep last-leaf errors. + const lastLeafPath = valueErrors[0].path; + const lastLeafErrors = valueErrors.filter((error) => + error.path === lastLeafPath + ); + const [unionErrors, otherErrors] = partition( + lastLeafErrors, + (error: ValueError) => error.type === UNION_TYPE_ERROR_CODE, + ); + + // Prefer union errors. + const errors = (unionErrors.length ? unionErrors : otherErrors); + + return { isValid, errors }; } export async function packFiles( From e6f44f7dc4fb8b2c2a10067222d4b59c4a3bb912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20P=C3=A9rez=20Manr=C3=ADquez?= Date: Wed, 30 Nov 2022 10:51:51 -0300 Subject: [PATCH 7/7] fixes the default parser --- app/cli/src/handlers/plugins/default-parser.ts | 8 ++++---- app/cli/test/main.test.ts | 2 +- packages/plugins/src/types.ts | 12 ++++++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/cli/src/handlers/plugins/default-parser.ts b/app/cli/src/handlers/plugins/default-parser.ts index 08cdbdcf..882842c3 100644 --- a/app/cli/src/handlers/plugins/default-parser.ts +++ b/app/cli/src/handlers/plugins/default-parser.ts @@ -48,10 +48,10 @@ const DefaultDataParser: IParser = { index: columnIndex, value, }); - tabularDataObject.rows.push({ - index: rowIndex, - values, - }); + }); + tabularDataObject.rows.push({ + index: rowIndex, + values, }); }); diff --git a/app/cli/test/main.test.ts b/app/cli/test/main.test.ts index 0c86e078..2504f72d 100644 --- a/app/cli/test/main.test.ts +++ b/app/cli/test/main.test.ts @@ -79,7 +79,7 @@ describe("manifest commands", () => { const rows = get(manifest, "supplementalInfo.data[0].rows"); assertEquals(rows[0].values.length, 7); - assertEquals(rows.length, 147); + assertEquals(rows.length, 21); }); it("validate", async () => { diff --git a/packages/plugins/src/types.ts b/packages/plugins/src/types.ts index f2445f42..fe701481 100644 --- a/packages/plugins/src/types.ts +++ b/packages/plugins/src/types.ts @@ -52,11 +52,19 @@ export interface IParser extends IPlugin { * Parser interface that any Exporter Plugin should extend. */ export interface IExporter extends IPlugin { + type: PluginTypes.EXPORTER; /** * Receives an fsml manifest and returns a * file and data object of some unknown type (e.g, json, yaml, etc.). */ run: ( - manifest: TManifest, - ) => Promise<{ file: string | Uint8Array; data: unknown }>; + // This could be either a filepath to a manifest, a buffer stream + // or an already parsed JSON manifest. + + // NOTE: allowing multiple input type alternatives + // passes the input type handling responsibility to Plugin implementation. + // We could have different input vars for each input type alternative as well + // or implement some sort of common input adapter + manifest: string | Uint8Array | TManifest, + ) => Promise<{ file?: string | Uint8Array; data: unknown }>; }