Adds CLI tests

app/cli/deno.jsonc

@@ -4,7 +4,7 @@
     "compile:deb": "deno task bundle && bash ./compile.sh && rm -rf ./tmp ./bin",
     "bundle": "mkdir -p bin && deno bundle src/mod.ts bin/run",
     "fsml": "deno run --allow-net --allow-read --allow-env --allow-write --allow-run src/mod.ts",
-    "test": "deno test --coverage=./cov_profile/ --allow-read --allow-env",
+    "test": "deno test --coverage=./cov_profile/ --allow-read --allow-env --allow-net --allow-write",
     "coverage": "deno coverage ./cov_profile --lcov --output=cov_profile.lcov"
   },
   "importMap": "../../import_map.json",

app/cli/src/commands/manifest/mod.ts

@@ -1,9 +1,10 @@
 import create from "./create.ts";
 import _import from "./import.ts";
+import validate from "./validate.ts";
 import { commandFactory } from "../utils.ts";
 
 export default commandFactory({
   command: "manifest <subcommand>",
   describe: "Operates with the FSML manifest",
-  subCommands: [create, _import],
+  subCommands: [create, _import, validate],
 });

app/cli/src/commands/manifest/validate.ts

@@ -0,0 +1,16 @@
+import { Yargs } from "@fsml/cli/deps/yargs.ts";
+import { validate } from "@fsml/cli/handlers/manifest/mod.ts";
+
+function builder(yargs: Yargs) {
+  yargs.positional("filepath", {
+    describe: "FSML Manifest absolute file path",
+    type: "string",
+  });
+}
+
+export default {
+  command: "validate <filepath>",
+  describe: "Validate an FSML manifest",
+  builder,
+  handler: validate,
+};

app/cli/src/handlers/defaults/mod.ts

@@ -8,7 +8,7 @@ import {
   saveConfigs,
 } from "./utils.ts";
 
-/** CLI "defaults" commmand handlers **/
+/** CLI "defaults" command handlers **/
 async function edit(args: Arguments) {
   const { section } = args;
   const configs = await getConfigs();
@@ -22,18 +22,18 @@ async function list(args: Arguments) {
   const configs = await getConfigs();
   const _format = format || configs?.defaults?.format;
 
-  const stdout_text: string = await jsonToText({
+  const stdout_text: string = jsonToText({
     format: _format,
     content: configs,
   });
-  await toStdOut(stdout_text);
+  toStdOut(stdout_text);
 }
 
 async function set(args: Arguments) {
   const { key, value } = args;
   const configs = await getConfigs();
   _set(configs, key, parseConfigValue(value));
-  await saveConfigs(configs);
+  saveConfigs(configs);
 }
 
 async function reset(args: Arguments) {

app/cli/src/handlers/defaults/utils.ts

@@ -1,15 +1,16 @@
-import { fs, path, yaml } from "@fsml/cli/deps/mod.ts";
+import { fs, path } from "@fsml/cli/deps/mod.ts";
 import { merge, set } from "@fsml/packages/utils/deps/lodash.ts";
-import { TypeCompiler } from "@fsml/packages/utils/deps/typebox.ts";
 import {
   jsonToText,
   read,
+  textToJson,
   toFile,
   toStdOut,
+  validateType,
 } from "@fsml/packages/utils/mod.ts";
 import { Configs, TConfigs, TConfigValue } from "@fsml/cli/types/configs.ts";
 
-const DEFAULT_CONFIGS = {
+export const DEFAULT_CONFIGS = {
   "defaults": { "filepath": "./configs.yaml", "format": "yaml" },
   "manifest": {
     "author": null,
@@ -37,9 +38,9 @@ function getDefaultConfigs() {
 async function getConfigs({ section }: { section?: string } = {}) {
   const defaultConfigs = getDefaultConfigs();
   fs.ensureFileSync(USER_CONFIG_FILEPATH);
-  const configsText = await read(USER_CONFIG_FILEPATH);
+  const configsText = read(USER_CONFIG_FILEPATH);
 
-  const configs = await yaml.parse(configsText);
+  const configs = await textToJson({ format: "yaml", text: configsText });
 
   const finalConfigs = merge(defaultConfigs, configs || {});
 
@@ -48,12 +49,15 @@ async function getConfigs({ section }: { section?: string } = {}) {
   return finalConfigs;
 }
 
-async function saveConfigs(
+function saveConfigs(
   newConfigs: Partial<TConfigs>,
 ) {
   if (validateConfigs(newConfigs)) {
-    const newConfigTextFile = yaml.stringify(newConfigs);
-    await toFile({
+    const newConfigTextFile = jsonToText({
+      format: "yaml",
+      content: newConfigs,
+    });
+    toFile({
       filepath: USER_CONFIG_FILEPATH,
       content: newConfigTextFile,
     });
@@ -101,16 +105,10 @@ function parseConfigValue(value: string | null) {
 }
 
 function validateConfigs(configs: Partial<TConfigs>) {
-  //@ts-ignore:next-line : This seems like an issue with typebox types.
-  const ConfigsCompiler = TypeCompiler.Compile(Configs);
-  const isValid = ConfigsCompiler.Check(configs);
-  // NOTE: these config error messages may be too verbose
-  // typebox documentation also references ajv for validation: https://deno.land/x/[email protected]#validation
-  const configsErrors = [...ConfigsCompiler.Errors(configs)];
+  const { isValid, errors } = validateType(Configs, configs);
+
   if (!isValid) {
-    configsErrors.forEach((error) =>
-      toStdOut(jsonToText({ format: "json", content: error }))
-    );
+    toStdOut(JSON.stringify(errors, null, 2));
   }
   return isValid;
 }

app/cli/src/handlers/manifest/generator.ts

@@ -94,7 +94,7 @@ const ManifestGenerator = (
   async function generate(args: {
     author: string;
     filepath: string;
-    parser: string;
+    parser?: string;
   }): Promise<TManifest> {
     const { author: _author, filepath, parser } = args;
     const provenanceObject = author(_author);

app/cli/src/handlers/manifest/mod.ts

@@ -1,6 +1,17 @@
 import { Arguments } from "@fsml/cli/deps/yargs.ts";
-import { jsonToText, remove, toStdOut } from "@fsml/packages/utils/mod.ts";
-import { generateManifest, packManifest, writeManifest } from "./utils.ts";
+import {
+  jsonToText,
+  read,
+  remove,
+  textToJson,
+  toStdOut,
+} from "@fsml/packages/utils/mod.ts";
+import {
+  generateManifest,
+  packManifest,
+  validateManifest,
+  writeManifest,
+} from "./utils.ts";
 
 /** CLI "manifest" command handlers **/
 
@@ -49,7 +60,15 @@ async function _import() {}
 
 // async function _export(args: Arguments) {}
 
-// async function validate(args: Arguments) {}
+async function validate(args: Arguments) {
+  const { filepath } = args;
+
+  const manifestString = read(filepath);
+
+  const manifest = await textToJson({ format: "json", text: manifestString });
+
+  return validateManifest(manifest);
+}
 
 // async function pack(args: Arguments) {}
 
@@ -65,5 +84,5 @@ export {
   // score,
   // unpack,
   // update,
-  // validate,
+  validate,
 };

app/cli/src/handlers/manifest/utils.ts

@@ -119,16 +119,10 @@ async function getDataFilepath(
   return { filepath: datafilepath, isPack };
 }
 
-function validateManifest(manifest: TManifest): boolean {
+export function validateManifest(manifest: TManifest): boolean {
   const { isValid, errors } = validateType(Manifest, manifest);
   if (!isValid) {
-    toStdOut("Error in Manifest: \n");
-    // TypeBox's TypeCompiler errors are quite verbosy and the escalate upwards the JSON tree.
-    // so if the error is located at a given leaf in the JSON tree, additional errors upwards the tree
-    // will be generated.
-
-    // For this reason, it might be best to throw the leaf error only, which is the first one.
-    console.info(errors[0]);
+    toStdOut(JSON.stringify(errors, null, 2));
   }
   return isValid;
 }

app/cli/src/handlers/plugins/default-parser.ts

@@ -2,7 +2,7 @@
  * whatever interface we end up designing for data parsers. **/
 import { IParser, PluginTypes } from "@fsml/packages/plugins/types.ts";
 import { set } from "@fsml/packages/utils/deps/lodash.ts";
-import { createTemplateForType } from "@fsml/packages/utils/mod.ts";
+import { createTemplateForType, read } from "@fsml/packages/utils/mod.ts";
 import {
   TabularData,
   TTabularData,
@@ -15,6 +15,8 @@ import {
   TColumn,
   TKind,
 } from "@fsml/packages/standard/manifest/data/tabular/column/mod.ts";
+import papaparse from "https://esm.sh/[email protected]";
+import { TValue } from "../../../../../packages/standard/src/manifest/data/tabular/value.ts";
 
 // TODO: This one shall be extended so that it becomes
 // an actually useful default parser by somehow auto-detecting
@@ -24,8 +26,9 @@ import {
 const DefaultDataParser: IParser = {
   name: "defaultDataParser",
   type: PluginTypes.PARSER,
-  run: async (filepath) => {
-    console.info(`Parsing file '${filepath}'...`);
+  run: async (file) => {
+    console.info(`Parsing file '${file}'...`);
+
     const tabularDataObject = <TTabularData> createTemplateForType(TabularData);
     const columnObject = <TColumn> createTemplateForType(Column);
     const kindObject = <TKind> createTemplateForType(Kind);
@@ -36,13 +39,32 @@ const DefaultDataParser: IParser = {
     set(columnObject, "kind", kindObject);
     set(tabularDataObject, "column", columnObject);
 
+    const data = typeof file === "string" ? read(file) : file.toString();
+    const dataRows = <string[][]> papaparse.parse(data).data;
+    dataRows.forEach((csvRow, rowIndex) => {
+      const values: TValue[] = [];
+      csvRow.forEach((value, columnIndex) => {
+        values.push({
+          index: columnIndex,
+          value,
+        });
+      });
+      tabularDataObject.rows.push({
+        index: rowIndex,
+        values,
+      });
+    });
+
     // TODO: write data object to file.
     return await Promise.resolve({ data: tabularDataObject });
   },
-
-  isApplicable: async (filepath) => {
-    console.info(`Checking if ${filepath} can be parsed...`);
-    return await Promise.resolve(true);
+  isApplicable: async (file) => {
+    console.info(`Checking if file can be parsed...`);
+    if (typeof file === "string" && file.endsWith(".csv")) {
+      console.info(`File suitable for the defaultDataParser...`);
+      return await Promise.resolve(true);
+    }
+    return await Promise.resolve(false);
   },
 };
 

app/cli/src/handlers/plugins/handler/utils.ts

@@ -33,6 +33,8 @@ const defaultVersionResolver = async (
       version = match[1];
     }
   }
+  // Just close response read stream. We don't need to read its body.
+  await resp.body?.cancel();
 
   return version;
 };

app/cli/src/handlers/plugins/registry.ts

@@ -37,7 +37,7 @@ async function addModuleToRegistry(
     pluginsRegistry,
   );
 
-  await toFile({
+  toFile({
     filepath: PLUGINS_REGISTRY_FILEPATH,
     content: pluginsRegistryString_updated,
   });
@@ -75,7 +75,7 @@ async function removeModuleFromRegistry(
 
 async function getPluginRegistry(): Promise<TPluginsRegistry> {
   fs.ensureFileSync(PLUGINS_REGISTRY_FILEPATH);
-  const pluginsRegistryString = await read(PLUGINS_REGISTRY_FILEPATH);
+  const pluginsRegistryString = read(PLUGINS_REGISTRY_FILEPATH);
   const pluginsRegistry =
     (await yaml.parse(pluginsRegistryString) as TPluginsRegistry) ||
     PLUGIN_REGISTRY_TEMPLATE;

app/cli/test/fixtures/example_data.csv

@@ -0,0 +1,21 @@
+Clone,Time,Time Units,Acetone Concentration ,Acetone Concentration Unit,D-Glucose ,D-Glucose Unit
+C_1,1,hrs,0.49279369,g/L,0.60864757,g/L
+C_2,2,hrs,2.46021607,g/L,1.75225928,g/L
+C_3,3,hrs,1.887799053,g/L,1.06535886,g/L
+C_4,4,hrs,0.36752897,g/L,0.75896524,g/L
+C_5,5,hrs,2.287611982,g/L,1.54513438,g/L
+C_6,6,hrs,1.145001585,g/L,0.1740019,g/L
+C_7,7,hrs,1.249359273,g/L,0.29923113,g/L
+C_8,8,hrs,0.770046661,g/L,0.27594401,g/L
+C_9,9,hrs,1.444788622,g/L,0.53374635,g/L
+C_10,10,hrs,1.599680583,g/L,0.7196167,g/L
+C_11,11,hrs,1.163177704,g/L,0.19581324,g/L
+C_12,12,hrs,1.51270074,g/L,0.61524089,g/L
+C_13,13,hrs,1.183822988,g/L,0.22058759,g/L
+C_14,14,hrs,1.397965591,g/L,0.47755871,g/L
+C_15,15,hrs,1.342762838,g/L,0.41131541,g/L
+C_16,16,hrs,0.715611165,g/L,0.3412666,g/L
+C_17,17,hrs,0.681662316,g/L,0.38200522,g/L
+C_18,18,hrs,2.287762838,g/L,1.54531541,g/L
+C_19,19,hrs,0.669134432,g/L,0.39703868,g/L
+C_20,20,hrs,1.476763052,g/L,0.57211566,g/L