Inspect Multiple Files and Folders

pyflask/apis/neuroconv.py

@@ -13,7 +13,11 @@
     validate_metadata,
     listen_to_neuroconv_events,
     generate_dataset,
+    inspect_nwb_file,
+    inspect_nwb_folder,
+    inspect_multiple_filesystem_objects,
 )
+
 from errorHandlers import notBadRequestException
 
 neuroconv_api = Namespace("neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.")
@@ -141,25 +145,7 @@ class InspectNWBFile(Resource):
     @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
     def post(self):
         try:
-            import json
-            from nwbinspector import inspect_nwbfile
-            from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
-
-            return json.loads(
-                json.dumps(
-                    list(
-                        inspect_nwbfile(
-                            ignore=[
-                                "check_description",
-                                "check_data_orientation",
-                            ],  # TODO: remove when metadata control is exposed
-                            **neuroconv_api.payload,
-                        )
-                    ),
-                    cls=InspectorOutputJSONEncoder,
-                )
-            )
-
+            return inspect_nwb_file(neuroconv_api.payload)
         except Exception as e:
             if notBadRequestException(e):
                 neuroconv_api.abort(500, str(e))
@@ -170,24 +156,30 @@ class InspectNWBFolder(Resource):
     @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
     def post(self):
         try:
-            import json
-            from nwbinspector import inspect_all
-            from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
-
-            messages = list(
-                inspect_all(
-                    n_jobs=-2,  # uses number of CPU - 1
-                    ignore=[
-                        "check_description",
-                        "check_data_orientation",
-                    ],  # TODO: remove when metadata control is exposed
-                    **neuroconv_api.payload,
-                )
-            )
-
-            # messages = organize_messages(messages, levels=["importance", "message"])
-
-            return json.loads(json.dumps(messages, cls=InspectorOutputJSONEncoder))
+            return inspect_nwb_folder(neuroconv_api.payload)
+
+        except Exception as e:
+            if notBadRequestException(e):
+                neuroconv_api.abort(500, str(e))
+
+
+@neuroconv_api.route("/inspect")
+class InspectNWBFolder(Resource):
+    @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+    def post(self):
+        from os.path import isfile
+
+        try:
+            paths = neuroconv_api.payload["paths"]
+
+            if len(paths) == 1:
+                if isfile(paths[0]):
+                    return inspect_nwb_file({"path": paths[0]})
+                else:
+                    return inspect_nwb_folder({"path": paths[0]})
+
+            else:
+                return inspect_multiple_filesystem_objects(paths)
 
         except Exception as e:
             if notBadRequestException(e):

pyflask/manageNeuroconv/__init__.py

@@ -9,6 +9,9 @@
     upload_folder_to_dandi,
     listen_to_neuroconv_events,
     generate_dataset,
+    inspect_nwb_file,
+    inspect_nwb_folder,
+    inspect_multiple_filesystem_objects,
 )
 
 

pyflask/manageNeuroconv/info/__init__.py

@@ -1 +1,7 @@
-from .urls import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH
+from .urls import (
+    resource_path,
+    GUIDE_ROOT_FOLDER,
+    STUB_SAVE_FOLDER_PATH,
+    CONVERSION_SAVE_FOLDER_PATH,
+    TUTORIAL_SAVE_FOLDER_PATH,
+)

pyflask/manageNeuroconv/info/urls.py

@@ -20,6 +20,7 @@ def resource_path(relative_path):
 )  # NOTE: Must have pyflask for running the GUIDE as a whole, but errors for just the server
 f = path_config.open()
 data = json.load(f)
+GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"])
 STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["preview"])
 CONVERSION_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["conversions"])
 TUTORIAL_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["tutorial"])

pyflask/manageNeuroconv/manage_neuroconv.py

@@ -9,7 +9,7 @@
 from pathlib import Path
 
 from sse import MessageAnnouncer
-from .info import STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH
+from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH
 
 announcer = MessageAnnouncer()
 
@@ -507,3 +507,89 @@ def generate_dataset(test_data_directory_path: str):
             phy_output_dir.symlink_to(phy_base_directory, True)
 
     return {"output_directory": str(output_directory)}
+
+
+def inspect_nwb_file(payload):
+    from nwbinspector import inspect_nwbfile
+    from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
+
+    return json.loads(
+        json.dumps(
+            list(
+                inspect_nwbfile(
+                    ignore=[
+                        "check_description",
+                        "check_data_orientation",
+                    ],  # TODO: remove when metadata control is exposed
+                    **payload,
+                )
+            ),
+            cls=InspectorOutputJSONEncoder,
+        )
+    )
+
+
+def inspect_nwb_file(payload):
+    from nwbinspector import inspect_nwbfile
+    from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
+
+    return json.loads(
+        json.dumps(
+            list(
+                inspect_nwbfile(
+                    ignore=[
+                        "check_description",
+                        "check_data_orientation",
+                    ],  # TODO: remove when metadata control is exposed
+                    **payload,
+                )
+            ),
+            cls=InspectorOutputJSONEncoder,
+        )
+    )
+
+
+def inspect_nwb_folder(payload):
+    from nwbinspector import inspect_all
+    from nwbinspector.nwbinspector import InspectorOutputJSONEncoder
+
+    messages = list(
+        inspect_all(
+            n_jobs=-2,  # uses number of CPU - 1
+            ignore=[
+                "check_description",
+                "check_data_orientation",
+            ],  # TODO: remove when metadata control is exposed
+            **payload,
+        )
+    )
+
+    # messages = organize_messages(messages, levels=["importance", "message"])
+
+    return json.loads(json.dumps(messages, cls=InspectorOutputJSONEncoder))
+
+
+def aggregate_symlinks_in_new_directory(paths, reason="", folder_path=None):
+    if folder_path is None:
+        folder_path = GUIDE_ROOT_FOLDER / ".temp" / reason / f"temp_{datetime.now().strftime('%Y%m%d-%H%M%S')}"
+
+    folder_path.mkdir(parents=True)
+
+    for path in paths:
+        path = Path(path)
+        new_path = folder_path / path.name
+        if path.is_dir():
+            aggregate_symlinks_in_new_directory(
+                list(map(lambda name: os.path.join(path, name), os.listdir(path))), None, new_path
+            )
+        else:
+            new_path.symlink_to(path, path.is_dir())
+
+    return folder_path
+
+
+def inspect_multiple_filesystem_objects(paths):
+    tmp_folder_path = aggregate_symlinks_in_new_directory(paths, "inspect")
+    result = inspect_nwb_folder({"path": tmp_folder_path})
+    rmtree(tmp_folder_path)
+    return result

src/renderer/src/stories/FileSystemSelector.js

@@ -10,7 +10,7 @@ function getObjectTypeReferenceString(type, multiple, { nested, native } = {}) {
             .join(" / ")}`;
 
     const isDir = type === "directory";
-    return multiple && (!isDir || (isDir && !native))
+    return multiple && (!isDir || (isDir && !native) || dialog)
         ? type === "directory"
             ? "directories"
             : "files"
@@ -122,7 +122,10 @@ export class FilesystemSelector extends LitElement {
     };
 
     #onCancel = () => {
-        this.#onThrow(`No ${this.type} selected`, "The request was cancelled by the user");
+        this.#onThrow(
+            `No ${getObjectTypeReferenceString(this.type, this.multiple, { native: true })} selected`,
+            "The request was cancelled by the user"
+        );
     };
 
     #checkType = (value) => {
@@ -133,12 +136,18 @@ export class FilesystemSelector extends LitElement {
 
     #handleFiles = async (pathOrPaths, type) => {
         if (!pathOrPaths)
-            this.#onThrow("No paths detected", `Unable to parse ${this.type} path${this.multiple ? "s" : ""}`);
+            this.#onThrow(
+                "No paths detected",
+                `Unable to parse ${getObjectTypeReferenceString(this.type, false, { native: true })} path${
+                    this.multiple ? "s" : ""
+                }`
+            );
 
         if (Array.isArray(pathOrPaths)) pathOrPaths.forEach(this.#checkType);
         else if (!type) this.#checkType(pathOrPaths);
 
         let resolvedValue = pathOrPaths;
+
         if (Array.isArray(resolvedValue) && !this.multiple) {
             if (resolvedValue.length > 1)
                 this.#onThrow(
@@ -158,9 +167,9 @@ export class FilesystemSelector extends LitElement {
 
     async selectFormat(type = this.type) {
         if (dialog) {
-            const file = await this.#useElectronDialog(type);
-            const path = file.filePath ?? file.filePaths?.[0];
-            this.#handleFiles(path, type);
+            const results = await this.#useElectronDialog(type);
+            // const path = file.filePath ?? file.filePaths?.[0];
+            this.#handleFiles(results.filePath ?? results.filePaths, type);
         } else {
             let handles = await (type === "directory"
                 ? window.showDirectoryPicker()
@@ -236,7 +245,7 @@ export class FilesystemSelector extends LitElement {
                                         native: true,
                                     })}`}</span
                           >${this.multiple &&
-                          (this.type === "directory" || (isMultipleTypes && this.type.includes("directory")))
+                          (this.type === "directory" || (isMultipleTypes && this.type.includes("directory") && !dialog))
                               ? html`<br /><small
                                         >Multiple directory support only available using drag-and-drop.</small
                                     >`

src/renderer/src/stories/JSONSchemaInput.js

@@ -149,6 +149,8 @@ export class JSONSchemaInput extends LitElement {
         `;
     }
 
+    #onThrow = (...args) => (this.onThrow ? this.onThrow(...args) : this.form?.onThrow(...args));
+
     #render() {
         const { validateOnChange, info, path: fullPath } = this;
 
@@ -158,7 +160,8 @@ export class JSONSchemaInput extends LitElement {
         const isArray = info.type === "array"; // Handle string (and related) formats / types
 
         const hasItemsRef = "items" in info && "$ref" in info.items;
-        if (!("items" in info) || (!("type" in info.items) && !hasItemsRef)) info.items = { type: "string" };
+        if (!("items" in info)) info.items = {};
+        if (!("type" in info.items) && !hasItemsRef) info.items.type = "string";
 
         // Handle file and directory formats
         const createFilesystemSelector = (format) => {
@@ -167,7 +170,7 @@ export class JSONSchemaInput extends LitElement {
                 value: this.value,
                 onSelect: (filePath) => this.#updateData(fullPath, filePath),
                 onChange: (filePath) => validateOnChange && this.#triggerValidation(name, el, path),
-                onThrow: (...args) => this.form?.onThrow(...args),
+                onThrow: (...args) => this.#onThrow(...args),
                 dialogOptions: this.form?.dialogOptions,
                 dialogType: this.form?.dialogType,
                 multiple: isArray,
@@ -211,7 +214,7 @@ export class JSONSchemaInput extends LitElement {
                             this.form.checkAllLoaded();
                         }
                     },
-                    onThrow: (...args) => this.form?.onThrow(...args),
+                    onThrow: (...args) => this.#onThrow(...args),
                 };
 
                 return (this.form.tables[name] =

src/renderer/src/stories/pages/inspect/InspectPage.js

@@ -6,7 +6,7 @@ import { Button } from "../../Button.js";
 import { run } from "../guided-mode/options/utils.js";
 import { JSONSchemaInput } from "../../JSONSchemaInput.js";
 import { Modal } from "../../Modal";
-import { truncateFilePaths } from "../../preview/NWBFilePreview.js";
+import { getSharedPath, truncateFilePaths } from "../../preview/NWBFilePreview.js";
 import { InspectorList } from "../../preview/inspector/InspectorList.js";
 
 export class InspectPage extends Page {
@@ -16,24 +16,29 @@ export class InspectPage extends Page {
 
     showReport = async (value) => {
         if (!value) {
-            const message = "Please provide a folder to inspect.";
+            const message = "Please provide filesystem entries to inspect.";
             onThrow(message);
             throw new Error(message);
         }
 
-        const items = truncateFilePaths(
-            await run("inspect_folder", { path: value }, { title: "Inspecting your files" }).catch((e) => {
-                this.notify(e.message, "error");
-                throw e;
-            }),
-            value
-        );
+        const result = await run(
+            "inspect",
+            { paths: value },
+            { title: "Inspecting selected filesystem entries." }
+        ).catch((e) => {
+            this.notify(e.message, "error");
+            throw e;
+        });
+
+        if (!result.length) return this.notify("No messages received from the NWB Inspector");
+
+        const items = truncateFilePaths(result, getSharedPath(result.map((o) => o.file_path)));
 
         const list = new InspectorList({ items });
         list.style.padding = "25px";
 
         const modal = new Modal({
-            header: value,
+            header: value.length === 1 ? value : `Selected Filesystem Entries`,
         });
         modal.append(list);
         document.body.append(modal);
@@ -42,17 +47,20 @@ export class InspectPage extends Page {
     };
 
     input = new JSONSchemaInput({
-        path: ["folder_path"],
+        path: ["filesystem_paths"],
         info: {
-            type: "string",
-            format: "directory",
+            type: "array",
+            items: {
+                format: ["file", "directory"],
+                multiple: true,
+            },
         },
         onThrow,
     });
 
     render() {
         const button = new Button({
-            label: "Inspect Files",
+            label: "Start Inspection",
             onClick: async () => this.showReport(this.input.value),
         });