From dbd04b3d4edeb5cd76d32b54692aa964515b9959 Mon Sep 17 00:00:00 2001
From: CGDogan <126820728+CGDogan@users.noreply.github.com>
Date: Sun, 17 Sep 2023 10:52:12 +0100
Subject: [PATCH 1/3] add fs updated methods

---
 caracal.js               |   2 +
 handlers/dataHandlers.js | 207 +++++++++++++++++++++++++++++++++++++++
 routes.json              |  12 +++
 routes.json.example      |  12 +++
 4 files changed, 233 insertions(+)

diff --git a/caracal.js b/caracal.js
index cb00f28..762283c 100644
--- a/caracal.js
+++ b/caracal.js
@@ -125,6 +125,8 @@ var HANDLERS = {
   "removePresetlabels": function() {
     return dataHandlers.Presetlabels.remove;
   },
+  "addedFileToFS": dataHandlers.FSChanged.added,
+  "removedFileFromFS": dataHandlers.FSChanged.removed,
 };
 
 // TODO! -- remove these by fully depreciating tfjs serverside
diff --git a/handlers/dataHandlers.js b/handlers/dataHandlers.js
index dce2dc4..30c1580 100644
--- a/handlers/dataHandlers.js
+++ b/handlers/dataHandlers.js
@@ -1,5 +1,6 @@
 const DISABLE_SEC = (process.env.DISABLE_SEC === 'true') || false;
 const mongoDB = require("../service/database");
+const path = require('node:path');
 
 var General = {};
 General.find = function(db, collection) {
@@ -377,11 +378,217 @@ User.wcido = function(req, res, next) {
   }
 };
 
+var FSChanged = {};
+
+function badPath(path) {
+  if (path.includes("..")) return true;
+  if (path.includes("/./")) return true;
+  if (path.includes("//")) return true;
+  if (path.startsWith(".")) return true;
+  return false;
+}
+
+FSChanged.added = function(db, collection, loader) {
+  return function(req, res) {
+    var query = req.query;
+    if (!query.hasOwnProperty("filepath")) {
+      res.send({error: "filepath parameter undefined"});
+      return;
+    }
+    if (query.filepath == '' || query.filepath.endsWith("/") || query.filepath.startsWith("/")) {
+      res.send({error: "expected a relative path to an image file"});
+      return;
+    }
+    if (badPath(query.filepath)) {
+      res.send({error: "filepath not canonical or invalid"});
+      return;
+    }
+    mongoDB.find(db, collection).then((slides) => {
+      // If contains any file from the parent path, do nothing.
+      // otherwise, add it as a new entry
+      var parentDir = path.dirname(query.filepath);
+
+      var identifier;
+      if (parentDir == '.') {
+        // This is a single file not in any subfolder, and does not have companion files
+        identifier = query.filepath;
+      } else {
+        // This is a file in a subdirectory.
+        // caMicroscope design decision: every subdir in the images directory is one image
+        // so traverse up if needed.
+        do {
+          identifier = parentDir;
+          parentDir = path.dirname(parentDir);
+        } while (parentDir != '.');
+      }
+
+      // Here, we can be more fault tolerant and handle the case that despite still being an entry,
+      // it was somehow deleted from the filesystem.
+      // It may be replaced with any other file in the folder or the user requested path
+      // given that we verify that it exists.
+      // but that's the purpose of FSChanged.removed
+      if (slides.some((s) => s["filepath"] && s["filepath"].includes(identifier))) {
+        // Success, to allow the client to notify for every new file, even if that won't make a new series.
+        res.send({success: "another file from the same subdirectory is already in database"});
+        return;
+      }
+      var filepath = query.filepath;
+      var name = path.basename(filepath);
+      fetch(loader + "/data/one/" + filepath).then((r) => {
+        if (!r.ok) {
+          res.send({error: "SlideLoader error: perhaps the filepath points to an inexistant file?"});
+          return;
+        }
+        r.json().then((data) => {
+          if (data.error) {
+            res.send({error: "SlideLoader error: the filepath points to an inexistant file?"});
+            return;
+          }
+          data.name = name;
+          mongoDB.add(db, collection, data).then(() => {
+            res.send({success: "added successfully"});
+          }).catch((e) => {
+            res.send({error: "mongo failure"});
+            console.log(e);
+          });
+        });
+      });
+    }).catch((e) => {
+      res.send({error: ""});
+    });
+  };
+};
+
+
+FSChanged.removed = function(db, collection, loader) {
+  return function(req, res) {
+    var query = req.query;
+    if (!query.hasOwnProperty("filepath")) {
+      res.send({error: "filepath parameter undefined"});
+      return;
+    }
+    if (query.filepath == '' || query.filepath.endsWith("/") || query.filepath.startsWith("/")) {
+      res.send({error: "expected a relative path to an image file"});
+      return;
+    }
+    if (badPath(query.filepath)) {
+      res.send({error: "filepath not canonical or invalid"});
+      return;
+    }
+
+    (async () => {
+      var identifier; // scanning pattern
+      var replace; // true: replace entries. false: delete entries.
+      var replacer; // MongoDB object
+
+      // check that the file doesn't exist
+      try {
+        var metadata = await fetch(loader + "/data/one/" + query.filepath);
+        metadata = await metadata.json();
+      } catch (e) {
+        res.send({error: "slideloader failure"});
+        console.log(e);
+        return;
+      }
+      if (!metadata.hasOwnProperty("error")) {
+        res.send({error: "file " + query.filepath + " still exists"});
+        return;
+      }
+
+      var parentDir = path.dirname(query.filepath);
+      if (parentDir == '.') {
+        // file in the top level folder
+        // Delete entries with identifier.
+        replace = false;
+        identifier = query.filepath;
+      } else {
+        // This is a file in a subdirectory.
+        // caMicroscope design decision: every subdir in the images directory is one image
+        // so traverse up if needed.
+        do {
+          identifier = parentDir;
+          parentDir = path.dirname(parentDir);
+        } while (parentDir != '.');
+        var basename = path.basename(query.filepath);
+        // get folder contents. Pick any replacements from the array.
+        // if no other files in the folder, delete db entries
+        try {
+          var contents = await fetch(loader + "/data/folder/" + identifier);
+          contents = await contents.json();
+          contents = contents.contents;
+        } catch (e) {
+          res.send({error: "slideloader failure"});
+          console.log(e);
+          return;
+        }
+        if (contents.length == 0) {
+          // Delete DB entries
+          replace = false;
+        } else {
+          // Replace DB entries
+          replace = true;
+          // TODO: here it would be better to check if this is a folder and if it's a folder
+          // use any tree search to find a file and if none check other r.contents entries
+          var newFilePath = identifier + '/' + contents[0];
+          try {
+            replacer = await fetch(loader + "/data/one/" + newFilePath);
+            replacer = await replacer.json();
+          } catch (e) {
+            res.send({error: "slideloader failure"});
+            console.log(e);
+            return;
+          }
+          if (replacer.hasOwnProperty("error")) {
+            // See the TODO above
+            res.send({error: "picked " + newFilePath + " which could not be reader"});
+            return;
+          }
+          replacer = {$set: replacer};
+        }
+      }
+
+      try {
+        var slides = await mongoDB.find(db, collection);
+      } catch (e) {
+        res.send({error: "mongo failure"});
+        console.log(e);
+        return;
+      }
+
+      if (replace) {
+        for (const entry of slides) {
+          if (entry["filepath"] && entry["filepath"].includes(identifier)) {
+            try {
+              replacer.$set.name = entry.name;
+              await mongoDB.update(db, collection, {_id: entry._id.$oid}, replacer);
+            } catch (e) {
+              console.log(e);
+            }
+          }
+        }
+        res.send({success: "replaced if any entries were found"});
+      } else {
+        for (const entry of slides) {
+          if (entry["filepath"] && entry["filepath"].includes(identifier)) {
+            try {
+              await mongoDB.delete(db, collection, {"_id": entry._id.$oid});
+            } catch (e) {
+              console.log(e);
+            }
+          }
+        }
+        res.send({success: "removed entries if any"});
+      }
+    })();
+  };
+};
+
 dataHandlers = {};
 dataHandlers.Heatmap = Heatmap;
 dataHandlers.Mark = Mark;
 dataHandlers.User = User;
 dataHandlers.Presetlabels = Presetlabels;
+dataHandlers.FSChanged = FSChanged;
 
 dataHandlers.General = General;
 module.exports = dataHandlers;
diff --git a/routes.json b/routes.json
index c3b6cba..a86b1c4 100644
--- a/routes.json
+++ b/routes.json
@@ -491,5 +491,17 @@
       {"function":"permissionHandler", "args": [["Admin", "Editor"]]},
       {"function":"sendTrainedModel", "args": []}
     ]
+  },{
+    "route":"/fs/addedFile",
+    "method":"get",
+    "handlers":[
+      {"function":"addedFileToFS", "args": ["camic", "slide", "http://ca-load:4000"]}
+    ]
+  },{
+    "route":"/fs/deletedFile",
+    "method":"get",
+    "handlers":[
+      {"function":"removedFileFromFS", "args": ["camic", "slide", "http://ca-load:4000"]}
+    ]
   }
 ]
diff --git a/routes.json.example b/routes.json.example
index fd8fbf8..8048673 100644
--- a/routes.json.example
+++ b/routes.json.example
@@ -1102,5 +1102,17 @@
         "args": []
       }
     ]
+  },{
+    "route":"/fs/addedFile",
+    "method":"get",
+    "handlers":[
+      {"function":"addedFileToFS", "args": ["camic", "slide", "http://ca-load:4000"]}
+    ]
+  },{
+    "route":"/fs/deletedFile",
+    "method":"get",
+    "handlers":[
+      {"function":"removedFileFromFS", "args": ["camic", "slide", "http://ca-load:4000"]}
+    ]
   }
 ]

From b0849ffb9a0528382333cf396f4308f48810019c Mon Sep 17 00:00:00 2001
From: CGDogan <126820728+CGDogan@users.noreply.github.com>
Date: Mon, 18 Sep 2023 10:57:49 +0100
Subject: [PATCH 2/3] Reorder code

---
 handlers/dataHandlers.js | 77 +++++++++++++++++++++-------------------
 1 file changed, 41 insertions(+), 36 deletions(-)

diff --git a/handlers/dataHandlers.js b/handlers/dataHandlers.js
index 30c1580..5eb5922 100644
--- a/handlers/dataHandlers.js
+++ b/handlers/dataHandlers.js
@@ -403,58 +403,63 @@ FSChanged.added = function(db, collection, loader) {
       res.send({error: "filepath not canonical or invalid"});
       return;
     }
-    mongoDB.find(db, collection).then((slides) => {
-      // If contains any file from the parent path, do nothing.
-      // otherwise, add it as a new entry
-      var parentDir = path.dirname(query.filepath);
 
-      var identifier;
-      if (parentDir == '.') {
-        // This is a single file not in any subfolder, and does not have companion files
-        identifier = query.filepath;
-      } else {
-        // This is a file in a subdirectory.
-        // caMicroscope design decision: every subdir in the images directory is one image
-        // so traverse up if needed.
-        do {
-          identifier = parentDir;
-          parentDir = path.dirname(parentDir);
-        } while (parentDir != '.');
-      }
+    // If contains any other file from the subfolder, do nothing.
+    // otherwise, add it as a new entry.
+    // Likewise for single files in the flat images directory.
+    var parentDir = path.dirname(query.filepath);
+
+    var identifier;
+    if (parentDir == '.') {
+      // This is a single file not in any subfolder, and does not have companion files
+      identifier = query.filepath;
+    } else {
+      // This is a file in a subdirectory.
+      // caMicroscope design decision: every subdir in the images directory is one image
+      // so traverse up if needed.
+      do {
+        identifier = parentDir;
+        parentDir = path.dirname(parentDir);
+      } while (parentDir != '.');
+    }
+
+    var filepath = query.filepath;
+    var name = path.basename(filepath);
 
-      // Here, we can be more fault tolerant and handle the case that despite still being an entry,
-      // it was somehow deleted from the filesystem.
-      // It may be replaced with any other file in the folder or the user requested path
-      // given that we verify that it exists.
-      // but that's the purpose of FSChanged.removed
-      if (slides.some((s) => s["filepath"] && s["filepath"].includes(identifier))) {
-        // Success, to allow the client to notify for every new file, even if that won't make a new series.
-        res.send({success: "another file from the same subdirectory is already in database"});
+    fetch(loader + "/data/one/" + filepath).then((r) => {
+      if (!r.ok) {
+        res.send({error: "SlideLoader error: perhaps the filepath points to an inexistant file?"});
         return;
       }
-      var filepath = query.filepath;
-      var name = path.basename(filepath);
-      fetch(loader + "/data/one/" + filepath).then((r) => {
-        if (!r.ok) {
-          res.send({error: "SlideLoader error: perhaps the filepath points to an inexistant file?"});
+      r.json().then((data) => {
+        if (data.error) {
+          res.send({error: "SlideLoader error: the filepath points to an inexistant file?"});
           return;
         }
-        r.json().then((data) => {
-          if (data.error) {
-            res.send({error: "SlideLoader error: the filepath points to an inexistant file?"});
+        data.name = name;
+
+        mongoDB.find(db, collection).then((slides) => {
+          // Here, we can be more fault tolerant and handle the case that despite still being an entry,
+          // it was somehow deleted from the filesystem.
+          // It may be replaced with any other file in the folder or the user requested path
+          // given that we verify that it exists.
+          // but that's the purpose of FSChanged.removed
+          if (slides.findLast((s) => s["filepath"] && s["filepath"].includes(identifier))) {
+            // Success, to allow the client to notify for every new file, even if that won't make a new series.
+            res.send({success: "another file from the same subdirectory is already in database"});
             return;
           }
-          data.name = name;
           mongoDB.add(db, collection, data).then(() => {
             res.send({success: "added successfully"});
           }).catch((e) => {
             res.send({error: "mongo failure"});
             console.log(e);
           });
+        }).catch((e) => {
+          res.send({error: "mongo failure"});
+          print(e)
         });
       });
-    }).catch((e) => {
-      res.send({error: ""});
     });
   };
 };

From dd52b7600dac9b8fd72754102d4ab1fa62a3c420 Mon Sep 17 00:00:00 2001
From: CGDogan <126820728+CGDogan@users.noreply.github.com>
Date: Mon, 18 Sep 2023 10:58:16 +0100
Subject: [PATCH 3/3] typo

---
 handlers/dataHandlers.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/handlers/dataHandlers.js b/handlers/dataHandlers.js
index 5eb5922..e0c585f 100644
--- a/handlers/dataHandlers.js
+++ b/handlers/dataHandlers.js
@@ -457,7 +457,7 @@ FSChanged.added = function(db, collection, loader) {
           });
         }).catch((e) => {
           res.send({error: "mongo failure"});
-          print(e)
+          console.log(e);
         });
       });
     });