From e1021fba2c0c80187b2f257f5f8e89d0eb32739b Mon Sep 17 00:00:00 2001 From: Rahuljagwani Date: Tue, 26 Dec 2023 01:43:13 +0530 Subject: [PATCH 01/10] added mongosh check in setup script --- setup_script.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/setup_script.sh b/setup_script.sh index c90afff..7b65f18 100755 --- a/setup_script.sh +++ b/setup_script.sh @@ -24,9 +24,12 @@ fi ### ## check if the system has required services installed ### -if ! command -v "mongo" &>/dev/null; then - echo "mongo could not be found on path. Please ensure that mongo is installed and is on PATH" - exit +if command -v "mongo" &>/dev/null; then + MONGO="mongo" +elif command -v "mongosh" &>/dev/null; then + MONGO="mongosh" +else + echo "mongo or mongo could not be found on path. Please ensure that mongo is installed and is on PATH" fi if ! command -v "node" &>/dev/null; then @@ -109,7 +112,7 @@ echo "Copying files complete!" ### ## try connecting to mongodb instance ### -until mongo --host "${HOST}" --eval "print(\"Connected!\")" >/dev/null; do +until $MONGO --host "${HOST}" --eval "print(\"Connected!\")" >/dev/null; do sleep 2 done echo "[ database ] : connection established" @@ -118,8 +121,8 @@ echo "[ database ] : connection established" ## check if database exists ### QUERY="db.getMongo().getDBNames().indexOf(\"${DB_NAME}\")" -COMMAND="mongo ${HOST} --eval '${QUERY}' --quiet" -if [ $(mongo ${HOST} --eval ${QUERY} --quiet) -lt 0 ]; then +COMMAND="${MONGO} ${HOST} --eval '${QUERY}' --quiet" +if [ $(${MONGO} ${HOST} --eval ${QUERY} --quiet) -lt 0 ]; then echo "[ database ] : does not exist" exit 1 else @@ -149,11 +152,11 @@ case $yn in echo "[ resource ] : clearing old configurations" echo "[ resource ] : seeding collections" - mongo --quiet --host $HOST $DB_NAME .seeder.collection.js + $MONGO --quiet --host $HOST $DB_NAME .seeder.collection.js echo "[ resource ] : seeding indexes" - mongo --quiet --host $HOST $DB_NAME .seeder.idx.js + $MONGO --quiet --host $HOST $DB_NAME .seeder.idx.js echo "[ resource ] : seeding configurations" - mongo --quiet --host $HOST $DB_NAME .seeder.default.js + $MONGO --quiet --host $HOST $DB_NAME .seeder.default.js ### ## ask the user if they want to remove the seeding files From b548f3a9a336dfcf0ee862d46c20d6d3fb24944f Mon Sep 17 00:00:00 2001 From: Birm Date: Thu, 11 Jan 2024 10:38:58 -0500 Subject: [PATCH 02/10] partially encode filepaths for pathdb --- handlers/pathdbIipHandler.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/handlers/pathdbIipHandler.js b/handlers/pathdbIipHandler.js index 78a7eec..ebc8de1 100644 --- a/handlers/pathdbIipHandler.js +++ b/handlers/pathdbIipHandler.js @@ -28,7 +28,10 @@ iipCheck = function(req, res, next) { console.log(x); // get path if (x && x['field_iip_path'] && x['field_iip_path'].length && x['field_iip_path'][0]['value']) { - req.newFilepath = x['field_iip_path'][0]['value']; + newFilepath = x['field_iip_path'][0]['value']; + newFilepath = encodeURIComponent(newFilepath); + newFilepath = newFilepath.replaceAll("%2F", "/"); + req.newFilepath = newFilepath; console.log(req.newFilepath); next(); } else { From 00472fff6c25129fd1dd5ba502b34e3e367eb78d Mon Sep 17 00:00:00 2001 From: Birm Date: Fri, 15 Mar 2024 12:20:27 -0400 Subject: [PATCH 03/10] no more backup dev branch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b88a2c..2a5b6a4 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ An item with no filter value is returned in all cases, and is thus also public. ## Local Development Environment In order to quickly setup a development environment, make use of the `setup_script.sh` script. This will setup the project, initialize and seed the database configurations, import routes and initialize environment config files and generate the keys required. -First clone the Caracal (backup-dev branch), caMicroscope and the Distro repositories and make sure that all of them are in the same parent directory. +First clone Caracal, caMicroscope and the Distro repositories and make sure that all of them are in the same parent directory. Run the script with `./setup_script` or `bash ./setup_script.sh` From dbd518a499ea150648ae1071710b96ef4ac100ea Mon Sep 17 00:00:00 2001 From: Birm Date: Tue, 23 Apr 2024 18:27:26 -0400 Subject: [PATCH 04/10] add mongo count --- caracal.js | 1 + service/database/index.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/caracal.js b/caracal.js index 762283c..e17323f 100644 --- a/caracal.js +++ b/caracal.js @@ -80,6 +80,7 @@ var HANDLERS = { "mongoUpdate": dataHandlers.General.update, "mongoDelete": dataHandlers.General.delete, "mongoDistinct": dataHandlers.General.distinct, + "mongoCount": dataHandlers.General.count, "filterHandler": auth.filterHandler, "permissionHandler": permissionHandler, "editHandler": auth.editHandler, diff --git a/service/database/index.js b/service/database/index.js index 81e9a97..542b01a 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -42,6 +42,34 @@ class Mongo { } } + /** + * Runs the MongoDB count() method to count documents. + * + * @async + * @param {string} database Name of the database + * @param {string} collectionName Name of the collection to run operation on + * @param {document} query Specifies selection filter using query operators. + * To return all documents in a collection, omit this parameter or pass an empty document ({}). + * @param {boolean} [transform=false] check to transform the IDs to ObjectID in response + * + * {@link https://docs.mongodb.com/manual/reference/method/db.collection.count/ Read MongoDB Reference} + */ + static async count(database, collectionName, query, transform = true) { + try { + query = transformIdToObjectId(query); + + const collection = getConnection(database).collection(collectionName); + const count = await collection.count(query); + + data =[{"count": count}]; + + return data; + } catch (e) { + console.error(e); + throw e; + } + } + /** * Runs a distinct find operation based on given query * From efdc88d952458dc323c1ccf7b7efb1367f26f061 Mon Sep 17 00:00:00 2001 From: Birm Date: Tue, 23 Apr 2024 18:29:57 -0400 Subject: [PATCH 05/10] add count as handler --- handlers/dataHandlers.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/handlers/dataHandlers.js b/handlers/dataHandlers.js index e0c585f..b345aaf 100644 --- a/handlers/dataHandlers.js +++ b/handlers/dataHandlers.js @@ -13,6 +13,16 @@ General.find = function(db, collection) { }; }; +General.count = function(db, collection) { + return function(req, res, next) { + var query = req.query; + mongoDB.count(db, collection, query).then((x) => { + req.data = x; + next(); + }).catch((e) => next(e)); + }; +}; + General.findWithRegex = function(db, collection) { return function(req, res, next) { var query = req.query; From 56a2e5d6d589e7abd803e28647239f0b4f456510 Mon Sep 17 00:00:00 2001 From: Birm Date: Tue, 23 Apr 2024 18:32:49 -0400 Subject: [PATCH 06/10] add count to index list --- service/database/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/database/index.js b/service/database/index.js index 542b01a..a3cb91b 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -234,5 +234,6 @@ module.exports = { aggregate: Mongo.aggregate, distinct: Mongo.distinct, createIndex: Mongo.createIndex, - createCollection: Mongo.createCollection + createCollection: Mongo.createCollection, + count: Mongo.count, }; From 3466a650aec5fb74033ada91fa748a2ac58e2a35 Mon Sep 17 00:00:00 2001 From: Birm Date: Tue, 23 Apr 2024 18:34:11 -0400 Subject: [PATCH 07/10] declare data in count --- service/database/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/database/index.js b/service/database/index.js index a3cb91b..cca743c 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -61,7 +61,7 @@ class Mongo { const collection = getConnection(database).collection(collectionName); const count = await collection.count(query); - data =[{"count": count}]; + let data =[{"count": count}]; return data; } catch (e) { From af188689000ed3e00107bf6238a0b4543775d346 Mon Sep 17 00:00:00 2001 From: Birm Date: Wed, 24 Apr 2024 10:13:08 -0400 Subject: [PATCH 08/10] limit find query by default --- service/database/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service/database/index.js b/service/database/index.js index cca743c..377492a 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -24,7 +24,9 @@ class Mongo { query = transformIdToObjectId(query); const collection = getConnection(database).collection(collectionName); - const data = await collection.find(query).toArray(); + let { _page = 0, _pageSize = 1000, ...filterQuery } = query; + const _skip = _page * _pageSize; + const data = await collection.find(filterQuery).skip(_skip).limit(_pageSize).toArray(); /** allow caller method to toggle response transformation */ if (transform) { From 0e959c76d8ba61a286d4ea73534bab1805e17c66 Mon Sep 17 00:00:00 2001 From: Birm Date: Wed, 24 Apr 2024 10:18:52 -0400 Subject: [PATCH 09/10] parse int for page size --- service/database/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/service/database/index.js b/service/database/index.js index 377492a..8692d7b 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -26,6 +26,7 @@ class Mongo { const collection = getConnection(database).collection(collectionName); let { _page = 0, _pageSize = 1000, ...filterQuery } = query; const _skip = _page * _pageSize; + _pageSize = parseInt(_pageSize, 10); const data = await collection.find(filterQuery).skip(_skip).limit(_pageSize).toArray(); /** allow caller method to toggle response transformation */ From 72090349b4379137a884f020d0f15412df7c32fd Mon Sep 17 00:00:00 2001 From: Birm Date: Wed, 24 Apr 2024 15:27:47 -0400 Subject: [PATCH 10/10] add pagniated find explicitly differently from find --- caracal.js | 1 + handlers/dataHandlers.js | 10 +++++++++ service/database/index.js | 44 +++++++++++++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/caracal.js b/caracal.js index e17323f..2e646c5 100644 --- a/caracal.js +++ b/caracal.js @@ -75,6 +75,7 @@ var HANDLERS = { }, "monitorCheck": monitor.check, "mongoFind": dataHandlers.General.find, + "mongoPaginatedFind": dataHandlers.General.paginatedFind, "mongoFindWithRegex": dataHandlers.General.findWithRegex, "mongoAdd": dataHandlers.General.add, "mongoUpdate": dataHandlers.General.update, diff --git a/handlers/dataHandlers.js b/handlers/dataHandlers.js index b345aaf..4e69192 100644 --- a/handlers/dataHandlers.js +++ b/handlers/dataHandlers.js @@ -13,6 +13,16 @@ General.find = function(db, collection) { }; }; +General.paginatedFind = function(db, collection) { + return function(req, res, next) { + var query = req.query; + mongoDB.paginatedFind(db, collection, query).then((x) => { + req.data = x; + next(); + }).catch((e) => next(e)); + }; +}; + General.count = function(db, collection) { return function(req, res, next) { var query = req.query; diff --git a/service/database/index.js b/service/database/index.js index 8692d7b..e4bdefd 100644 --- a/service/database/index.js +++ b/service/database/index.js @@ -24,10 +24,7 @@ class Mongo { query = transformIdToObjectId(query); const collection = getConnection(database).collection(collectionName); - let { _page = 0, _pageSize = 1000, ...filterQuery } = query; - const _skip = _page * _pageSize; - _pageSize = parseInt(_pageSize, 10); - const data = await collection.find(filterQuery).skip(_skip).limit(_pageSize).toArray(); + const data = await collection.find(query).toArray(); /** allow caller method to toggle response transformation */ if (transform) { @@ -45,6 +42,44 @@ class Mongo { } } + /** + * Runs the MongoDB find() method to fetch documents with pagination. + * + * @async + * @param {string} database Name of the database + * @param {string} collectionName Name of the collection to run operation on + * @param {document} query Specifies selection filter using query operators. + * To return all documents in a collection, omit this parameter or pass an empty document ({}). + * @param {boolean} [transform=false] check to transform the IDs to ObjectID in response + * + * {@link https://docs.mongodb.com/manual/reference/method/db.collection.find/ Read MongoDB Reference} + */ + static async paginatedFind(database, collectionName, query, transform = true) { + try { + query = transformIdToObjectId(query); + + const collection = getConnection(database).collection(collectionName); + let { _page = 0, _pageSize = 1000, ...filterQuery } = query; + const _skip = _page * _pageSize; + _pageSize = parseInt(_pageSize, 10); + const data = await collection.find(filterQuery).skip(_skip).limit(_pageSize).toArray(); + + /** allow caller method to toggle response transformation */ + if (transform) { + data.forEach((x) => { + x["_id"] = { + $oid: x["_id"], + }; + }); + } + + return data; + } catch (e) { + console.error(e); + throw e; + } + } + /** * Runs the MongoDB count() method to count documents. * @@ -232,6 +267,7 @@ class Mongo { module.exports = { add: Mongo.add, find: Mongo.find, + paginatedFind: Mongo.paginatedFind, update: Mongo.update, delete: Mongo.delete, aggregate: Mongo.aggregate,