From 5f26ef6f4ee9e4732db25940727ba13a65da8193 Mon Sep 17 00:00:00 2001
From: meerkatt <84426779+meerkatt@users.noreply.github.com>
Date: Fri, 23 Sep 2022 15:13:37 -0500
Subject: [PATCH 01/24] Update installing_clowder.rst (#383)

I believe there was a formatting issue in this yaml code on lines 216 and 239
---
 doc/src/sphinx/userguide/installing_clowder.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/src/sphinx/userguide/installing_clowder.rst b/doc/src/sphinx/userguide/installing_clowder.rst
index cb2e370cf..e3cc88348 100644
--- a/doc/src/sphinx/userguide/installing_clowder.rst
+++ b/doc/src/sphinx/userguide/installing_clowder.rst
@@ -213,7 +213,7 @@ for details.), we must tell Clowder which ports the services are using.
          # Enable Clowder to communicate with the necessary services (Mongo, RabbitMQ, ElsticSearch)
          # Each service runs as a Docker container.
 
-         services:
+       services:
          mongo:
             ports:
                - 27017:27017
@@ -236,7 +236,7 @@ for details.), we must tell Clowder which ports the services are using.
          # Enable Clowder to communicate with the necessary services (Mongo, RabbitMQ, ElsticSearch)
          # Each service runs as a Docker container.
 
-         services:
+       services:
          mongo:
             ports:
                - 27017:27017

From c88d68192543ec75a01cbbbca554bff99af5bd84 Mon Sep 17 00:00:00 2001
From: Dipannita Dey <ddey2@illinois.edu>
Date: Mon, 7 Nov 2022 15:58:56 -0600
Subject: [PATCH 02/24] button disabled while ajax call is going on

---
 app/views/datasets/create.scala.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/views/datasets/create.scala.html b/app/views/datasets/create.scala.html
index 44e7b1854..572139ec8 100644
--- a/app/views/datasets/create.scala.html
+++ b/app/views/datasets/create.scala.html
@@ -194,6 +194,7 @@ <h1>@Messages("create.header", Messages("dataset.title"))</h1>
 
         $(function () {
             $('#createdataset').click(function () {
+                $('#createdataset').attr("disabled", true);
                 var name = $('#name');
                 var desc = $('#description');
                 var spaceList = [];

From f34fc641645381eab0370f937a6fa81be6d44d25 Mon Sep 17 00:00:00 2001
From: Dipannita Dey <ddey2@illinois.edu>
Date: Mon, 7 Nov 2022 16:01:24 -0600
Subject: [PATCH 03/24] adding to change log

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85bd19a86..47f2c2e22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+## Fixed
+- Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
 
 ## 1.21.0 - 2022-08-23
 

From 32da80b446e05f3c9c0721608c3e6bf2b6951f31 Mon Sep 17 00:00:00 2001
From: Dipannita <ddey2@illinois.edu>
Date: Mon, 5 Dec 2022 09:27:25 -0600
Subject: [PATCH 04/24] 386 datasetlastmodifieddate is not updated (#392)

* updateing lastModifiesdate of dataset
* Adding lastModifiedDate to UI

Co-authored-by: toddn <tcnichol@illinois.edu>
---
 CHANGELOG.md                                  |  1 +
 app/api/Datasets.scala                        |  2 +
 app/controllers/Datasets.scala                |  4 +-
 .../mongodb/MongoDBDatasetService.scala       | 78 ++++++++++---------
 app/views/dataset.scala.html                  |  1 +
 5 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47f2c2e22..c5e98ecde 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ## Fixed
+- Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
 
 ## 1.21.0 - 2022-08-23
diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala
index c0fe3fcbc..edb7b3ca6 100644
--- a/app/api/Datasets.scala
+++ b/app/api/Datasets.scala
@@ -882,6 +882,7 @@ class  Datasets @Inject()(
 
         datasets.index(id)
         Ok(toJson(Map("status" -> "success")))
+
       }
       case None => Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id"))
     }
@@ -928,6 +929,7 @@ class  Datasets @Inject()(
                 events.addObjectEvent(request.user, id, x.name, EventType.ADD_METADATA_DATASET.toString)
 
                 datasets.index(id)
+
                 Ok(toJson("Metadata successfully added to db"))
               }
               case e: JsError => {
diff --git a/app/controllers/Datasets.scala b/app/controllers/Datasets.scala
index 4f639a4cd..4304d4be5 100644
--- a/app/controllers/Datasets.scala
+++ b/app/controllers/Datasets.scala
@@ -8,7 +8,8 @@ import play.api.Logger
 import play.api.Play.current
 import play.api.libs.json.Json._
 import services._
-import util.{FileUtils, Formatters, RequiredFieldsConfig, SortingUtils }
+import util.{FileUtils, Formatters, RequiredFieldsConfig, SortingUtils}
+
 import scala.collection.immutable._
 import scala.collection.mutable.ListBuffer
 import play.api.i18n.Messages
@@ -728,7 +729,6 @@ class Datasets @Inject() (
     implicit val user = request.user
 
     Logger.debug("------- in Datasets.submit ---------")
-
     val folder = folderId.flatMap(id => folders.get(UUID(id)))
     val retMap = request.body.asFormUrlEncoded.get("datasetid").flatMap(_.headOption) match {
       case Some(ds) => {
diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala
index 15de0db8d..f4c3b1d66 100644
--- a/app/services/mongodb/MongoDBDatasetService.scala
+++ b/app/services/mongodb/MongoDBDatasetService.scala
@@ -729,12 +729,12 @@ class MongoDBDatasetService @Inject() (
           if (file.isInstanceOf[models.File]) {
             val theFile = file.asInstanceOf[models.File]
             if (!theFile.thumbnail_id.isEmpty) {
-              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe)
+              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
               return
             }
           }
         }
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
       }
       case None => Logger.debug(s"Dataset $datasetId not found")
     }
@@ -749,12 +749,12 @@ class MongoDBDatasetService @Inject() (
           if (file.isInstanceOf[File]) {
             val theFile = file.asInstanceOf[File]
             if (!theFile.thumbnail_id.isEmpty) {
-              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe)
+              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
               return
             }
           }
         }
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
       }
       case None => Logger.debug("No dataset found with id " + datasetId)
     }
@@ -929,13 +929,13 @@ class MongoDBDatasetService @Inject() (
     val md = JSON.parse(json).asInstanceOf[DBObject]
     Dataset.dao.collection.findOne(MongoDBObject("_id" -> new ObjectId(id.stringify)), MongoDBObject("metadata" -> 1)) match {
       case None => {
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> md), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> md, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
       }
       case Some(x) => {
         x.getAs[DBObject]("metadata") match {
           case Some(map) => {
             val union = map.asInstanceOf[DBObject] ++ md
-            Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> union), false, false, WriteConcern.Safe)
+            Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> union, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
           }
           case None => Map.empty
         }
@@ -947,23 +947,25 @@ class MongoDBDatasetService @Inject() (
     Logger.debug("Adding XML metadata to dataset " + id + " from file " + fileId + ": " + json)
     val md = JsonUtil.parseJSON(json).asInstanceOf[java.util.LinkedHashMap[String, Any]].toMap
     Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-      $addToSet("datasetXmlMetadata" -> DatasetXMLMetadata.toDBObject(models.DatasetXMLMetadata(md, fileId.stringify))), false, false, WriteConcern.Safe)
+      $addToSet("datasetXmlMetadata" -> DatasetXMLMetadata.toDBObject(models.DatasetXMLMetadata(md, fileId.stringify)))
+        ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def removeXMLMetadata(id: UUID, fileId: UUID) {
     Logger.debug("Removing XML metadata belonging to file " + fileId + " from dataset " + id + ".")
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("datasetXmlMetadata" -> MongoDBObject("fileId" -> fileId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("datasetXmlMetadata" -> MongoDBObject("fileId" -> fileId.stringify))
+      ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def addUserMetadata(id: UUID, json: String) {
     Logger.debug("Adding/modifying user metadata to dataset " + id + " : " + json)
     val md = com.mongodb.util.JSON.parse(json).asInstanceOf[DBObject]
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadata" -> md), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadata" -> md, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   /** Change the metadataCount field for a dataset */
   def incrementMetadataCount(id: UUID, count: Long) = {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $inc("metadataCount" -> count), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $inc("metadataCount" -> count) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   /**
@@ -971,20 +973,20 @@ class MongoDBDatasetService @Inject() (
    */
   def updateInformation(id: UUID, description: String, name: String) {
       val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-          $set("description" -> description, "name" -> name),
+          $set("description" -> description, "name" -> name, "lastModifiedDate" -> new Date()),
           false, false, WriteConcern.Safe)
   }
 
   def updateName(id: UUID, name: String) {
     events.updateObjectName(id, name)
     val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-      $set("name" -> name),
+      $set("name" -> name, "lastModifiedDate" -> new Date()),
       false, false, WriteConcern.Safe)
   }
 
   def updateDescription(id: UUID, description: String){
     val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-      $set("description" -> description),
+      $set("description" -> description, "lastModifiedDate" -> new Date()),
       false, false, WriteConcern.Safe)
   }
 
@@ -995,7 +997,7 @@ class MongoDBDatasetService @Inject() (
     //Don't allow duplicates
     if (Dataset.dao.find(MongoDBObject("_id" -> new ObjectId(id.stringify)) ++ MongoDBObject("creators" -> creator)).length == 0) {
       val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-        $push("creators" -> creator),
+        $push("creators" -> creator) ++ $set("lastModifiedDate" -> new Date()),
         false, false, WriteConcern.Safe)
     }
   }
@@ -1005,7 +1007,7 @@ class MongoDBDatasetService @Inject() (
    */
   def removeCreator(id: UUID, creator: String) {
     Dataset.dao.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-      $pull("creators" -> creator), false, false, WriteConcern.Safe)
+      $pull("creators" -> creator) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   /**
@@ -1016,8 +1018,8 @@ class MongoDBDatasetService @Inject() (
     if (Dataset.dao.find(MongoDBObject("_id" -> new ObjectId(id.stringify)) ++ MongoDBObject("creators" -> creator)).length != 0) {
       removeCreator(id, creator);
       Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-        $push("creators" -> MongoDBObject("$each" -> MongoDBList(creator), "$position" -> position)),
-        false, false, WriteConcern.Safe)
+        $push("creators" -> MongoDBObject("$each" -> MongoDBList(creator), "$position" -> position))
+        ++ $set("lastModifiedDate" -> new Date()),false, false, WriteConcern.Safe)
     }
   }
 
@@ -1032,7 +1034,7 @@ class MongoDBDatasetService @Inject() (
   def updateLicense(id: UUID, licenseType: String, rightsHolder: String, licenseText: String, licenseUrl: String, allowDownload: String) {
       val licenseData = models.LicenseData(m_licenseType = licenseType, m_rightsHolder = rightsHolder, m_licenseText = licenseText, m_licenseUrl = licenseUrl, m_allowDownload = allowDownload.toBoolean)
       val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-          $set("licenseData" -> LicenseData.toDBObject(licenseData)),
+          $set("licenseData" -> LicenseData.toDBObject(licenseData), "lastModifiedDate" -> new Date()),
           false, false, WriteConcern.Safe)
   }
 
@@ -1056,14 +1058,14 @@ class MongoDBDatasetService @Inject() (
       if (!existingTags.contains(shortTag)) {
         val tagObj = models.Tag(name = shortTag, userId = userIdStr, extractor_id = eid, created = createdDate)
         tagsAdded += tagObj
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("tags" -> Tag.toDBObject(tagObj)), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("tags" -> Tag.toDBObject(tagObj)) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
       }
     })
     tagsAdded.toList
   }
 
   def setUserMetadataWasModified(id: UUID, wasModified: Boolean) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadataWasModified" -> Some(wasModified)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadataWasModified" -> Some(wasModified), "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def findMetadataChangedDatasets(): List[Dataset] = {
@@ -1083,7 +1085,8 @@ class MongoDBDatasetService @Inject() (
 
   def removeTag(id: UUID, tagId: UUID) {
     Logger.debug("Removing tag " + tagId)
-    val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe)
+    val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify)))
+    ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def removeTags(id: UUID, tags: List[String]) {
@@ -1094,12 +1097,13 @@ class MongoDBDatasetService @Inject() (
     // Only remove existing tags.
     tags.intersect(existingTags).map {
       tag =>
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("name" -> tag)), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("name" -> tag))
+          ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
     }
   }
 
   def removeAllTags(id: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("tags" -> List()), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("tags" -> List(), "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   // ---------- Tags related code ends ------------------
@@ -1349,28 +1353,28 @@ class MongoDBDatasetService @Inject() (
   }
 
   def addFile(datasetId: UUID, file: File) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("files" -> new ObjectId(file.id.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("files" -> new ObjectId(file.id.stringify)) ++$set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def addFolder(datasetId: UUID, folderId: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("folders" -> new ObjectId(folderId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("folders" -> new ObjectId(folderId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def addCollection(datasetId: UUID, collectionId: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("collections" -> new ObjectId(collectionId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("collections" -> new ObjectId(collectionId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def removeCollection(datasetId: UUID, collectionId: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("collections" -> new ObjectId(collectionId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("collections" -> new ObjectId(collectionId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def removeFile(datasetId: UUID, fileId: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("files" -> new ObjectId(fileId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("files" -> new ObjectId(fileId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
     removeXMLMetadata(datasetId, fileId)
   }
 
   def removeFolder(datasetId: UUID, folderId: UUID) {
-    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("folders" -> new ObjectId(folderId.stringify)), false, false, WriteConcern.Safe)
+    Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("folders" -> new ObjectId(folderId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
   }
 
   def newThumbnail(datasetId: UUID) {
@@ -1381,12 +1385,12 @@ class MongoDBDatasetService @Inject() (
           if (file.isInstanceOf[models.File]) {
             val theFile = file.asInstanceOf[models.File]
             if (!theFile.thumbnail_id.isEmpty) {
-              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe)
+              Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
               return
             }
           }
         }
-        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe)
+        Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe)
       }
       case None =>
     }
@@ -1452,11 +1456,11 @@ class MongoDBDatasetService @Inject() (
   def addToSpace(datasetId: UUID, spaceId: UUID): Unit = {
     val result = Dataset.update(
       MongoDBObject("_id" -> new ObjectId(datasetId.stringify)),
-      $addToSet("spaces" -> Some(new ObjectId(spaceId.stringify))),
+      $addToSet("spaces" -> Some(new ObjectId(spaceId.stringify))) ++$set( "lastModifiedDate" -> new Date()),
       false, false)
     if (get(datasetId).exists(_.isTRIAL == true) && spaces.get(spaceId).exists(_.isTrial == false)) {
       Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)),
-        $set("status" -> DatasetStatus.DEFAULT.toString),
+        $set("status" -> DatasetStatus.DEFAULT.toString, "lastModifiedDate" -> new Date()),
         false, false)
     }
   }
@@ -1464,7 +1468,7 @@ class MongoDBDatasetService @Inject() (
   def removeFromSpace(datasetId: UUID, spaceId: UUID): Unit = {
     val result = Dataset.update(
       MongoDBObject("_id" -> new ObjectId(datasetId.stringify)),
-      $pull("spaces" -> Some(new ObjectId(spaceId.stringify))),
+      $pull("spaces" -> Some(new ObjectId(spaceId.stringify))) ++$set("lastModifiedDate" -> new Date()),
       false, false)
 
     if (play.Play.application().configuration().getBoolean("verifySpaces")) {
@@ -1472,7 +1476,7 @@ class MongoDBDatasetService @Inject() (
       get(datasetId) match {
         case Some(d) if !d.spaces.map(s => spaces.get(s)).flatten.exists(_.isTrial == false) =>
           Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)),
-            $set("status" -> DatasetStatus.TRIAL.toString),
+            $set("status" -> DatasetStatus.TRIAL.toString, "lastModifiedDate" -> new Date()),
             false, false)
         case _ =>
       }
@@ -1646,13 +1650,13 @@ class MongoDBDatasetService @Inject() (
   def incrementDownloads(id: UUID, user: Option[User]) = {
     Logger.debug("updating downloads for dataset "+id.toString)
     Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
-      $inc("stats.downloads" -> 1) ++ $set("stats.last_downloaded" -> new Date), true, false, WriteConcern.Safe)
+      $inc("stats.downloads" -> 1) ++ $set("stats.last_downloaded" -> new Date, "lastModifiedDate" -> new Date()), true, false, WriteConcern.Safe)
 
     user match {
       case Some(u) => {
         Logger.debug("updating downloads for user "+u.toString)
         DatasetStats.update(MongoDBObject("user_id" -> new ObjectId(u.id.stringify), "resource_id" -> new ObjectId(id.stringify), "resource_type" -> "dataset"),
-          $inc("downloads" -> 1) ++ $set("last_downloaded" -> new Date), true, false, WriteConcern.Safe)
+          $inc("downloads" -> 1) ++ $set("last_downloaded" -> new Date, "lastModifiedDate" -> new Date()), true, false, WriteConcern.Safe)
       }
       case None => {}
     }
diff --git a/app/views/dataset.scala.html b/app/views/dataset.scala.html
index 99d4f426c..e09d2003b 100644
--- a/app/views/dataset.scala.html
+++ b/app/views/dataset.scala.html
@@ -372,6 +372,7 @@ <h4>Statistics</h4>
                     <tr><td><b>Last viewed:</b></td><td> @view._2.format("MMM dd, yyyy HH:mm:ss")</td></tr>
                     <tr><td><b>Downloads:</b></td><td> @dataset.stats.downloads</td></tr>
                     <tr><td><b>Last downloaded:</b></td><td> @if(dataset.stats.last_downloaded.isDefined){@dataset.stats.last_downloaded.get.format("MMM dd, yyyy HH:mm:ss")}else{Never}</td></tr>
+                    <tr><td><b>Last Modified:</b></td><td> @dataset.lastModifiedDate.format("MMM dd, yyyy HH:mm:ss")</td></tr>
                 </table>
             </div>
             @spaces.spaceAllocation(dataset.id, ResourceRef.dataset,datasetSpaces_canRemove.getOrElse(Map.empty),None)

From ae5c90f3487811bc44673278415ac3f7abf72e08 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Wed, 14 Dec 2022 14:29:30 -0600
Subject: [PATCH 05/24] Support private extractor registration (#300)

* add private registration endpoint
* add user filter
* unique key + perms
* make extractor key an option
* list user extractors if no process type specified
* Permission check updates
---
 CHANGELOG.md                                  |   5 +
 app/api/Extractions.scala                     | 127 +++++++++---------
 app/controllers/Extractors.scala              |  43 ++++--
 app/controllers/Spaces.scala                  |   3 +-
 app/models/Extraction.scala                   |   9 +-
 app/services/ExtractorRoutingService.scala    |  31 +++--
 app/services/ExtractorService.scala           |   6 +-
 app/services/MessageService.scala             | 100 +++++++++-----
 .../mongodb/MongoDBExtractorService.scala     | 116 +++++++++++-----
 app/views/updateExtractors.scala.html         |   4 +-
 conf/routes                                   |  10 +-
 11 files changed, 288 insertions(+), 166 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c5e98ecde..747cf9e44 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+### Added
+- Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Extractors can now specify an extractor_key and an owner (email address) when sending a
+registration or heartbeat to Clowder that will restrict use of that extractor to them.
+
 ## Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala
index 0ee8701c9..120b02472 100644
--- a/app/api/Extractions.scala
+++ b/app/api/Extractions.scala
@@ -127,36 +127,6 @@ class Extractions @Inject()(
       }
   }
 
-  /**
-   *
-   * Given a file id (UUID), submit this file for extraction
-   */
-  def submitExtraction(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id)))(parse.json) { implicit request =>
-    if (UUID.isValid(id.stringify)) {
-      files.get(id) match {
-        case Some(file) => {
-          // FIXME dataset not available?
-          routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match {
-            case Some(jobId) => {
-              Ok(Json.obj("status" -> "OK", "job_id" -> jobId))
-            }
-            case None => {
-              val message = "No jobId found for Extraction"
-              Logger.error(message)
-              InternalServerError(toJson(Map("status" -> "KO", "message" -> message)))
-            }
-          }
-        }
-        case None => {
-          Logger.error("Could not retrieve file that was just saved.")
-          InternalServerError("Error uploading file")
-        }
-      } //file match
-    } else {
-      BadRequest("Not valid id")
-    }
-  }
-
   /**
    * For a given file id, checks for the status of all extractors processing that file.
    * REST endpoint  GET /api/extractions/:id/status
@@ -404,24 +374,24 @@ class Extractions @Inject()(
     Ok(jarr)
   }
 
-  def listExtractors(categories: List[String]) = AuthenticatedAction  { implicit request =>
-    Ok(Json.toJson(extractors.listExtractorsInfo(categories)))
+  def listExtractors(categories: List[String], space: Option[UUID]) = AuthenticatedAction  { implicit request =>
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
+    Ok(Json.toJson(extractors.listExtractorsInfo(categories, userid)))
   }
 
-  def getExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request =>
-    extractors.getExtractorInfo(extractorName) match {
+  def getExtractorInfo(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request =>
+    extractors.getExtractorInfo(extractorName, extractor_key, request.user) match {
       case Some(info) => Ok(Json.toJson(info))
       case None => NotFound(Json.obj("status" -> "KO", "message" -> "Extractor info not found"))
     }
   }
 
-  def deleteExtractor(extractorName: String) = ServerAdminAction { implicit request =>
-    extractors.deleteExtractor(extractorName)
+  def deleteExtractor(extractorName: String, extractor_key: Option[String]) = ServerAdminAction { implicit request =>
+    extractors.deleteExtractor(extractorName, extractor_key)
     Ok(toJson(Map("status" -> "success")))
   }
 
-  def addExtractorInfo() = AuthenticatedAction(parse.json) { implicit request =>
-
+  def addExtractorInfo(extractor_key: Option[String], user: Option[String]) = AuthenticatedAction(parse.json) { implicit request =>
     // If repository is of type object, change it into an array.
     // This is for backward compatibility with requests from existing extractors.
     var requestJson = request.body \ "repository" match {
@@ -438,34 +408,66 @@ class Extractions @Inject()(
         BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors)))
       },
       info => {
-        extractors.updateExtractorInfo(info) match {
-          case Some(u) => {
-            // Create/assign any default labels for this extractor
-            u.defaultLabels.foreach(labelStr => {
-              val segments = labelStr.split("/")
-              val (labelName, labelCategory) = if (segments.length > 1) {
-                (segments(1), segments(0))
-              } else {
-                (segments(0), "Other")
+        // Check private extractor flags
+        val submissionInfo: Option[ExtractorInfo] = extractor_key match {
+          case Some(ek) => {
+            user match {
+              case None => {
+                Logger.error("Extractors with a private key must also specify a user email.")
+                None
               }
-              extractors.getExtractorsLabel(labelName) match {
-                case None => {
-                  // Label does not exist - create and assign it
-                  val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name))
-                }
-                case Some(lbl) => {
-                  // Label already exists, assign it
-                  if (!lbl.extractors.contains(u.name)) {
-                    val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name))
-                    val updatedLabel = extractors.updateExtractorsLabel(label)
+              case Some(userEmail) => {
+                userservice.findByEmail(userEmail) match {
+                  case Some(u) => {
+                    val perms = List(new ResourceRef('user, u.id))
+                    Some(info.copy(unique_key=Some(ek), permissions=perms))
+                  }
+                  case None => {
+                    Logger.error("No user found with email "+userEmail)
+                    None
                   }
                 }
               }
-            })
+            }
+          }
+          case None => Some(info)
+        }
+
+        // TODO: Check user permissions if the extractor_key has already been registered
+
+        submissionInfo match {
+          case None => BadRequest("Extractors with a private key must also specify a non-anonymous user.")
+          case Some(subInfo) => {
+            extractors.updateExtractorInfo(subInfo) match {
+              case Some(u) => {
+                // Create/assign any default labels for this extractor
+                u.defaultLabels.foreach(labelStr => {
+                  val segments = labelStr.split("/")
+                  val (labelName, labelCategory) = if (segments.length > 1) {
+                    (segments(1), segments(0))
+                  } else {
+                    (segments(0), "Other")
+                  }
+                  extractors.getExtractorsLabel(labelName) match {
+                    case None => {
+                      // Label does not exist - create and assign it
+                      val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name))
+                    }
+                    case Some(lbl) => {
+                      // Label already exists, assign it
+                      if (!lbl.extractors.contains(u.name)) {
+                        val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name))
+                        val updatedLabel = extractors.updateExtractorsLabel(label)
+                      }
+                    }
+                  }
+                })
 
-            Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+                Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+              }
+              case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info"))
+            }
           }
-          case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info"))
         }
       }
     )
@@ -518,11 +520,14 @@ class Extractions @Inject()(
           }
           // if extractor_id is not specified default to execution of all extractors matching mime type
           (request.body \ "extractor").asOpt[String] match {
-            case Some(extractorId) =>
+            case Some(extractorId) => {
+              val extractorKey = (request.body \ "extractor").asOpt[String]
+              extractors.getExtractorInfo(extractorId, extractorKey, request.user)
               val job_id = routing.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra,
                 datasetId, newFlags, request.apiKey, request.user)
               sink.logSubmitFileToExtractorEvent(file, extractorId, request.user)
               Ok(Json.obj("status" -> "OK", "job_id" -> job_id))
+            }
             case None => {
               routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match {
                 case Some(job_id) => {
diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala
index b3e63e948..664fcbc3a 100644
--- a/app/controllers/Extractors.scala
+++ b/app/controllers/Extractors.scala
@@ -39,9 +39,9 @@ class Extractors  @Inject() (extractions: ExtractionService,
   /**
    * Gets a map of all updates from all jobs given to this extractor.
    */
-  def showJobHistory(extractorName: String) = AuthenticatedAction { implicit request =>
+  def showJobHistory(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request =>
     implicit val user = request.user
-    extractorService.getExtractorInfo(extractorName) match {
+    extractorService.getExtractorInfo(extractorName, extractor_key, user) match {
       case None => NotFound(s"No extractor found with name=${extractorName}")
       case Some(info) => {
         val allExtractions = extractions.findAll()
@@ -56,9 +56,10 @@ class Extractors  @Inject() (extractions: ExtractionService,
     */
   def selectExtractors() = AuthenticatedAction { implicit request =>
     implicit val user = request.user
-
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
     // Filter extractors by user filters necessary
-    var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty)
+    // TODO: Filter by multiple spaces
+    var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty, userid)
     val selectedExtractors: List[String] = extractorService.getEnabledExtractors()
     val groups = extractions.groupByType(extractions.findAll())
     val allLabels = extractorService.listExtractorsLabels()
@@ -166,7 +167,7 @@ class Extractors  @Inject() (extractions: ExtractionService,
   def manageLabels = ServerAdminAction { implicit request =>
     implicit val user = request.user
     val categories = List[String]("EXTRACT")
-    val extractors = extractorService.listExtractorsInfo(categories)
+    val extractors = extractorService.listExtractorsInfo(categories, None)
     val labels = extractorService.listExtractorsLabels()
 
     Ok(views.html.extractorLabels(labels, extractors))
@@ -211,7 +212,8 @@ class Extractors  @Inject() (extractions: ExtractionService,
 
   def showExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request =>
     implicit val user = request.user
-    val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName)
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
+    val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName)
     targetExtractor match {
       case Some(extractor) => {
         val labels = extractorService.getLabelsForExtractor(extractor.name)
@@ -223,6 +225,7 @@ class Extractors  @Inject() (extractions: ExtractionService,
 
   def showExtractorMetrics(extractorName: String) = AuthenticatedAction { implicit request =>
     implicit val user = request.user
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
 
     val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
     val todaydate = dateFormatter.format(new java.util.Date())
@@ -299,7 +302,7 @@ class Extractors  @Inject() (extractions: ExtractionService,
     }
     Logger.warn("last 10 average: " + lastTenAverage)
 
-    val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName)
+    val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName)
     targetExtractor match {
       case Some(extractor) => Ok(views.html.extractorMetrics(extractorName, average.toString, lastTenAverage.toString, lastweeksubmitted, lastmonthsubmitted))
       case None => InternalServerError("Extractor Info not found: " + extractorName)
@@ -308,11 +311,19 @@ class Extractors  @Inject() (extractions: ExtractionService,
 
   def submitFileExtraction(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, file_id))) { implicit request =>
     implicit val user = request.user
-    val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))
-    val extractors = all_extractors.filter(!_.process.file.isEmpty)
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
     fileService.get(file_id) match {
-
       case Some(file) => {
+        val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid)
+        var extractors = all_extractors.filter(!_.process.file.isEmpty)
+
+        val user_extra = userid match {
+          case Some(uid) => all_extractors.filter(_.permissions.contains(ResourceRef('user, uid)))
+          case None => List.empty
+        }
+
+        extractors = (extractors ++ user_extra).distinct
+
         val foldersContainingFile = folders.findByFileId(file.id).sortBy(_.name)
         var folderHierarchy = new ListBuffer[Folder]()
         if(foldersContainingFile.length > 0) {
@@ -352,7 +363,8 @@ class Extractors  @Inject() (extractions: ExtractionService,
 
   def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
     implicit val user = request.user
-    val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
+    val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid)
     val extractors = all_extractors.filter(!_.process.file.isEmpty)
     datasets.get(ds_id) match {
       case Some(dataset) => {
@@ -372,10 +384,13 @@ class Extractors  @Inject() (extractions: ExtractionService,
 
   def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request =>
     implicit val user = request.user
-    val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"))
-    val extractors = all_extractors.filter(!_.process.dataset.isEmpty)
+    val userid = request.user.map(u => Some(u.id)).getOrElse(None)
     datasetService.get(ds_id) match {
-      case Some(ds) => Ok(views.html.extractions.submitDatasetExtraction(extractors, ds))
+      case Some(ds) => {
+        val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid)
+        val extractors = all_extractors.filter(!_.process.dataset.isEmpty)
+        Ok(views.html.extractions.submitDatasetExtraction(extractors, ds))
+      }
       case None => InternalServerError("Dataset not found")
     }
   }
diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala
index a8388812f..49e724aea 100644
--- a/app/controllers/Spaces.scala
+++ b/app/controllers/Spaces.scala
@@ -83,10 +83,11 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
   def selectExtractors(id: UUID) = AuthenticatedAction {
     implicit request =>
       implicit val user = request.user
+      val userid = request.user.map(u => Some(u.id)).getOrElse(None)
       spaces.get(id) match {
         case Some(s) => {
           // get list of registered extractors
-          val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty)
+          val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty, userid)
           // list of extractors enabled globally
           val globalSelections: List[String] = extractors.getEnabledExtractors()
           // get list of extractors registered with a specific space
diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala
index e6b35b503..d38fe3b34 100644
--- a/app/models/Extraction.scala
+++ b/app/models/Extraction.scala
@@ -78,6 +78,7 @@ case class ExtractorDetail(
  *
  * @param id id internal to the system
  * @param name lower case, no spaces, can use dashes
+ * @param uniqueName name+suffix to uniquely identify extractor for private use e.g. clowder.extractor.v2.johndoe123
  * @param version the version, for example 1.3.5
  * @param updated date when this information was last updated
  * @param description short description of what the extractor does
@@ -117,7 +118,9 @@ case class ExtractorInfo(
   defaultLabels: List[String] = List[String](),
   process: ExtractorProcessTriggers = new ExtractorProcessTriggers(),
   categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString),
-  parameters: JsValue = JsObject(Seq())
+  parameters: JsValue = JsObject(Seq()),
+  unique_key: Option[String] = None,
+  permissions: List[ResourceRef] =List[ResourceRef]()
 )
 
 /** what are the categories of the extractor?
@@ -170,7 +173,9 @@ object ExtractorInfo {
       (JsPath \ "labels").read[List[String]].orElse(Reads.pure(List.empty)) and
       (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and
       (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and
-      (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq())))
+      (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and
+      (JsPath \ "unique_key").read[Option[String]].orElse(Reads.pure(None)) and
+      (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty))
     )(ExtractorInfo.apply _)
 }
 
diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala
index 545eb4194..4915bb35a 100644
--- a/app/services/ExtractorRoutingService.scala
+++ b/app/services/ExtractorRoutingService.scala
@@ -72,12 +72,12 @@ class ExtractorRoutingService {
    * @param resourceType the type of resource to check
    * @return filtered list of extractors
    */
-  private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value): List[String] = {
+  private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value, user: Option[User] = None): List[String] = {
     val extractorsService = DI.injector.getInstance(classOf[ExtractorService])
 
     extractorIds.flatMap(exId =>
-      extractorsService.getExtractorInfo(exId)).filter(exInfo =>
-      resourceType match {
+      extractorsService.getExtractorInfo(exId, None, None)).filter(exInfo => {
+      val processMatch = resourceType match {
         case ResourceType.dataset =>
           containsOperation(exInfo.process.dataset, operation)
         case ResourceType.file =>
@@ -87,7 +87,17 @@ class ExtractorRoutingService {
         case _ =>
           false
       }
-    ).map(_.name)
+      val permissionMatch = exInfo.unique_key match {
+        case Some(key) => {
+          user match {
+            case None => false // User must be provided for a key-protected extractor
+            case Some(u) => exInfo.permissions.contains(new ResourceRef('user,u.id))
+          }
+        }
+        case None => true
+      }
+      processMatch && permissionMatch
+    }).map(_.name)
   }
 
   /**
@@ -96,15 +106,15 @@ class ExtractorRoutingService {
    * @param operation The dataset operation requested.
    * @return A list of extractors IDs.
    */
-  private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value): (List[String], List[String]) = {
+  private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value, user: Option[User] = None): (List[String], List[String]) = {
     val spacesService = DI.injector.getInstance(classOf[SpaceService])
 
     var enabledExtractors = new ListBuffer[String]()
     var disabledExtractors = new ListBuffer[String]()
     dataset.spaces.map(space => {
       spacesService.getAllExtractors(space).foreach { extractors =>
-        enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType))
-        disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType))
+        enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType, user))
+        disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType, user))
       }
     })
     (enabledExtractors.toList, disabledExtractors.toList)
@@ -145,7 +155,7 @@ class ExtractorRoutingService {
    * @param contentType the content type of the file in the case of a file
    * @return a set of unique rabbitmq queues
    */
-  private def getQueues(dataset: Dataset, routingKey: String, contentType: String): Set[String] = {
+  private def getQueues(dataset: Dataset, routingKey: String, contentType: String, user: Option[User] = None): Set[String] = {
     val extractorsService = DI.injector.getInstance(classOf[ExtractorService])
 
     // drop the first fragment from the routing key and replace characters to create operation id
@@ -160,9 +170,9 @@ class ExtractorRoutingService {
       else
         return Set.empty[String]
     // get extractors enabled at the global level
-    val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType)
+    val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType, user)
     // get extractors enabled/disabled at the space level
-    val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType)
+    val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType, user)
     // get queues based on RabbitMQ bindings (old method).
     val queuesFromBindings = getQueuesFromBindings(routingKey)
     // take the union of queues so that we publish to a specific queue only once
@@ -229,6 +239,7 @@ class ExtractorRoutingService {
     var jobId: Option[UUID] = None
     dataset match {
       case Some(d) => {
+        // TODO: Check private extractor behavior
         getQueues(d, routingKey, file.contentType).foreach { queue =>
           val source = Entity(ResourceRef(ResourceRef.file, file.id), Some(file.contentType), sourceExtra)
 
diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala
index 75acf38e1..d750eaf58 100644
--- a/app/services/ExtractorService.scala
+++ b/app/services/ExtractorService.scala
@@ -35,13 +35,13 @@ trait ExtractorService {
   
   def dropAllExtractorStatusCollection()
 
-  def listExtractorsInfo(categories: List[String]): List[ExtractorInfo]
+  def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo]
 
-  def getExtractorInfo(extractorName: String): Option[ExtractorInfo]
+  def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo]
 
   def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo]
 
-  def deleteExtractor(extractorName: String)
+  def deleteExtractor(extractorName: String, extractorKey: Option[String])
 
   def listExtractorsLabels(): List[ExtractorsLabel]
 
diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala
index 5eb95d557..c6eda512a 100644
--- a/app/services/MessageService.scala
+++ b/app/services/MessageService.scala
@@ -298,6 +298,7 @@ class EventFilter(channel: Channel, queue: String) extends Actor {
  * @param queue
  */
 class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor {
+  val users: UserService = DI.injector.getInstance(classOf[UserService])
   val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService])
   val extractorsService: ExtractorService = DI.injector.getInstance(classOf[ExtractorService])
 
@@ -305,6 +306,7 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor {
     case statusBody: String =>
       Logger.debug("Received extractor heartbeat: " + statusBody)
       val json = Json.parse(statusBody)
+      Logger.debug(json.toString)
       // TODO store running extractors ids
       val id = UUID((json \ "id").as[String])
       val queue = (json \ "queue").as[String]
@@ -313,52 +315,78 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor {
       // Validate document
       val extractionInfoResult = extractor_info.validate[ExtractorInfo]
 
+      // Determine if there is a user associated with this request
+      val owner = (json \ "owner").as[String]
+      val user: Option[User] = if (owner.length > 0) {
+        users.findByEmail(owner)
+      } else {
+        None
+      }
+
       // Update database
       extractionInfoResult.fold(
-        errors => {
-          Logger.debug("Received extractor heartbeat with bad format: " + extractor_info)
-        },
+        errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info),
         info => {
-          extractorsService.getExtractorInfo(info.name) match {
-            case Some(infoFromDB) => {
-              // TODO only update if new semantic version is greater than old semantic version
-              if (infoFromDB.version != info.version) {
-                // TODO keep older versions of extractor info instead of just the latest one
-                extractorsService.updateExtractorInfo(info)
-                Logger.info("Updated extractor definition for " + info.name)
+          if (info.unique_key.isDefined && user.isEmpty) {
+            Logger.error("Extractor keys must have a user associated with them.")
+          } else {
+            extractorsService.getExtractorInfo(info.name, info.unique_key, user) match {
+              case Some(infoFromDB) => {
+                if (info.unique_key.isDefined) {
+                  // Retain existing permissions
+                  val registrationInfo = info.unique_key match {
+                    case Some(ek) => info.copy(permissions=infoFromDB.permissions)
+                    case None => info
+                  }
+                  extractorsService.updateExtractorInfo(registrationInfo)
+                  Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}")
+                } else {
+                  // TODO only update if new semantic version is greater than old semantic version
+                  if (infoFromDB.version != info.version) {
+                    // TODO keep older versions of extractor info instead of just the latest one
+                    extractorsService.updateExtractorInfo(info)
+                    Logger.info(s"Updated extractor definition for ${info.name}")
+                  }
+                }
               }
-            }
-            case None => {
-              extractorsService.updateExtractorInfo(info) match {
-                case None => {}
-                case Some(eInfo) => {
-                  // Create (if needed) and assign default labels
-                  eInfo.defaultLabels.foreach(labelStr => {
-                    val segments = labelStr.split("/")
-                    val (labelName, labelCategory) = if (segments.length > 1) {
-                      (segments(1), segments(0))
-                    } else {
-                      (segments(0), "Other")
-                    }
-                    extractorsService.getExtractorsLabel(labelName) match {
-                      case None => {
-                        // Label does not exist - create and assign it
-                        val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name))
+              case None => {
+                // Inject user into permissions list if a key is given
+                val registrationInfo = info.unique_key match {
+                  case Some(ek) => info.copy(permissions=List(ResourceRef('user, user.get.id)))
+                  case None => info
+                }
+                extractorsService.updateExtractorInfo(registrationInfo) match {
+                  case None => {}
+                  case Some(eInfo) => {
+                    // Create (if needed) and assign default labels
+                    eInfo.defaultLabels.foreach(labelStr => {
+                      val segments = labelStr.split("/")
+                      val (labelName, labelCategory) = if (segments.length > 1) {
+                        (segments(1), segments(0))
+                      } else {
+                        (segments(0), "Other")
                       }
-                      case Some(lbl) => {
-                        // Label already exists, assign it
-                        if (!lbl.extractors.contains(eInfo.name)) {
-                          val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name))
-                          val updatedLabel = extractorsService.updateExtractorsLabel(label)
+                      extractorsService.getExtractorsLabel(labelName) match {
+                        case None => {
+                          // Label does not exist - create and assign it
+                          val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name))
+                        }
+                        case Some(lbl) => {
+                          // Label already exists, assign it
+                          if (!lbl.extractors.contains(eInfo.name)) {
+                            val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name))
+                            val updatedLabel = extractorsService.updateExtractorsLabel(label)
+                          }
                         }
                       }
-                    }
-                  })
+                    })
+                  }
                 }
-              }
 
-              Logger.info(s"New extractor ${info.name} registered from heartbeat")
+                Logger.info(s"New extractor ${info.name} registered from heartbeat with key "+info.unique_key.toString)
+              }
             }
+
           }
         }
       )
diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala
index 039d7df06..3938e34bb 100644
--- a/app/services/mongodb/MongoDBExtractorService.scala
+++ b/app/services/mongodb/MongoDBExtractorService.scala
@@ -1,6 +1,6 @@
 package services.mongodb
 
-import javax.inject.Singleton
+import javax.inject.{Inject, Singleton}
 import com.mongodb.casbah.Imports._
 import com.mongodb.casbah.WriteConcern
 import com.mongodb.casbah.commons.MongoDBObject
@@ -12,11 +12,12 @@ import play.api.Play.current
 import play.api.libs.json.{JsArray, JsNumber, JsObject, JsString, JsValue, Json}
 import services._
 import services.mongodb.MongoContext.context
-
 import org.bson.types.ObjectId
 
 @Singleton
-class MongoDBExtractorService extends ExtractorService {
+class MongoDBExtractorService @Inject() (
+    users: MongoDBUserService
+  ) extends ExtractorService {
 
   def getExtractorServerIPList() = {
     var listServersIPs = List[String]()
@@ -169,51 +170,106 @@ class MongoDBExtractorService extends ExtractorService {
     }
   }
 
-  def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] = {
+  def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] = {
+    Logger.info("listing: "+categories.toString)
     var list_queue = List[ExtractorInfo]()
 
     val allDocs = ExtractorInfoDAO.findAll().sort(orderBy = MongoDBObject("name" -> -1))
     for (doc <- allDocs) {
-      // If no categories are specified, return all extractor names
-      var category_match = categories.isEmpty
-      if (!category_match) {
+      // If no filters are specified, return all extractor names
+      var filter_match = (categories.isEmpty && doc.permissions.isEmpty)
+      if (!filter_match) {
         // Otherwise check if any extractor categories overlap requested categories (force uppercase)
+        val user_match = user match {
+          case Some(u) => {
+            val rr = new ResourceRef('user, u)
+            doc.permissions.contains(rr) || doc.permissions.isEmpty
+          }
+          case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see
+        }
         val upper_categories = categories.map(cat => cat.toUpperCase)
-        category_match = doc.categories.intersect(upper_categories).length > 0
+        val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0
+        filter_match = (category_match && user_match)
       }
 
-      if (category_match)
+      if (filter_match)
         list_queue = doc :: list_queue
     }
 
     list_queue
   }
 
-  def getExtractorInfo(extractorName: String): Option[ExtractorInfo] = {
-    ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName))
+  def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] = {
+    extractorKey match {
+      case Some(ek) => {
+        user match {
+          case None =>  {
+            Logger.error("User authentication required to view extractor info with a unique key.")
+            None
+          }
+          case Some(u) => {
+            val userRef = new ResourceRef('user, u.id)
+            ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek, "permissions" -> userRef))
+          }
+        }
+      }
+      case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false)))
+    }
   }
 
   def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] = {
-    ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match {
-      case Some(old) => {
-        val updated = e.copy(id = old.id)
-        ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe)
-        Some(updated)
-      }
+    // TODO: Make this account for version as well
+    e.unique_key match {
       case None => {
-        ExtractorInfoDAO.save(e)
-        Some(e)
+        ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false))) match {
+          case Some(old) => {
+            val updated = e.copy(id = old.id)
+            ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false)), updated, false, false, WriteConcern.Safe)
+            Some(updated)
+          }
+          case None => {
+            ExtractorInfoDAO.save(e)
+            Some(e)
+          }
+        }
+      }
+      case Some(ek) => {
+        ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match {
+          case Some(old) => {
+            val updated = e.copy(id = old.id)
+            ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> ek), updated, false, false, WriteConcern.Safe)
+            Some(updated)
+          }
+          case None => {
+            ExtractorInfoDAO.save(e)
+            Some(e)
+          }
+        }
       }
     }
   }
 
-  def deleteExtractor(extractorName: String) {
-    ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) match {
-      case Some(extractor) => {
-        ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name))
+  def deleteExtractor(extractorName: String, extractorKey: Option[String]) {
+    extractorKey match {
+      case Some(ek) => {
+        ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) match {
+          case Some(extractor) => {
+            ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> ek))
+          }
+          case None => {
+            Logger.error(s"No extractor found with name ${extractorName} and key ${ek}")
+          }
+        }
       }
       case None => {
-        Logger.info("No extractor found with name: " + extractorName)
+        ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) match {
+          case Some(extractor) => {
+            ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> MongoDBObject("$exists" -> false)))
+          }
+          case None => {
+            Logger.error("No extractor found with name: " + extractorName)
+          }
+        }
       }
     }
   }
@@ -246,15 +302,11 @@ class MongoDBExtractorService extends ExtractorService {
 
   def getLabelsForExtractor(extractorName: String): List[ExtractorsLabel] = {
     var results = List[ExtractorsLabel]()
-    ExtractorInfoDAO.findOne(MongoDBObject("name"->extractorName)) match {
-      case Some(info) => {
-        ExtractorsLabelDAO.findAll().foreach(label => {
-          if (label.extractors.contains(extractorName)) {
-            results = results ++ List[ExtractorsLabel](label)
-          }
-        })
+    ExtractorsLabelDAO.findAll().foreach(label => {
+      if (label.extractors.contains(extractorName) && !results.contains(label)) {
+        results = results ++ List[ExtractorsLabel](label)
       }
-    }
+    })
     results
   }
 }
diff --git a/app/views/updateExtractors.scala.html b/app/views/updateExtractors.scala.html
index 198f709c3..932713ed1 100644
--- a/app/views/updateExtractors.scala.html
+++ b/app/views/updateExtractors.scala.html
@@ -152,7 +152,7 @@ <h1>Extractor Catalog</h1>
                             <th>Name</th>
                             <th>Authors</th>
                             <th>Version</th>
-                            <th>Maturity</th>
+                            <th>Unique Key</th>
                             <!-- th>My Jobs</th -->
                             <th>All Jobs</th>
                             @if(showOptional("ratings")) { <th>Rating</th> }
@@ -187,7 +187,7 @@ <h1>Extractor Catalog</h1>
 
                                     <!-- Extractor maturity (e.g. Development, Staging, Production) -->
                                     <td class="v-center">
-                                        @extractor.maturity
+                                        @extractor.unique_key
                                     </td>
 
                                     <!-- TODO: Extractor Job Counts: My Job Submissions -->
diff --git a/conf/routes b/conf/routes
index 0eb5f62b6..9ab4f1b56 100644
--- a/conf/routes
+++ b/conf/routes
@@ -183,7 +183,7 @@ GET            /extractors/labels
 GET            /extractors/:extractorName                                               @controllers.Extractors.showExtractorInfo(extractorName)
 GET            /extractors/:extractorName/metrics                                       @controllers.Extractors.showExtractorMetrics(extractorName)
 GET            /extractors/:extractorName/logs                                          @controllers.Extractors.showExtractorLog(extractorName)
-GET            /extractors/:extractorName/history                                       @controllers.Extractors.showJobHistory(extractorName)
+GET            /extractors/:extractorName/history                                       @controllers.Extractors.showJobHistory(extractorName, extractor_key: Option[String] ?= None)
 
 
 # ----------------------------------------------------------------------
@@ -423,9 +423,9 @@ POST           /api/files/:id/sendUnarchiveRequest
 # ----------------------------------------------------------------------
 # EXTRACTORS ENDPOINTS
 # ----------------------------------------------------------------------
-GET            /api/extractors                                                          @api.Extractions.listExtractors(categories: List[String] ?= List.empty)
-GET            /api/extractors/:name                                                    @api.Extractions.getExtractorInfo(name: String)
-POST           /api/extractors                                                          @api.Extractions.addExtractorInfo()
+GET            /api/extractors                                                          @api.Extractions.listExtractors(categories: List[String] ?= List.empty, space: Option[UUID] ?= None)
+GET            /api/extractors/:name                                                    @api.Extractions.getExtractorInfo(name: String, extractor_key: Option[String] ?= None)
+POST           /api/extractors                                                          @api.Extractions.addExtractorInfo(extractor_key: Option[String] ?= None, user: Option[String] ?= None)
 POST           /api/extractors/labels                                                   @api.Extractions.createExtractorsLabel()
 PUT            /api/extractors/labels/:id                                               @api.Extractions.updateExtractorsLabel(id: UUID)
 DELETE         /api/extractors/labels/:id                                               @api.Extractions.deleteExtractorsLabel(id: UUID)
@@ -447,7 +447,7 @@ GET            /api/extractions/:id/status
 GET            /api/extractions/:id/metadata                                            @api.Extractions.fetch(id:UUID)
 GET            /api/extractions/:id/statuses                                            @api.Extractions.checkExtractionsStatuses(id:UUID)
 
-DELETE         /api/extractions/:extractorName/delete                                   @api.Extractions.deleteExtractor(extractorName : String)
+DELETE         /api/extractions/:extractorName/delete                                   @api.Extractions.deleteExtractor(extractorName : String, extractor_key: Option[String] ?= None)
 
 DELETE         /api/files/:file_id/extractions/:msg_id                                  @api.Extractions.cancelFileExtractionSubmission(file_id:UUID, msg_id: UUID)
 DELETE         /api/datasets/:ds_id/extractions/:msg_id                                 @api.Extractions.cancelDatasetExtractionSubmission(ds_id:UUID, msg_id: UUID)

From 23148f36c21a28edce119206544fc6608b1ef0f2 Mon Sep 17 00:00:00 2001
From: Dipannita <ddey2@illinois.edu>
Date: Wed, 14 Dec 2022 14:31:23 -0600
Subject: [PATCH 06/24] Added a dropdown menu for specific space types #374
 (#382)

* Added a dropdown menu for specific space types
---
 CHANGELOG.md                           |  1 +
 app/views/main.scala.html              |  8 ++++++--
 app/views/spaces/listSpaces.scala.html |  7 +++++++
 public/javascripts/displayPanels.js    | 19 +++++++++++++++++++
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 747cf9e44..ca9fce978 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
+- Added a dropdown menu to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
 
 ## Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
diff --git a/app/views/main.scala.html b/app/views/main.scala.html
index 2e580ab1a..5bd618415 100644
--- a/app/views/main.scala.html
+++ b/app/views/main.scala.html
@@ -84,7 +84,7 @@
                                 <li><a href="@routes.Datasets.list(showPublic=false)"><span class="glyphicon glyphicon-briefcase"></span> @Messages("datasets.title")</a></li>
                                 <li><a href="@routes.Collections.list(showPublic=false)"><span class="glyphicon glyphicon-th-large"></span> @Messages("collections.title")</a></li>
                             } else {
-                                <li><a href="@routes.Spaces.list(owner=Some(user.get.id.stringify))"><span class="glyphicon glyphicon-hdd"></span> @Messages("spaces.title")</a></li>
+                                <li onclick="setMySpaceTypeSessionStorage()"><a href="@routes.Spaces.list(owner=Some(user.get.id.stringify))"><span class="glyphicon glyphicon-hdd"></span> @Messages("spaces.title")</a></li>
                                 <li><a href="@routes.Datasets.list(owner=Some(user.get.id.stringify))"><span class="glyphicon glyphicon-briefcase"></span> @Messages("datasets.title")</a></li>
                                 <li><a href="@routes.Collections.list(owner=Some(user.get.id.stringify))"><span class="glyphicon glyphicon-th-large"></span> @Messages("collections.title")</a></li>
                             }
@@ -96,7 +96,7 @@
                             Explore <b class="caret"></b>
                         </a>
                         <ul class="dropdown-menu">
-                            <li><a href="@routes.Spaces.list()"><span class="glyphicon glyphicon-hdd"> </span> @Messages("spaces.title")</a></li>
+                            <li><a href="@routes.Spaces.list(showAll=false)"><span class="glyphicon glyphicon-hdd"> </span> @Messages("spaces.title")</a></li>
                             @user match {
                                 case Some(x) => {
                                     @if(play.Play.application().configuration().getBoolean("verifySpaces") && api.Permission.checkServerAdmin(user)) {
@@ -324,6 +324,10 @@
         $.cookie("superAdmin", $.cookie("superAdmin") != "true", { path: '/' });
         location.reload();
     }
+
+    function setMySpaceTypeSessionStorage (){
+        sessionStorage.setItem("#spacesOwnership", "2");
+    }
 </script>
 
 <script>
diff --git a/app/views/spaces/listSpaces.scala.html b/app/views/spaces/listSpaces.scala.html
index 76ad41dd8..1518df0ae 100644
--- a/app/views/spaces/listSpaces.scala.html
+++ b/app/views/spaces/listSpaces.scala.html
@@ -44,6 +44,13 @@ <h1>@title</h1>
                     }
                     case (_,_) => {}
                 }
+                <div class="btn-group btn-group-sm" id="spacesOwnership-displayed-dropdown">
+                    <select id="spacesOwnership" class="form-control" onchange="getSpaceType('@user.get.id')">
+                        <option value="1">Joined Spaces</option>
+                        <option value="2">My Spaces</option>
+                        <option value="3">All Spaces</option>
+                    </select>
+                </div>
                 <div class="btn-group btn-group-sm" id="number-displayed-dropdown">
                     <select id="numPageItems" class="form-control" onchange="getValue()">
                         <option value="12">12</option>
diff --git a/public/javascripts/displayPanels.js b/public/javascripts/displayPanels.js
index 7f503f91d..b8907a53e 100644
--- a/public/javascripts/displayPanels.js
+++ b/public/javascripts/displayPanels.js
@@ -12,6 +12,9 @@ window.onload = function(){
   if (pageSize){
     document.getElementById('numPageItems').value = pageSize;
   }
+  var spaceType = sessionStorage.getItem("#spacesOwnership");
+  if (spaceType)
+    document.getElementById('spacesOwnership').value = spaceType;
 };
 
 function getValue(){
@@ -20,6 +23,22 @@ function getValue(){
   window.location.href = updateQueryStringParameter(url, "size", pageItems);
 };
 
+function getSpaceType(owner){
+  var type =$("#spacesOwnership").val();
+  var url = window.location.href;
+  if (type == 1) {
+    window.location.href = updateQueryStringParameter(url, "showAll", false);
+    sessionStorage.setItem("#spacesOwnership", "1");
+  } else if (type == 2) {
+    window.location.href = updateQueryStringParameter(url, "owner", owner);
+    sessionStorage.setItem("#spacesOwnership", "2");
+  } else {
+    window.location.href = url.substring(0, url.indexOf("?"));
+    sessionStorage.setItem("#spacesOwnership", "3");
+  }
+  updatePage();
+};
+
 function updateQueryStringParameter(uri, key, value) {
   var re = new RegExp("([?&])" + key + "=.*?(&|#|$)", "i");
   if( value === undefined ) {

From f28613938dab739a40a15c128d4caba502474ef2 Mon Sep 17 00:00:00 2001
From: Dipannita <ddey2@illinois.edu>
Date: Wed, 14 Dec 2022 14:32:19 -0600
Subject: [PATCH 07/24] New enhancement to delete a section (#385)

---
 CHANGELOG.md                |  3 ++-
 app/views/file.scala.html   | 25 +++++++++++++++++++++++++
 public/stylesheets/main.css |  8 ++++++--
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca9fce978..a201097e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,9 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## Unreleased
+### Unreleased
 
 ### Added
+- Added Trash button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
 - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
diff --git a/app/views/file.scala.html b/app/views/file.scala.html
index d6788f083..ed6b3d72b 100644
--- a/app/views/file.scala.html
+++ b/app/views/file.scala.html
@@ -483,6 +483,7 @@ <h4>Sections</h4>
 										<th class="galleriaPosition">Position</th>
 										<th class="galleriaDescr">Description</th>
 										<th class="galleriaFindSimilar">Find</th>
+										<th class="galleriaDelete">Delete</th>
 									</tr>
 								</thead>
 								<tbody>
@@ -541,6 +542,9 @@ <h4>Sections</h4>
 											}
 										</td>
 										<td class="galleriaFindSimilar"><a href="@routes.Search.callSearchMultimediaIndexView(s.id)">Similar</a></td>
+										<td class="galleriaDelete">
+											<button class="btn btn-link" title="Delete the section" onclick="removeSection('@s.id')"><span class="glyphicon glyphicon-trash"></span></button>
+										</td>
 									</tr>
 								}
 								</tbody>
@@ -1167,6 +1171,27 @@ <h4>Tags</h4>
 
 							return false;
 						}
+
+						function removeSection(sectionId) {
+							var request = window.jsRoutes.api.Sections.delete(sectionId).ajax({
+								type: 'DELETE'
+							});
+
+							request.done(function (response, textStatus, jqXHR) {
+								console.log("Response " + textStatus);
+								location.reload();
+							});
+
+							request.fail(function (jqXHR, textStatus, errorThrown){
+								console.error("The following error occurred: " + textStatus, errorThrown);
+								var errMsg = "You must be logged in to remove a section from a file.";
+								if (!checkErrorAndRedirect(jqXHR, errMsg)) {
+									notify("The section was not removed from the file due to : " + errorThrown, "error");
+								}
+							});
+
+							return false;
+						}
 						//The removeTag code is almost identical to the code in tags.scala.html. Should be unified.
 						function removeTag() {
 							var tagId = $(this).attr("id");
diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css
index 774cd1737..906bc9a46 100644
--- a/public/stylesheets/main.css
+++ b/public/stylesheets/main.css
@@ -760,11 +760,15 @@ table#sensorsListTable>thead>tr>th.headerSortDown {
 }
 
 #scrollTableGalleria .galleriaDescr {
-	width: 35%;
+	width: 30%;
 }
 
 #scrollTableGalleria .galleriaFindSimilar {
-	width: 20%;
+	width: 15%;
+}
+
+#scrollTableGalleria .galleriaDelete {
+	width: 10%;
 }
 
 #scrollTableGalleria thead {

From 2d8b9295279ef98606ba6f2e55b152ff7cb1e418 Mon Sep 17 00:00:00 2001
From: Rob Kooper <kooper@illinois.edu>
Date: Mon, 11 Sep 2023 12:32:51 -0500
Subject: [PATCH 08/24] documentation on testing (#407)

---
 CHANGELOG.md |  1 +
 TESTING.md   | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 TESTING.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a201097e2..ba104cea9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
 - Added a dropdown menu to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
+- Documentation on how to do easy testing of pull requests
 
 ## Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
diff --git a/TESTING.md b/TESTING.md
new file mode 100644
index 000000000..fdfc36690
--- /dev/null
+++ b/TESTING.md
@@ -0,0 +1,82 @@
+# Testing Clowder PR
+
+Download the [docker-compose.yml](https://raw.githubusercontent.com/clowder-framework/clowder/develop/docker-compose.yml) file in a new folder. Next create a .env file with the following data:
+
+```ini
+COMPOSE_PROJECT_NAME=clowder
+TRAEFIK_HOST=Host:yourmachine.ncsa.illinois.edu;
+TRAEFIK_HTTP_PORT=80
+TRAEFIK_HTTPS_PORT=443
+TRAEFIK_HTTPS_OPTIONS=TLS
+TRAEFIK_ACME_ENABLE=true
+TRAEFIK_ACME_EMAIL=youremail@ncsa.illinois.edu
+TRAEFIK_HTTP_REDIRECT=Redirect.EntryPoint:https
+CLOWDER_SSL=true
+CLOWDER_ADMINS=youremail@ncsa.illinois.edu
+```
+
+Next create a docker-compose.override.yml file:
+
+```yaml
+version: '3.5'
+
+services:
+  # point to the PR image (in this case PR-404)
+  clowder:
+    image: ghcr.io/clowder-framework/clowder:PR-404
+
+  # add any more extractors if you want
+  # extract preview image
+  imagepreview:
+    image: clowder/extractors-image-preview:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+
+  # extract image metadata
+  imagemetadata:
+    image: clowder/extractors-image-metadata:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+
+  # digest
+  digest:
+    image: clowder/extractors-digest:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+```
+
+It is best practice to start with a `docker-compose pull` to make sure you have all the latest versions of the containers, followed by a  `docker-compose up -d`. This will start all containers. You should be able to go to https://yourmachine.ncsa.illinois.edu.
+
+If this is the first time running the stack (or if you removed the mongo database), you will need to create the initial user again:
+
+```bash
+docker run --rm -it \
+    --network clowder_clowder \
+    -e "FIRSTNAME=Admin" \
+    -e "LASTNAME=User" \
+    -e "ADMIN=true" \
+    -e "PASSWORD=areallygoodpassword" \
+    -e "EMAIL_ADDRESS=youremail@ncsa.illinois.edu" \
+    -e "MONGO_URI=mongodb://mongo:27017/clowder" \
+    clowder/mongo-init
+```
+
+

From 5d7c19e06876df87c8a86e6d73cc4bf43d9e0c33 Mon Sep 17 00:00:00 2001
From: Chen Wang <cwang138@illinois.edu>
Date: Mon, 11 Sep 2023 14:17:24 -0500
Subject: [PATCH 09/24] 419 authenticator against keycloak (#420)

* follow cilogon example

* add keycloak

* keycloak is working now

* changelog

* revert accidental change

* replace the default conf to clowder

* typo

* use the correct group

* check roles also

* provide examples of filtering by grouops and roles

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                                  |   1 +
 app/services/KeycloakProvider.scala           | 103 ++++++++++++++++++
 conf/play.plugins                             |   1 +
 conf/securesocial.conf                        |  12 ++
 .../images/providers/keycloak.png             | Bin 0 -> 2924 bytes
 5 files changed, 117 insertions(+)
 create mode 100644 app/services/KeycloakProvider.scala
 create mode 100644 public/securesocial/images/providers/keycloak.png

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba104cea9..22577f118 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
 - Added a dropdown menu to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
+- Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419)
 - Documentation on how to do easy testing of pull requests
 
 ## Fixed
diff --git a/app/services/KeycloakProvider.scala b/app/services/KeycloakProvider.scala
new file mode 100644
index 000000000..dab72cda6
--- /dev/null
+++ b/app/services/KeycloakProvider.scala
@@ -0,0 +1,103 @@
+package services
+
+import play.api.libs.ws.WS
+import play.api.{Application, Logger}
+import play.api.libs.json.JsObject
+import securesocial.core._
+import scala.collection.JavaConverters._
+
+
+/**
+ * A Keycloak OAuth2 Provider
+ */
+class KeycloakProvider(application: Application) extends OAuth2Provider(application) {
+  val Error = "error"
+  val Message = "message"
+  val Type = "type"
+  val Sub = "sub"
+  val Name = "name"
+  val GivenName = "given_name"
+  val FamilyName = "family_name"
+  //  todo: picture wont work
+  val Picture = "picture"
+  val Email = "email"
+  val Groups = "groups"
+
+  override def id = KeycloakProvider.Keycloak
+
+  def fillProfile(user: SocialUser): SocialUser = {
+    val UserInfoApi = loadProperty("userinfoUrl").getOrElse(throwMissingPropertiesException())
+    val accessToken = user.oAuth2Info.get.accessToken
+    val promise = WS.url(UserInfoApi.toString).withHeaders(("Authorization", "Bearer " + accessToken)).get()
+
+    try {
+      val response = awaitResult(promise)
+      val me = response.json
+      Logger.debug("Got back from Keycloak : " + me.toString())
+      (me \ Error).asOpt[JsObject] match {
+        case Some(error) =>
+          val message = (error \ Message).as[String]
+          val errorType = ( error \ Type).as[String]
+          Logger.error("[securesocial] error retrieving profile information from Keycloak. Error type = %s, message = %s"
+            .format(errorType,message))
+          throw new AuthenticationException()
+        case _ =>
+          val userId = (me \ Sub).as[String]
+          val firstName = (me \ GivenName).asOpt[String]
+          val lastName = (me \ FamilyName).asOpt[String]
+          val fullName = (me \ Name).asOpt[String]
+          val avatarUrl = ( me \ Picture).asOpt[String]
+          val email = ( me \ Email).asOpt[String]
+          val groups = ( me \ Groups).asOpt[List[String]]
+          val roles = ( me \ "resource_access" \ "account" \ "roles").asOpt[List[String]]
+          (application.configuration.getList("securesocial.keycloak.groups"), groups) match {
+            case (Some(conf), Some(keycloak)) => {
+              val conflist = conf.unwrapped().asScala.toList
+              if (keycloak.intersect(conflist).isEmpty) {
+                throw new AuthenticationException()
+              }
+            }
+            case (Some(_), None) => throw new AuthenticationException()
+            case (None, _) => Logger.debug("[securesocial] No check needed for groups")
+          }
+          (application.configuration.getList("securesocial.keycloak.roles"), roles) match {
+            case (Some(conf), Some(keycloak)) => {
+              val conflist = conf.unwrapped().asScala.toList
+              if (keycloak.intersect(conflist).isEmpty) {
+                throw new AuthenticationException()
+              }
+            }
+            case (Some(_), None) => throw new AuthenticationException()
+            case (None, _) => Logger.debug("[securesocial] No check needed for roles")
+          }
+          user.copy(
+            identityId = IdentityId(userId, id),
+            firstName = firstName.getOrElse(""),
+            lastName = lastName.getOrElse(""),
+            fullName = fullName.getOrElse({
+              if (firstName.isDefined && lastName.isDefined) {
+                firstName.get + " " + lastName.get
+              } else if (firstName.isDefined) {
+                firstName.get
+              } else if (lastName.isDefined) {
+                lastName.get
+              } else {
+                ""
+              }
+            }),
+            avatarUrl = avatarUrl,
+            email = email
+          )
+      }
+    } catch {
+      case e: Exception => {
+        Logger.error( "[securesocial] error retrieving profile information from Keycloak", e)
+        throw new AuthenticationException()
+      }
+    }
+  }
+}
+
+object KeycloakProvider {
+  val Keycloak = "keycloak"
+}
diff --git a/conf/play.plugins b/conf/play.plugins
index 15e5e7ec4..faaae5ba4 100644
--- a/conf/play.plugins
+++ b/conf/play.plugins
@@ -21,6 +21,7 @@
 #10050:services.CrowdProvider
 #10051:services.CILogonProvider
 #10052:services.LdapProvider
+#10053:services.KeycloakProvider
 #10090:services.MailerPlugin
 #10091:services.AdminsNotifierPlugin
 #10100:services.TempFilesPlugin
diff --git a/conf/securesocial.conf b/conf/securesocial.conf
index 9afb107b9..afa4fc019 100644
--- a/conf/securesocial.conf
+++ b/conf/securesocial.conf
@@ -161,6 +161,18 @@ securesocial {
         #groups=["cn=org_isda,ou=Groups,dc=ncsa,dc=illinois,dc=edu"]
     }
 
+    keycloak {
+        authorizationUrl="http://localhost:8080/keycloak/realms/clowder/protocol/openid-connect/auth"
+        accessTokenUrl="http://localhost:8080/keycloak/realms/clowder/protocol/openid-connect/token"
+        userinfoUrl="http://localhost:8080/keycloak/realms/clowder/protocol/openid-connect/userinfo"
+        clientId=your_client_id
+        clientSecret=your_client_secret
+        scope="profile email roles"
+        # Example of filtering by groups and/or roles
+        # groups=["group1", "group2"]
+        # roles=["role1", "role2"]
+    }
+
     ldap {
         url="http://localhost/ldap"
         hostname="ldap.example.com"
diff --git a/public/securesocial/images/providers/keycloak.png b/public/securesocial/images/providers/keycloak.png
new file mode 100644
index 0000000000000000000000000000000000000000..7387557f708367460a65444aef81fdcdf5fcc945
GIT binary patch
literal 2924
zcmY*bc|4SB8-DE>oRG4$j3L`$#@0eJB1?uw3_>V7lMKe#WlfVD`*JKXgOQ1d5|U*i
zhO!NvL_^skyF_;1sJ_nkJ@4;%@9Vjr>$>jee&2tdSQF!md>}Cp004YQ16?#TD(%k$
zoXoc+>jr@t*xj^I+5k|Q$g}Oh!CZr#4A3Y52$lwbh}!_L$xKB|1Are>IAae08kqnf
z?0vt^6wchZ?P7&=MWFzBCO!agum}QdOvJ)`02VP|-;Id?Ll*Jh7|kO4i^B>41Ps9b
zi*uD3_eUUe?rVNVHWJIfjwIIKY$X!gzj$Akn^Wz_4BXxZ*1iBBh~J+qBd#ZyHZ0Z{
zb1S?R>XL?&mnYQ0*~`%d8tCc0uL9tK8cgWvf_DH1dfvqPY6K!6KNuQJyq|_az&|K>
z4+O*tWdhdm!nuG?LzSRP5KRym42I*JT{Y0U`oHAN6$0Xp$9rqQU;zOE&;V7a7tRf)
ztgfyOQ&NGcs3<ZSioQWuyhETO)>rbU$$x!xU3{Hz7;ikr3k%-&b#V0Z!y_P&{X)OT
z&vW82uK!lT`u=K**&u9x2c`^Fg8lYpO5yuijmtQU3$yaRzos($2lM}Ozv{qY`^Ep~
zFh4u}k!7~334+6Ze>P2!=(SNU=Fgu;>S~+2bIe<L+%%^h-gR9VZkV-m8(16ipn*jG
z(vdYD(nzW@q*NG{Mw#%wibw-Jt3sJdLdMMFqbLOhyY5%@DCaj#w98CP4?oum%qo#9
zbEf1NTmh3g8CCWMV0&>`TieLMEmY6*S-+d(S{0<7r<Qdgi=Fq?S8^BOt_!RifA(nQ
zMZ(#FIKT2vh_<ULd2+rgbHxtVSsX_Mn`fk_%Vo?iE;b?aXt*7z*`!RFkUtytN-J}y
zVXtF~&n=pj5M|epaXw8$6%Nsho2#>?AF79A`?1(Bb>(YwgZ<$v?G5u;uYrZ}4|bJ~
zt;Pd>#7O=DiVhKU4`RwPDO+@}<L-&Mk4<})JVz9ka?koSXB+Nfzjt(iW@l$l%}6ex
zuBW2Qh0$Uc+6@g9IZ=t+64RW+D$*3iQ!{@L?XYWV<!vsPSd=>Q=B>CrSqP!$b(WBH
zt71_|tuRBdTvD3cBdHEk><|Vla&pgLjB!UKq2<}J1Cj@dm2PnI7QXTsJ|WN>kDn#e
z9#mw%e$)S?sI=5bK>1K}P}J8`9`DvM+P?GxJ#oq8c9`k|7x8BL<1R0gTY8fPLQqGq
zXH-1Lb|im!Ip&T+oCIU}jC=S-F3zyz_UED6_x&ra<=x#0nEd)KskOO4$u?e+aV~Z%
zMz}hy7`<%#!9sbsqjuROhVh+Rs6}Sa+!0mq_*m-dH!n;R_I!2Dk8eHn{XtKqgrW0|
zoW$Jf80@sB-MQ-E)fumAjiKt(XA{QQPb9uk>sm&w?}R<hot^Su$mO!)LWFiWtBf!h
zjEi|i21oL`mDwm}g~n4$qb=u*3h1k`qYkN3+fDPwUoJ$d$)DI%DAz4mZ_r50`fRbS
zLyWa#;NEW7**3y%FY;Y)J`{^}CXddb4X&l(nt5mpM{|B^K(+qF&DR&aWu-8G{1a!J
zQzMjM-+Juys`wZDW;2zlHT=0Nhk(ml5-ck#L%A(+{c&_1efH=zVu!Oih1%OAE#yx#
zqsTp`rhCg>!<9R7-btcITah*^Y^8nrGALD->O~(>{R-K&?f65xbda4}(|ywqJ-!*~
zy$reW_S0#;ezWoIvDN1-sS?zY)X6W7j(pI`*4Eaia#b}I4spV;wTh7Qe0+ST1H|qu
zmrNNCR1*5C8LKb(H>d>P$)_e|DuL*4El&<#GPEV)dMUwApXWYVZtU&Fng=RmyXp@m
zj0+Yz)wpDpdxkR}NZhc&N~iiOz7b1((<N~^NH`|tjPVj9Iawe{8;1Q*?Fn@nX$Yy2
zPCRjb!D)%>;PGvLB(ftlR<~~r(>FjT>eKASR^;7MTqHs*?evl>+W!_)1D!X+P;i7L
zg`kP5!omzj6YcKU`McDqt3n{OpzlbRi;>)4&tk2iUPoOoeX=_L#y7_cbPV|@M5Qpe
z*70ggOpIpT^`Zc8i?6KMt+n|g3kwT3Rt^s3v(^1X|7WLiebP8|rinI$Nr;YLrMPB_
z>zbU>Lkej9lf4$_^05$U2%02g60vznP`tkwGhpW3pk$e6$+#IkFj@P+*1JA*>89xX
zxN<jJ;|7FHy4|E!F2(X&T6A<YD(AH45bYgA2$V&}XGSJ_<O@^~ZVB$B|C1>$7ty3u
z%xXA&aV0?{-})`MnW6#?`h3h;b2H*f+TE!4GTwi%xK=6*@^(tEl5GkRx0@52+l;L4
z9XjZKMW(Prti8w9Z%mnWUbOq#==1Wl7KQPc-a5ZoEPv9X$AopNkUlxyt237Vz~XkZ
zv)I;7OrRyKI5!p@qqR~XRArD_5RDMMXB-=KPjoIY^rWwVSmeoxic36GbuZ}sA6hDT
zN+8pk;m4QPPj#Q^RevKH@p5%-ZLF`zs1`SQk1K89E^V9y`P5;8Z0n5|_g50pl}{fW
z+}>UZpFG+t47GLlsZUNPT(z;;6H3s{kh!Di+nBrtGvnS?KSV|&wb+!+i#PeZ8Cu!R
zYGj`K#|6&f5%JlgPrb{*QB!??n>zeK6$+ayE?FDNQk`i}KBFgPOU}Cy3Sv4_>D8xA
z?GP7Fsl6R9tNEyJjgc#v>eQYcZ6W2_GDMDFG0IR_7TMdXbh4_uDcYTPPR2MF{-v}0
zq4M$YxyqLX1(3&rm0V|!P%|Fcy?8hH%wwR1?*1*U=YUp?*+P(_JkhUoiByq%fKZ`A
zCp^A3ONO|CQ4(K=2uDT^S8*fE2^abJU?(q2*h^y=#hjew-15Bkj|Gc_q6T|Oby@K~
z39y;(&|u+j##8b=FYZe||MPoW@Y$mssy+0`<ujq^kxD1Rq$styUf3IR1B2O6w|);#
zG03b?-EH7lp^qgBXo0fqSsGXANOQjMyZ)sInx>Hs{#|g}yB4F9FT))iHsSh5?YLdy
zzV8kOeKB_{*4qMErYKl(B7&9MZ5fH-J@JW%K@n|UR)>^DFk0*)&C0qO<inL4t){`{
zvq{L#&er%cVIBX9<TSiT@^^2IquvsLLsTj6w>&J~{v!>2r=U-pYUJ%K)p7Z#2d94p
z6-7DFcPAvNTx$)>ZT)0bH8nM|?1-=KS9E_JYh;8}sAOR(VrhF<`Q+R7W1lmicAI~V
zes3~%<Gy(L)w6sWQGc)VYTEZaJ@Ss9e4Fz06;Q)<_Y08IY=?_K2&@OZK<A?+0z*>1
zCQFu;Eha+!Yfd<QvR-CqtBqzL^8{~9zE1ZP@?Fg+?8A|JU?~Vgj@S9khoxQ#pkiDn
z<dbG%#Ix93ZY#FPu$kWpzo5L?Vyd$K;?sju^&S1|5_WSP+#~n55x{G<T~dDLO2b^;
z6@oOsNwOiuQ9hO3R^wa#q-W+^g)l(&Aq2w{Hbg@8*R|6G+OF8xy!pr)8b+)DCk}Uc
z82aNjdf&faSG_tnJCV*JIB`%&Qz9^;KH!N&T!I}0V%}5J8@Y6Fb#+xETx1wUB9c7}
z+1hwiMX-r`Th7~o<-`eF#~i0PzY(){w8!D40h~A+HW|5EvkhebVtvpHj@DeMQ3}_>
nQN1wz|GiWsxqaGep@y@_wmCf-nfY*e{|<@NGuAD?U?24#d0Z;5

literal 0
HcmV?d00001


From 6a89ba8cd24f00d8da1467a25d6433956e5748cb Mon Sep 17 00:00:00 2001
From: Rob Kooper <kooper@illinois.edu>
Date: Mon, 11 Sep 2023 14:21:15 -0500
Subject: [PATCH 10/24] add citation.cff (#402)

* add citation.cff

* remove extra title

* fix citation

* add grants

* fix version number

* remove leftover "

* Updated Todd's affiliation in citation.

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md |   5 ++
 citation.cff | 207 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 212 insertions(+)
 create mode 100644 citation.cff

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22577f118..cacd2da78 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,11 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
 
+## Unreleased
+
+### Added
+- added a citation.cff file
+
 ## 1.21.0 - 2022-08-23
 
 **_Important:_** This update requires a MongoDB update schema due to a bug in the original migration  of showing summary statistics at the
diff --git a/citation.cff b/citation.cff
new file mode 100644
index 000000000..e0affaf67
--- /dev/null
+++ b/citation.cff
@@ -0,0 +1,207 @@
+cff-version: 1.2.0
+message: If you use this software, please cite it using these metadata.
+title: "Clowder: Open Source Data Management for Long Tail Data"
+abstract: "A customizable and scalable data management system you can install in the cloud or on your own hardware."
+type: software
+version: "1.21.0"
+license: "NCSA"
+repository-code: "https://github.com/clowder-framework/clowder"
+keywords:
+  - data-management
+  - cyberinfrastructure
+  - clowder
+  - open-data
+  - open-science
+preferred-citation:
+  type: article
+  title: "Clowder: Open Source Data Management for Long Tail Data"
+  abstract: "Clowder is an open source data management system to support data curation of long tail data and metadata across multiple research domains and diverse data types. Institutions and labs can install and customize their own instance of the framework on local hardware or on remote cloud computing resources to provide a shared service to distributed communities of researchers. Data can be ingested directly from instruments or manually uploaded by users and then shared with remote collaborators using a web front end. We discuss some of the challenges encountered in designing and developing a system that can be easily adapted to different scientific areas including digital preservation, geoscience, material science, medicine, social science, cultural heritage and the arts. Some of these challenges include support for large amounts of data, horizontal scaling of domain specific preprocessing algorithms, ability to provide new data visualizations in the web browser, a comprehensive Web service API for automatic data ingestion and curation, a suite of social annotation and metadata management features to support data annotation by communities of users and algorithms, and a web based front-end to interact with code running on heterogeneous clusters, including HPC resources."
+  isbn: 9781450364461
+  publisher:
+    name: "Association for Computing Machinery"
+  doi: 10.1145/3219104.3219159
+  collection-title: "Proceedings of the Practice and Experience on Advanced Research Computing"
+  keywords:
+    - scientific gateways
+    - metadata management
+    - linked data
+    - data management
+    - data curation
+  location:
+    name: "Pittsburgh, PA, USA"
+  year: 2018
+  authors:
+    - family-names: Marini
+      given-names: Luigi
+    - family-names: Gutierrez-Polo
+      given-names: Indira
+    - family-names: Kooper
+      given-names: Rob
+    - family-names: Satheesan
+      given-names: Sandeep Puthanveetil
+    - family-names: Burnette
+      given-names: Maxwell
+    - family-names: Lee
+      given-names: Jong
+    - family-names: Nicholson
+      given-names: Todd
+    - family-names: Zhao
+      given-names: Yan
+    - family-names: McHenry
+      given-names: Kenton
+references:
+  - institution: "National Science Foundation" 
+    number: "#BCS-0941268" 
+  - institution: "National Science Foundation"  
+    number: "#EAR- 331906"
+  - institution: "National Science Foundation"  
+    number: "#ACI-1261582"
+  - institution: "National Science Foundation"  
+    number: "#ACI-1443013"
+  - institution: "National Science Foundation"  
+    number: "#OCI-0940824"
+  - institution: "National Science Foundation"  
+    number: "#OCI-0525308"
+  - institution: "National Science Foundation"  
+    number: "#OAC-1835834"
+  - institution: "National Institutes of Health"
+    number: "#1P01AI089556-01A1"
+  - institution: "Illinois - Indiana Sea Grant"
+    number: "#DW92329201"
+  - institution: "European Commission"
+    number: "#RI-261600"
+  - institution: "XSEDE"
+    number: "#OCI-1053575"
+  - institution: "ARPA-E"
+    number: "#DE-AR0000594"
+authors:
+  - family-names: "Marini"
+    given-names: "Luigi"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+    orcid: "0000-0002-8511-0211"
+  - family-names: "Kooper"
+    given-names: "Rob"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+    orcid: "0000-0002-5781-7287"
+  - family-names: "Gutierrez"
+    given-names: "Indira"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+    orcid: "0000-0001-5684-3419"
+  - family-names: "Sophocleous"
+    given-names: "Constantinos"
+  - family-names: "Burnette"
+    given-names: "Max"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Nicholson"
+    given-names: "Todd"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Ondrejcek"
+    given-names: "Michal"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Zhang"
+    given-names: "Bing"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Zharnitsky"
+    given-names: "Inna"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Puthanveetil Satheesan"
+    given-names: "Sandeep Puthanveetil"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+    orcid: "0000-0001-9075-3740"
+  - family-names: "Padhy"
+    given-names: "Smruti"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Zhao"
+    given-names: "Yan"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Liu"
+    given-names: "Rui"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Vaidya"
+    given-names: "Ashwini"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Myers"
+    given-names: "Jim"
+    orcid: "0000-0001-8462-650X"
+  - family-names: "Felarca"
+    given-names: "Mario"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Angelo"
+    given-names: "Brock"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Roeder"
+    given-names: "Gene"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Lee"
+    given-names: "Jong"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Hennessy"
+    given-names: "Will"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Issaranon"
+    given-names: "Theerasit"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Guo"
+    given-names: "Yibo"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Yuan"
+    given-names: "Xiaocheng"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Kethineedi"
+    given-names: "Varun"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Kumar"
+    given-names: "Avinash"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Nayudu"
+    given-names: "Nishant"
+    affiliation: "University of Illinois at Urbana-Champaign"
+  - family-names: "Poelmans"
+    given-names: "Ward"
+    affiliation: "Center for Molecular Modeling, Ghent University"
+  - family-names: "Jansz"
+    given-names: "Winston"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Jansen"
+    given-names: "Gregory"
+    affiliation: "College of Information Studies, University of Maryland"
+  - family-names: "Navarro"
+    given-names: "Chris"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Pitcel"
+    given-names: "Michelle"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Tenczar"
+    given-names: "Nicholas"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Wang"
+    given-names: "Chen"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Lambert"
+    given-names: "Mike"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "McHenry"
+    given-names: "Kenton"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+    orcid: "0000-0003-0367-2550"
+  - family-names: "Habib"
+    given-names: "Aaraj"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Galewsky"
+    given-names: "Ben"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Constantinou"
+    given-names: "Chrysovalantis"
+  - family-names: "Karimi-Asli"
+    given-names: "Kaveh"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Tzima"
+    given-names: "Maria-Spyridoula"
+  - family-names: "Johnson"
+    given-names: "Michael"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Bobak"
+    given-names: "Mike"
+    affiliation: "National Center for Supercomputing Applications, University of Illinois at Urbana-Champaign"
+  - family-names: "Yardley"
+    given-names: "Tim"

From 8517d91c3657dbd7a21fd9d19bbfc0e5ecae8f3e Mon Sep 17 00:00:00 2001
From: Bing Zhang <bing@illinois.edu>
Date: Mon, 11 Sep 2023 14:21:44 -0500
Subject: [PATCH 11/24] add missing SMTP_FROM in docker-comopse yml file (#418)

* add missing SMTP_FROM in docker-comopse yml file

* add smtp.from in custom.conf

* add default smtp.from in custom.conf

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md       | 1 +
 docker-compose.yml | 1 +
 docker/custom.conf | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cacd2da78..41efc4ab3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
 - Added a dropdown menu to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
+- Add SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417)
 - Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419)
 - Documentation on how to do easy testing of pull requests
 
diff --git a/docker-compose.yml b/docker-compose.yml
index 45b90a271..1e9f05fcb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -67,6 +67,7 @@ services:
       - RABBITMQ_CLOWDERURL=${RABBITMQ_CLOWDERURL:-http://clowder:9000}
       - SMTP_MOCK=${SMTP_MOCK:-true}
       - SMTP_SERVER=${SMTP_SERVER:-smtp}
+      - SMTP_FROM=${SMTP_FROM:-devnull@ncsa.illinois.edu}
       - CLOWDER_STORAGE=${CLOWDER_STORAGE:-services.filesystem.DiskByteStorageService}
       - CLOWDER_DISKPATH=${CLOWDER_DISKPATH:-/home/clowder/data}
       - S3_ENDPOINT=${S3_ENDPOINT:-http://minio:9000}
diff --git a/docker/custom.conf b/docker/custom.conf
index a7f02f2f1..52f1fb090 100644
--- a/docker/custom.conf
+++ b/docker/custom.conf
@@ -32,6 +32,8 @@ smtp.mock=true
 smtp.mock=${?SMTP_MOCK}
 smtp.host="smtp"
 smtp.host=${?SMTP_SERVER}
+smtp.from="devnull@ncsa.illinois.edu"
+smtp.from=${?SMTP_FROM}
 
 # storage
 service.byteStorage=services.filesystem.DiskByteStorageService

From 7204558de811e79999c9ffcc2089eef8c0d88354 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Sep 2023 14:22:29 -0500
Subject: [PATCH 12/24] Bump certifi from 2021.5.30 to 2022.12.7 in
 /doc/src/sphinx (#399)

Bumps [certifi](https://github.com/certifi/python-certifi) from 2021.5.30 to 2022.12.7.
- [Release notes](https://github.com/certifi/python-certifi/releases)
- [Commits](https://github.com/certifi/python-certifi/compare/2021.05.30...2022.12.07)

---
updated-dependencies:
- dependency-name: certifi
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 doc/src/sphinx/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/src/sphinx/requirements.txt b/doc/src/sphinx/requirements.txt
index d962a2340..4e4862029 100644
--- a/doc/src/sphinx/requirements.txt
+++ b/doc/src/sphinx/requirements.txt
@@ -3,7 +3,7 @@ sphinx-rtd-theme==0.5.0
 sphinx_design==0.0.13
 alabaster==0.7.12
 babel==2.9.1
-certifi==2021.5.30
+certifi==2022.12.7
 chardet==4.0.0
 commonmark==0.9.1
 docutils==0.16

From e09f44fd36b5484ef55eaba8a28e2c16e50d2396 Mon Sep 17 00:00:00 2001
From: Sandeep Puthanveetil Satheesan <sandeeps@illinois.edu>
Date: Mon, 11 Sep 2023 14:24:30 -0500
Subject: [PATCH 13/24] #395 Fix previewer source URL (#396)

* Fix previewers source code folder URL.

* Update CHANGELOG.md

* Update CHANGELOG.md

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                          | 9 ++++-----
 doc/src/sphinx/develop/previewers.rst | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 41efc4ab3..4117fd2e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-### Unreleased
+
+## Unreleased
 
 ### Added
 - Added Trash button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
@@ -16,15 +17,13 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 - Add SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417)
 - Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419)
 - Documentation on how to do easy testing of pull requests
+- Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
+- Added a citation.cff file
 
 ## Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
 
-## Unreleased
-
-### Added
-- added a citation.cff file
 
 ## 1.21.0 - 2022-08-23
 
diff --git a/doc/src/sphinx/develop/previewers.rst b/doc/src/sphinx/develop/previewers.rst
index d1f87488b..b9dc3da31 100644
--- a/doc/src/sphinx/develop/previewers.rst
+++ b/doc/src/sphinx/develop/previewers.rst
@@ -12,4 +12,4 @@ Previewer can work together with extractors and external services.
 
 
 Here is a list of previewer embedded with the core
-`source <https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS/repos/clowder/browse/public/javascripts/previewers>`_.
\ No newline at end of file
+`source <https://github.com/clowder-framework/clowder/tree/master/public/javascripts/previewers>`_.

From e57bf562af657a84e6e09b5acef2b558a294f7b4 Mon Sep 17 00:00:00 2001
From: Rob Kooper <kooper@illinois.edu>
Date: Mon, 11 Sep 2023 14:58:24 -0500
Subject: [PATCH 14/24] add readonly mode (#406)

* add readonly mode

tested with create spaces/datasets/files but might be missing special cases.
fixes #405

* more readonly checks

* send email with correct status

* allow to download (based on permission)

* fix user status email

* typo

* fix page for user change

* use latest checkout action

* bump java action to v3

* update cache

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 .github/workflows/ci.yml                      |  40 ++---
 .github/workflows/swagger.yml                 |   2 +-
 CHANGELOG.md                                  |   1 +
 app/api/Admin.scala                           |  41 +++---
 app/api/ApiController.scala                   |   3 +-
 app/api/Metadata.scala                        |  10 +-
 app/api/Permissions.scala                     |   1 +
 app/api/Spaces.scala                          |  50 ++++---
 app/controllers/SecuredController.scala       |   3 +
 app/controllers/Spaces.scala                  |   8 +-
 app/models/User.scala                         |   2 +-
 app/views/admin/users.scala.html              | 139 +++++++++---------
 app/views/collectionList.scala.html           |   2 +
 app/views/collections/miniList.scala.html     |   4 +-
 app/views/datasetList.scala.html              |   3 +
 app/views/datasets/filesAndFolders.scala.html |   2 +
 app/views/datasets/miniList.scala.html        |   4 +-
 app/views/emails/userActivated.scala.html     |  20 ---
 ...dmin.scala.html => userChanged.scala.html} |   7 +-
 app/views/files/listitem.scala.html           |   2 +-
 app/views/files/tile.scala.html               |   2 +-
 app/views/home.scala.html                     |  14 +-
 app/views/main.scala.html                     |  53 +++----
 app/views/selected.scala.html                 |  28 ++--
 app/views/spaces/listSpaces.scala.html        |  19 ++-
 app/views/spaces/miniList.scala.html          |   4 +-
 26 files changed, 241 insertions(+), 223 deletions(-)
 delete mode 100644 app/views/emails/userActivated.scala.html
 rename app/views/emails/{userAdmin.scala.html => userChanged.scala.html} (64%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c0c5d8d13..a77d911b5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,7 +30,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -46,9 +46,10 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
         uses: actions/cache@v1
         with:
@@ -84,7 +85,7 @@ jobs:
         ports:
           - 27017:27017
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -100,16 +101,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
@@ -128,7 +130,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -144,16 +146,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
@@ -204,7 +207,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -220,16 +223,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml
index b0f328a67..b55dfb26f 100644
--- a/.github/workflows/swagger.yml
+++ b/.github/workflows/swagger.yml
@@ -20,7 +20,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: openapi-lint 
         uses: mbowman100/swagger-validator-action@master
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4117fd2e5..16e57e087 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ### Added
+- Users can be marked as ReadOnly [#405](https://github.com/clowder-framework/clowder/issues/405)
 - Added Trash button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
 - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
diff --git a/app/api/Admin.scala b/app/api/Admin.scala
index d8c878df0..27bc8f482 100644
--- a/app/api/Admin.scala
+++ b/app/api/Admin.scala
@@ -125,10 +125,10 @@ class Admin @Inject() (userService: UserService,
       list.foreach(id =>
         userService.findById(UUID(id)) match {
           case Some(u: ClowderUser) => {
-            if (u.status == UserStatus.Inactive) {
+            if (u.status != UserStatus.Active) {
               userService.update(u.copy(status = UserStatus.Active))
-              val subject = s"[${AppConfiguration.getDisplayName}] account activated"
-              val body = views.html.emails.userActivated(u, active = true)(request)
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now active"
+              val body = views.html.emails.userChanged(u, "activated")(request)
               util.Mail.sendEmail(subject, request.user, u, body)
             }
           }
@@ -138,10 +138,10 @@ class Admin @Inject() (userService: UserService,
       list.foreach(id =>
         userService.findById(UUID(id)) match {
           case Some(u: ClowderUser) => {
-            if (!(u.status == UserStatus.Inactive)) {
+            if (u.status != UserStatus.Inactive) {
               userService.update(u.copy(status = UserStatus.Inactive))
-              val subject = s"[${AppConfiguration.getDisplayName}] account deactivated"
-              val body = views.html.emails.userActivated(u, active = false)(request)
+              val subject = s"[${AppConfiguration.getDisplayName}] account is deactivated"
+              val body = views.html.emails.userChanged(u, "deactivated")(request)
               util.Mail.sendEmail(subject, request.user, u, body)
             }
           }
@@ -150,26 +150,27 @@ class Admin @Inject() (userService: UserService,
     (request.body \ "admin").asOpt[List[String]].foreach(list =>
       list.foreach(id =>
         userService.findById(UUID(id)) match {
-          case Some(u: ClowderUser) if (u.status == UserStatus.Active) => {
-
-            userService.update(u.copy(status = UserStatus.Admin))
-            val subject = s"[${AppConfiguration.getDisplayName}] admin access granted"
-            val body = views.html.emails.userAdmin(u, admin = true)(request)
-            util.Mail.sendEmail(subject, request.user, u, body)
-
+          case Some(u: ClowderUser) => {
+            if (u.status != UserStatus.Admin) {
+              userService.update(u.copy(status = UserStatus.Admin))
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now an admin"
+              val body = views.html.emails.userChanged(u, "an admin account")(request)
+              util.Mail.sendEmail(subject, request.user, u, body)
+            }
           }
           case _ => Logger.error(s"Could not update user with id=${id}")
         }))
-    (request.body \ "unadmin").asOpt[List[String]].foreach(list =>
+    (request.body \ "readonly").asOpt[List[String]].foreach(list =>
       list.foreach(id =>
         userService.findById(UUID(id)) match {
-          case Some(u: ClowderUser) if (u.status == UserStatus.Admin) => {
-            userService.update(u.copy(status = UserStatus.Active))
-            val subject = s"[${AppConfiguration.getDisplayName}] admin access revoked"
-            val body = views.html.emails.userAdmin(u, admin = false)(request)
-            util.Mail.sendEmail(subject, request.user, u, body)
+          case Some(u: ClowderUser) => {
+            if (u.status != UserStatus.ReadOnly) {
+              userService.update(u.copy(status = UserStatus.ReadOnly))
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now an read-only"
+              val body = views.html.emails.userChanged(u, "read-only")(request)
+              util.Mail.sendEmail(subject, request.user, u, body)
+            }
           }
-
           case _ => Logger.error(s"Could not update user with id=${id}")
         }))
     Ok(toJson(Map("status" -> "success")))
diff --git a/app/api/ApiController.scala b/app/api/ApiController.scala
index d996b6786..2a5c435b1 100644
--- a/app/api/ApiController.scala
+++ b/app/api/ApiController.scala
@@ -88,12 +88,13 @@ trait ApiController extends Controller {
       userRequest.user match {
         case Some(u) if !AppConfiguration.acceptedTermsOfServices(u.termsOfServices) => Future.successful(Unauthorized("Terms of Service not accepted"))
         case Some(u) if (u.status == UserStatus.Inactive) => Future.successful(Unauthorized("Account is not activated"))
+        case Some(u) if (u.status == UserStatus.ReadOnly && !api.Permission.READONLY.contains(permission) && permission != Permission.DownloadFiles) => Future.successful(Unauthorized("Account is ReadOnly"))
         case Some(u) if u.superAdminMode || Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
         case Some(u) => {
           affectedResource match {
             case Some(resource) if Permission.checkOwner(u, resource) => block(userRequest)
             case _ => Future.successful(Unauthorized("Not authorized"))
-           }
+          }
         }
         case None if Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
         case _ => Future.successful(Unauthorized("Not authorized"))
diff --git a/app/api/Metadata.scala b/app/api/Metadata.scala
index 1c2be48fb..1dec8f676 100644
--- a/app/api/Metadata.scala
+++ b/app/api/Metadata.scala
@@ -257,7 +257,7 @@ class Metadata @Inject() (
 
   // Given a list of terms, create a new standard vocabulary from the list
   // Expects a JSON array of Strings as the request body
-  def createVocabulary() = AuthenticatedAction(parse.json) {
+  def createVocabulary() = PermissionAction(Permission.CreateVocabulary)(parse.json) {
     implicit request =>
       request.user match {
         case None => BadRequest(toJson("Invalid user"))
@@ -278,7 +278,7 @@ class Metadata @Inject() (
 
   // Given an ID, replace the entire terms list of a standard vocabulary
   // Expects a JSON array of Strings as the request body
-  def updateVocabulary(id: UUID) = AuthenticatedAction(parse.json) {
+  def updateVocabulary(id: UUID) = PermissionAction(Permission.EditVocabulary)(parse.json) {
     implicit request =>
       request.user match {
         case None => BadRequest(toJson("Invalid user"))
@@ -304,7 +304,7 @@ class Metadata @Inject() (
   }
 
   // Given an ID, delete the standard vocabulary with that ID
-  def deleteVocabulary(id: UUID) = AuthenticatedAction(parse.empty) {
+  def deleteVocabulary(id: UUID) = PermissionAction(Permission.DeleteVocabulary)(parse.empty) {
     implicit request =>
       request.user match {
         case None => BadRequest(toJson("Invalid user"))
@@ -341,7 +341,7 @@ class Metadata @Inject() (
     }
   }
 
-  def editDefinition(id: UUID, spaceId: Option[String]) = AuthenticatedAction(parse.json) {
+  def editDefinition(id: UUID, spaceId: Option[String]) = PermissionAction(Permission.EditVocabulary)(parse.json) {
     implicit request =>
       request.user match {
         case Some(user) => {
@@ -387,7 +387,7 @@ class Metadata @Inject() (
       }
   }
 
-  def deleteDefinition(id: UUID) = AuthenticatedAction { implicit request =>
+  def deleteDefinition(id: UUID) = PermissionAction(Permission.CreateVocabulary) { implicit request =>
     implicit val user = request.user
     user match {
       case Some(user) => {
diff --git a/app/api/Permissions.scala b/app/api/Permissions.scala
index ac902a31d..e3c6ca5c3 100644
--- a/app/api/Permissions.scala
+++ b/app/api/Permissions.scala
@@ -426,6 +426,7 @@ object Permission extends Enumeration {
   def checkPermission(user: User, permission: Permission, resourceRef: ResourceRef): Boolean = {
     // check if user is owner, in that case they can do what they want.
     if (user.superAdminMode) return true
+    if (user.status == UserStatus.ReadOnly && !READONLY.contains(permission) && permission != Permission.DownloadFiles) return false
     if (checkOwner(users.findByIdentity(user), resourceRef)) return true
 
     resourceRef match {
diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala
index 8dd34edda..e48de645b 100644
--- a/app/api/Spaces.scala
+++ b/app/api/Spaces.scala
@@ -32,35 +32,39 @@ class Spaces @Inject()(spaces: SpaceService,
   val spaceTitle: String = Messages("space.title")
 
   //TODO- Minimal Space created with Name and description. URLs are not yet put in
-  def createSpace() = AuthenticatedAction(parse.json) { implicit request =>
+  def createSpace() = PermissionAction(Permission.CreateSpace)(parse.json) { implicit request =>
     Logger.debug("Creating new space")
-    val nameOpt = (request.body \ "name").asOpt[String]
-    val descOpt = (request.body \ "description").asOpt[String]
-    (nameOpt, descOpt) match {
-      case (Some(name), Some(description)) => {
-        // TODO: add creator
-        val userId = request.user.get.id
-        val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
-          homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
-          datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
-        spaces.insert(c) match {
-          case Some(id) => {
-            appConfig.incrementCount('spaces, 1)
-            events.addObjectEvent(request.user, c.id, c.name, "create_space")
-            userService.findRoleByName("Admin") match {
-              case Some(realRole) => {
-                spaces.addUser(userId, realRole, UUID(id))
-              }
-              case None => Logger.info("No admin role found")
+    if(request.user.get.status == UserStatus.ReadOnly) {
+      BadRequest(toJson("User is Read-Only"))
+    } else {
+      val nameOpt = (request.body \ "name").asOpt[String]
+      val descOpt = (request.body \ "description").asOpt[String]
+      (nameOpt, descOpt) match {
+        case (Some(name), Some(description)) => {
+          // TODO: add creator
+          val userId = request.user.get.id
+          val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
+            homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
+            datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
+          spaces.insert(c) match {
+            case Some(id) => {
+              appConfig.incrementCount('spaces, 1)
+              events.addObjectEvent(request.user, c.id, c.name, "create_space")
+              userService.findRoleByName("Admin") match {
+                case Some(realRole) => {
+                  spaces.addUser(userId, realRole, UUID(id))
+                }
+                case None => Logger.info("No admin role found")
 
+              }
+              Ok(toJson(Map("id" -> id)))
             }
-            Ok(toJson(Map("id" -> id)))
+            case None => Ok(toJson(Map("status" -> "error")))
           }
-          case None => Ok(toJson(Map("status" -> "error")))
-        }
 
+        }
+        case (_, _) => BadRequest(toJson("Missing required parameters"))
       }
-      case (_, _) => BadRequest(toJson("Missing required parameters"))
     }
   }
 
diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala
index 0cbedeec3..ce4dae8c6 100644
--- a/app/controllers/SecuredController.scala
+++ b/app/controllers/SecuredController.scala
@@ -104,6 +104,9 @@ trait SecuredController extends Controller {
       userRequest.user match {
         case Some(u) if !AppConfiguration.acceptedTermsOfServices(u.termsOfServices) => Future.successful(Results.Redirect(routes.Application.tos(Some(request.uri))))
         case Some(u) if (u.status==UserStatus.Inactive) => Future.successful(Results.Redirect(routes.Error.notActivated()))
+        case Some(u) if (u.status==UserStatus.ReadOnly && !api.Permission.READONLY.contains(permission) && permission != Permission.DownloadFiles) => {
+          Future.successful(Results.Redirect(routes.Error.notAuthorized("Account is ReadOnly", "", "")))
+        }
         case Some(u) if u.superAdminMode || Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
         case Some(u) => notAuthorizedMessage(userRequest.user, resourceRef)
         case None if Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala
index 49e724aea..1c197060a 100644
--- a/app/controllers/Spaces.scala
+++ b/app/controllers/Spaces.scala
@@ -222,7 +222,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
     }
   }
 
-  def newSpace() = AuthenticatedAction { implicit request =>
+  def newSpace() = PermissionAction(Permission.CreateSpace) { implicit request =>
     implicit val user = request.user
     Ok(views.html.spaces.newSpace(spaceForm))
   }
@@ -396,10 +396,10 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
    * Submit action for new or edit space
    */
   // TODO this should check to see if user has editspace for specific space
-  def submit() = AuthenticatedAction { implicit request =>
+  def submit() = PermissionAction(Permission.CreateSpace) { implicit request =>
     implicit val user = request.user
     user match {
-      case Some(identity) => {
+      case Some(identity) if identity.status != UserStatus.ReadOnly => {
         val userId = request.user.get.id
         //need to get the submitValue before binding form data, in case of errors we want to trigger different forms
         request.body.asMultipartFormData.get.dataParts.get("submitValue").headOption match {
@@ -483,7 +483,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
           case None => { BadRequest("Did not get any submit button value.") }
         }
       } //some identity
-      case None => Redirect(routes.Spaces.list()).flashing("error" -> "You are not authorized to create/edit $spaceTitle.")
+      case _ => Redirect(routes.Spaces.list()).flashing("error" -> "You are not authorized to create/edit $spaceTitle.")
     }
   }
   def followingSpaces(index: Int, limit: Int, mode: String) = PrivateServerAction { implicit request =>
diff --git a/app/models/User.scala b/app/models/User.scala
index 5daae5c46..d07fe8020 100644
--- a/app/models/User.scala
+++ b/app/models/User.scala
@@ -13,7 +13,7 @@ import play.api.libs.json._
 
 object UserStatus extends Enumeration {
 	  type UserStatus = Value
-	  val Inactive, Active, Admin = Value
+	  val Inactive, Active, Admin, ReadOnly = Value
 	}
 
 /**
diff --git a/app/views/admin/users.scala.html b/app/views/admin/users.scala.html
index a5b0ed626..046a47c1a 100644
--- a/app/views/admin/users.scala.html
+++ b/app/views/admin/users.scala.html
@@ -18,32 +18,32 @@
     	    	<td>never</td>
     	    }	
         }
-        @if(user.fold("")(_.id.stringify) == u.id.stringify) {
-            @if(!(u.status==UserStatus.Inactive)) {
-                <td><input class="active" type="checkbox" id="active-@u.id" data-original=true disabled checked title="Can not change yourself"></td>
+        @if(user.fold("")(_.id.stringify) == u.id.stringify || configAdmins.contains(u.email.getOrElse(""))) {
+        <td><select id="status-@u.id" data-original="@u.status" disabled>
+        } else {
+        <td><select id="status-@u.id" data-original="@u.status">
+        }
+            @if(u.status==UserStatus.Inactive) {
+                <option value="Inactive" selected>Inactive</option>
             } else {
-                <td><input class="active" type="checkbox" id="active-@u.id" data-original=false disabled ></td>
+                <option value="Inactive">Inactive</option>
             }
-        } else {
-            @if(!(u.status==UserStatus.Inactive)) {
-                <td><input class="active" type="checkbox" id="active-@u.id" data-original=true checked></td>
+            @if(u.status==UserStatus.Active) {
+                <option value="Active" selected>Active</option>
             } else {
-                <td><input class="active" type="checkbox" id="active-@u.id" data-original=false></td>
+                <option value="Active">Active</option>
             }
-        }
-        @if(user.fold("")(_.id.stringify) == u.id.stringify) {
-            <td><input type="checkbox" id="admin-@u.id" data-original=true checked disabled title="Can not change yourself"></td>
-        } else {
-            @if(configAdmins.contains(u.email.getOrElse(""))) {
-                <td><input type="checkbox" id="admin-@u.id" data-original=true checked disabled title="Specified in custom.conf"></td>
+            @if(u.status==UserStatus.ReadOnly) {
+                <option value="ReadOnly" selected>ReadOnly</option>
             } else {
-                @if(u.status==UserStatus.Admin) {
-                        <td><input type="checkbox" id="admin-@u.id" data-original=true checked></td>
-                } else {
-                        <td><input type="checkbox" id="admin-@u.id" data-original=false></td>
-                }
+                <option value="ReadOnly">ReadOnly</option>
             }
-        }
+            @if(u.status==UserStatus.Admin) {
+                <option value="Admin" selected>Admin</option>
+            } else {
+                <option value="Admin">Admin</option>
+            }
+        </select></td>
     </tr>
 }
 
@@ -66,12 +66,11 @@ <h4 class="panel-title">
                     <table class="table table-condensed">
                         <thead>
                             <tr>
-                                <th class="col-md-3">Fullname</th>
+                                <th class="col-md-4">Fullname</th>
                                 <th class="col-md-3">Email</th>
                                 <th class="col-md-2">Provider</th>
                                 <th class="col-md-2">Last Login</th>
-                                <th class="col-md-1">Active</th>
-                                <th class="col-md-1">Admin</th>
+                                <th class="col-md-1">Status</th>
                             </tr>
                         </thead>
                         <tbody id="inactive-users">
@@ -95,12 +94,11 @@ <h4 class="panel-title">
                     <table class="table table-condensed">
                         <thead>
                             <tr>
-                                <th class="col-md-3">Fullname</th>
+                                <th class="col-md-4">Fullname</th>
                                 <th class="col-md-3">Email</th>
                                 <th class="col-md-2">Provider</th>
                                 <th class="col-md-2">Last Login</th>
-                                <th class="col-md-1">Active</th>
-                                <th class="col-md-1">Admin</th>
+                                <th class="col-md-1">Status</th>
                             </tr>
                         </thead>
                         <tbody id="admin-users">
@@ -124,12 +122,11 @@ <h4 class="panel-title">
                     <table class="table table-condensed">
                         <thead>
                             <tr>
-                                <th class="col-md-3">Fullname</th>
+                                <th class="col-md-4">Fullname</th>
                                 <th class="col-md-3">Email</th>
                                 <th class="col-md-2">Provider</th>
                                 <th class="col-md-2">Last Login</th>
-                                <th class="col-md-1">Active</th>
-                                <th class="col-md-1">Admin</th>
+                                <th class="col-md-1">Status</th>
                             </tr>
                         </thead>
                         <tbody id="active-users">
@@ -140,6 +137,34 @@ <h4 class="panel-title">
             </div>
         </div>
 
+        <div class="panel panel-default">
+            <div class="panel-heading" role="tab" id="headingOne">
+                <h4 class="panel-title">
+                    <a role="button" data-toggle="collapse" href="#collapseReadOnly" aria-expanded="false" aria-controls="collapseReadOnly">
+                        ReadOnly Users
+                    </a>
+                </h4>
+            </div>
+            <div id="collapseReadOnly" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne">
+                <div class="panel-body">
+                    <table class="table table-condensed">
+                        <thead>
+                            <tr>
+                                <th class="col-md-4">Fullname</th>
+                                <th class="col-md-3">Email</th>
+                                <th class="col-md-2">Provider</th>
+                                <th class="col-md-2">Last Login</th>
+                                <th class="col-md-1">Status</th>
+                            </tr>
+                        </thead>
+                        <tbody id="readonly-users">
+                        @users.filter(u => u.status==UserStatus.ReadOnly).map(printRow)
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+
     </div>
 
     <button class="btn btn-primary" onclick="return submitUsers();"><span class="glyphicon glyphicon-saved"></span> Submit</button>
@@ -148,45 +173,17 @@ <h4 class="panel-title">
     <br/>
 
     <script>
-            $(".active").click(function() {
-                var input = $(this)[0];
-                var parts = input.id.split("-", 2);
-                $("#admin-" + parts[1]).prop('disabled', !input.checked);
-            });
-            var users = { active: [], inactive: [], admin: [], unadmin: [] };
+            var users = { active: [], inactive: [], admin: [], readonly: [] };
             function submitUsers() {
-
                 var sendit = false;
 
-                $(".active:not(:checked)").each(function() {
-                    var input = $(this)[0];
-                    var parts = input.id.split("-", 2);
-                    var admin = $("#admin-" + parts[1]);
-                    admin.prop('checked', false);
-                    admin.prop('disabled', true);
-                });
-
-                $("input[type=checkbox]").each(function() {
-                    var input = $(this)[0];
-                    if (input.getAttribute("data-original") != input.checked.toString()) {
+                $("select").each(function() {
+                    var select = $(this)[0];
+                    if (select.getAttribute("data-original") != select.value) {
                         sendit = true;
-                        var parts = input.id.split("-", 2);
-                        if (parts[0] == "admin") {
-                            if (input.checked) {
-                                users.admin.push(parts[1]);
-                            } else {
-                                users.unadmin.push(parts[1]);
-                            }
-                        } else if (parts[0] == "active") {
-                            if (input.checked) {
-                                users.active.push(parts[1]);
-                            } else {
-                                users.inactive.push(parts[1]);
-                            }
-                        } else {
-                            logger.error("Not sure what to do with " + parts[0]);
-                        }
-                        input.setAttribute("data-original", input.checked.toString());
+                        var parts = select.id.split("-", 2);
+                        users[select.value.toLowerCase()].push(parts[1]);
+                        select.setAttribute("data-original", select.value);
                     }
                 });
 
@@ -197,19 +194,19 @@ <h4 class="panel-title">
                         type: "POST",
                         contentType: "application/json"
                     }).done(function () {
-                        users.active.forEach(function(u) {
-                            $('#active-users').append($('#'+u))
-                        });
-                        users.unadmin.forEach(function(u) {
-                            $('#active-users').append($('#'+u))
-                        });
                         users.inactive.forEach(function(u) {
                             $('#inactive-users').append($('#'+u))
                         });
                         users.admin.forEach(function(u) {
                             $('#admin-users').append($('#'+u))
                         });
-                        users = { active: [], inactive: [], admin: [], unadmin: [] };
+                        users.active.forEach(function(u) {
+                            $('#active-users').append($('#'+u))
+                        });
+                        users.readonly.forEach(function(u) {
+                            $('#readonly-users').append($('#'+u))
+                        });
+                        users = { active: [], inactive: [], admin: [], readonly: [] };
                         notify("Users successfully updated.", "success", false, 5000);
                     }).fail(function (jqXHR, textStatus, errorThrown){
                         console.error("The following error occurred: " + textStatus, errorThrown);
diff --git a/app/views/collectionList.scala.html b/app/views/collectionList.scala.html
index 809ad8046..b190dbb74 100644
--- a/app/views/collectionList.scala.html
+++ b/app/views/collectionList.scala.html
@@ -43,6 +43,7 @@ <h1>@Html(title.getOrElse("Collections"))</h1>
 
 <div class="row">
     <div class="btn-toolbar pagination-toolbar pull-right">
+    @if(user.isDefined && user.get.status != UserStatus.ReadOnly) {
 
         @user match {
             case Some(u) => {
@@ -79,6 +80,7 @@ <h1>@Html(title.getOrElse("Collections"))</h1>
             }
             case _ => {}
         }
+    }
 
         <div class="btn-group btn-group-sm pull-right" id="number-displayed-dropdown">
             <select id="numPageItems" class="form-control" onchange="getValue()">
diff --git a/app/views/collections/miniList.scala.html b/app/views/collections/miniList.scala.html
index 2561298c8..b3c2525f5 100644
--- a/app/views/collections/miniList.scala.html
+++ b/app/views/collections/miniList.scala.html
@@ -24,7 +24,9 @@
     <div class="text-center">
         @if(ownProfile) {
             <div>@Messages("home.empty.message", Messages("collections.title").toLowerCase) </div>
-            <div><a class="btn-link" href="@routes.Collections.newCollection(None)" title="Create a new @Messages("collection.title")">@Messages("create.title", Messages("collection.title"))</a></div>
+            @if(user.get.status != UserStatus.ReadOnly) {
+                <div><a class="btn-link" href="@routes.Collections.newCollection(None)" title="Create a new @Messages("collection.title")">@Messages("create.title", Messages("collection.title"))</a></div>
+            }
         } else {
             <div>@Messages("profile.empty.message", Messages("collections.title").toLowerCase) </div>
         }
diff --git a/app/views/datasetList.scala.html b/app/views/datasetList.scala.html
index f5956c83d..4dab7a837 100644
--- a/app/views/datasetList.scala.html
+++ b/app/views/datasetList.scala.html
@@ -54,6 +54,8 @@ <h1>@Html(title.getOrElse("Datasets"))</h1>
 <div class="row">
     <div class="btn-toolbar pagination-toolbar pull-right">
 
+    @if(user.isDefined && user.get.status != UserStatus.ReadOnly) {
+
         @status match {
             case Some(st) => {}
             case None => {
@@ -95,6 +97,7 @@ <h1>@Html(title.getOrElse("Datasets"))</h1>
                 }
             }
         }
+    }
 
         <div class="btn-group btn-group-sm" id="number-displayed-dropdown">
             <select id="numPageItems" class="form-control" onchange="getValue()">
diff --git a/app/views/datasets/filesAndFolders.scala.html b/app/views/datasets/filesAndFolders.scala.html
index ff1350de1..db863798b 100644
--- a/app/views/datasets/filesAndFolders.scala.html
+++ b/app/views/datasets/filesAndFolders.scala.html
@@ -90,9 +90,11 @@
                     *@
                     '<li><a onclick="markVisibleFiles();"><span class="glyphicon glyphicon-check"/>&nbsp;Mark Visible</a></li>' +
                     '<li><a onclick="downloadMarked();"><span class="glyphicon glyphicon-download-alt"/>&nbsp;Download</a></li>' +
+                    @if(user.get.status != UserStatus.ReadOnly) {
                     '<li><a onclick="submitMarked();"><span class="glyphicon glyphicon-send"/>&nbsp;Submit</a></li>' +
                     '<li><a onclick="confirmTagMarked();"><span class="glyphicon glyphicon-tag"/>&nbsp;Tag</a></li>' +
                     '<li><a onclick="confirmDeleteMarked();"><span class="glyphicon glyphicon-trash"/>&nbsp;Delete</a></li>' +
+                    }
                     '<li><a onclick="clearMarked();"><span class="glyphicon glyphicon-erase"/>&nbsp;Unmark All</a></li></ul>' +
                     '</div>')
         } else {
diff --git a/app/views/datasets/miniList.scala.html b/app/views/datasets/miniList.scala.html
index 6d90e3b75..a3d97dfe7 100644
--- a/app/views/datasets/miniList.scala.html
+++ b/app/views/datasets/miniList.scala.html
@@ -27,7 +27,9 @@
     <div class="text-center">
         @if(ownProfile) {
             <div>@Messages("home.empty.message", Messages("datasets.title").toLowerCase)</div>
-            <div><a class="btn-link" href="@routes.Datasets.newDataset(None, None)" title="Create a new @Messages("dataset.title")">@Messages("create.title", Messages("dataset.title"))</a></div>
+            @if(user.get.status != UserStatus.ReadOnly) {
+                <div><a class="btn-link" href="@routes.Datasets.newDataset(None, None)" title="Create a new @Messages("dataset.title")">@Messages("create.title", Messages("dataset.title"))</a></div>
+            }
         } else {
             <div>@Messages("profile.empty.message", Messages("datasets.title").toLowerCase) </div>
         }
diff --git a/app/views/emails/userActivated.scala.html b/app/views/emails/userActivated.scala.html
deleted file mode 100644
index 1995e92d0..000000000
--- a/app/views/emails/userActivated.scala.html
+++ /dev/null
@@ -1,20 +0,0 @@
-@(user: User, active: Boolean, ssl: Boolean = true)(implicit request: RequestHeader)
-
-@import services.AppConfiguration
-
-<html>
-    <body>
-        @user.fullName,<br/>
-        <br>
-        <p>
-            Your account on <a href="@routes.Application.index().absoluteURL(ssl)" style="text-decoration: none">@AppConfiguration.getDisplayName</a><br/>
-            has been
-            @if(active) {
-                enabled and you can now login.
-            } else {
-                disabled.
-            }
-        </p>
-        @footer()
-    </body>
-</html>
diff --git a/app/views/emails/userAdmin.scala.html b/app/views/emails/userChanged.scala.html
similarity index 64%
rename from app/views/emails/userAdmin.scala.html
rename to app/views/emails/userChanged.scala.html
index 9ebcbc651..09c9dce91 100644
--- a/app/views/emails/userAdmin.scala.html
+++ b/app/views/emails/userChanged.scala.html
@@ -1,13 +1,14 @@
-@(user: User, admin: Boolean, ssl: Boolean = true)(implicit request: RequestHeader)
+@(user: User, status: String, ssl: Boolean = true)(implicit request: RequestHeader)
 
 @import services.AppConfiguration
 
 <html>
     <body>
-        <p>@user.fullName,</p>
+        @user.fullName,<br/>
+        <br>
         <p>
             Your account on <a href="@routes.Application.index().absoluteURL(ssl)" style="text-decoration: none">@AppConfiguration.getDisplayName</a>
-            now @if(!admin) { no longer } has admin privileges.
+            has been made @status.
         </p>
         @footer()
     </body>
diff --git a/app/views/files/listitem.scala.html b/app/views/files/listitem.scala.html
index 72a05c328..aad3b5163 100644
--- a/app/views/files/listitem.scala.html
+++ b/app/views/files/listitem.scala.html
@@ -36,7 +36,7 @@ <h3><a href="@(routes.Files.file(file.id, dataset, space, folder))" class="break
                                 <span class="glyphicon glyphicon-circle-arrow-down" title="@file.stats.downloads downloads"></span> @file.stats.downloads
                                 @if(user.isDefined) {
                                         <!-- If user can delete, the button is enabled, otherwise the button is present but disabled to provide consistent UE. -->
-                                    @if(user.get.id.equals(file.author.id) || Permission.checkPermission(Permission.DeleteFile, ResourceRef(ResourceRef.file, file.id))){
+                                    @if(user.get.status != UserStatus.ReadOnly && (user.get.id.equals(file.author.id) || Permission.checkPermission(Permission.DeleteFile, ResourceRef(ResourceRef.file, file.id)))){
                                         <button onclick="
                                                 @folder match {
                                                     case None => {confirmDeleteResource('file','file','@(file.id)','@(file.filename.replace("'","&#39;"))',false, '@redirect')}
diff --git a/app/views/files/tile.scala.html b/app/views/files/tile.scala.html
index 5d116ef16..8910f3ac4 100644
--- a/app/views/files/tile.scala.html
+++ b/app/views/files/tile.scala.html
@@ -29,7 +29,7 @@ <h4><a href="@(routes.Files.file(file.id))">@file.filename</a></h4>
                 <span class="glyphicon glyphicon-eye-open" title="@file.stats.views views"></span> @file.stats.views
                 <span class="glyphicon glyphicon-circle-arrow-down" title="@file.stats.downloads downloads"></span> @file.stats.downloads
                 @if(user.isDefined) {
-                    @if(user.get.id.equals(file.author.id) || Permission.checkPermission(Permission.DeleteFile, ResourceRef(ResourceRef.file, file.id))){
+                    @if(user.get.status != UserStatus.ReadOnly && (user.get.id.equals(file.author.id) || Permission.checkPermission(Permission.DeleteFile, ResourceRef(ResourceRef.file, file.id)))){
                         <button onclick="confirmDeleteResource('file','file','@(file.id)','@(file.filename.replace("'","&#39;"))', false, '@redirect')" class="btn btn-link" title="Delete the file">
                         <span class="glyphicon glyphicon-trash"></span></button>
                     } else {
diff --git a/app/views/home.scala.html b/app/views/home.scala.html
index f6b58bb28..e45e4b2b1 100644
--- a/app/views/home.scala.html
+++ b/app/views/home.scala.html
@@ -23,12 +23,14 @@ <h2>@profile.fullName</h2>
         <div class="button-margins">
             <a href="@routes.Profile.viewProfileUUID(profile.id)" class="btn btn-link" title="Show profile">
                 <span class="glyphicon glyphicon-user"></span> Profile</a>
-            <a href="@routes.Spaces.newSpace()" class="btn btn-link" title="Create a new @Messages("space.title")">
-                <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("space.title"))</a>
-            <a href="@routes.Datasets.newDataset(None, None)" class="btn btn-link" title="Create a new dataset">
-                <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("dataset.title"))</a>
-            <a href="@routes.Collections.newCollection(None)" class="btn btn-link" title="Create a new collection">
-                <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("collection.title"))</a>
+            @if(user.get.status != UserStatus.ReadOnly) {
+                <a href="@routes.Spaces.newSpace()" class="btn btn-link" title="Create a new @Messages("space.title")">
+                    <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("space.title"))</a>
+                <a href="@routes.Datasets.newDataset(None, None)" class="btn btn-link" title="Create a new dataset">
+                    <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("dataset.title"))</a>
+                <a href="@routes.Collections.newCollection(None)" class="btn btn-link" title="Create a new collection">
+                    <span class="glyphicon glyphicon-ok"></span> @Messages("create.title", Messages("collection.title"))</a>
+            }
         </div>
     <div class="row">
         <!-- Nav tabs -->
diff --git a/app/views/main.scala.html b/app/views/main.scala.html
index 5bd618415..268e76992 100644
--- a/app/views/main.scala.html
+++ b/app/views/main.scala.html
@@ -129,29 +129,30 @@
                         </ul>
                     </li>
                 @if(user.isDefined) {
-                    <li class="dropdown">
-                        <a href="#" class="dropdown-toggle" data-toggle="dropdown">
-                            Create <b class="caret"></b>
-                        </a>
-                        <ul class="dropdown-menu">
-                            <li><a href="@routes.Spaces.newSpace()"><span class="glyphicon glyphicon-hdd"> </span>
-                                @Messages("spaces.title")
-                            </a></li>
-                            <li><a href="@routes.Datasets.newDataset(None, None)"><span class="glyphicon glyphicon-briefcase"> </span>
-                                @Messages("datasets.title")</a></li>
-                            <li><a href="@routes.Collections.newCollection(None)"><span class="glyphicon glyphicon-th-large"> </span>
-                                @Messages("collections.title")</a></li>
-                            @play.api.Play.current.plugin[services.PostgresPlugin] match {
+                    @if(user.get.status != UserStatus.ReadOnly) {
+                        <li class="dropdown">
+                            <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+                                Create <b class="caret"></b>
+                            </a>
+                            <ul class="dropdown-menu">
+                                <li><a href="@routes.Spaces.newSpace()"><span class="glyphicon glyphicon-hdd"> </span>
+                                    @Messages("spaces.title")
+                                </a></li>
+                                <li><a href="@routes.Datasets.newDataset(None, None)"><span class="glyphicon glyphicon-briefcase"> </span>
+                                    @Messages("datasets.title")</a></li>
+                                <li><a href="@routes.Collections.newCollection(None)"><span class="glyphicon glyphicon-th-large"> </span>
+                                    @Messages("collections.title")</a></li>
+                                @play.api.Play.current.plugin[services.PostgresPlugin] match {
                                 case Some(db) if db.isEnabled => {
-                                    <li><a href="@routes.Geostreams.newSensor"><span class="glyphicon glyphicon-globe"> </span>
-                                        @(AppConfiguration.getSensorsTitle)</a></li>
-                                }
-                                case _ => {}
+                                <li><a href="@routes.Geostreams.newSensor"><span class="glyphicon glyphicon-globe"> </span>
+                                    @(AppConfiguration.getSensorsTitle)</a>
+                                </li>
                             }
-
-
-                        </ul>
-                    </li>
+                                case _ => {}
+                                }
+                            </ul>
+                        </li>
+                    }
 
                     <li class="dropdown">
                         <a href="#" class="dropdown-toggle" data-toggle="dropdown">
@@ -163,10 +164,12 @@
                             <li><a href="#" onclick="downloadAllSelections();"><span class="glyphicon glyphicon-download-alt"> </span>
                                 Download All
                             </a></li>
-                            <li><a href="#" onclick="showTagPopup();"><span class="glyphicon glyphicon-tag"> </span>
-                                Tag All</a></li>
-                            <li><a href="#" onclick="confirmDeleteSelected();"><span class="glyphicon glyphicon-trash"> </span>
-                                Delete All</a></li>
+                            @if(user.get.status != UserStatus.ReadOnly) {
+                                <li><a href="#" onclick="showTagPopup();"><span class="glyphicon glyphicon-tag"> </span>
+                                    Tag All</a></li>
+                                <li><a href="#" onclick="confirmDeleteSelected();"><span class="glyphicon glyphicon-trash"> </span>
+                                    Delete All</a></li>
+                            }
                             <li><a href="#" onclick="clearSelections();"><span class="glyphicon glyphicon-erase"> </span>
                                 Clear All</a></li>
                         </ul>
diff --git a/app/views/selected.scala.html b/app/views/selected.scala.html
index dbd3e4a55..e534e2f54 100644
--- a/app/views/selected.scala.html
+++ b/app/views/selected.scala.html
@@ -24,22 +24,26 @@ <h1>Selections</h1>
 					<a id='download-url' href="#" onclick="downloadAllSelections();"
 					class="btn btn-link disabled" title="Download all selected files as zip" role="button">
 						<span class="glyphicon glyphicon-download-alt"></span> Download All</a>
-					<a id='delete-url' href="#" onclick="confirmDeleteSelected();"
-					class="btn btn-link disabled" title="Delete all selected datasets" role="button">
-						<span class="glyphicon glyphicon-trash"></span> Delete All</a>
-					<a id='tag-url' href="#" onclick="showTagPopup();"
-					class="btn btn-link" title="Apply a tag to all selected datasets" role="button">
-						<span class="glyphicon glyphicon-tag"></span> Tag All</a>
+					@if(user.get.status != UserStatus.ReadOnly) {
+						<a id='delete-url' href="#" onclick="confirmDeleteSelected();"
+						class="btn btn-link disabled" title="Delete all selected datasets" role="button">
+							<span class="glyphicon glyphicon-trash"></span> Delete All</a>
+						<a id='tag-url' href="#" onclick="showTagPopup();"
+						class="btn btn-link" title="Apply a tag to all selected datasets" role="button">
+							<span class="glyphicon glyphicon-tag"></span> Tag All</a>
+					}
 				} else {
 					<a id='download-url' href="#" onclick="downloadAllSelections();"
 					class="btn btn-link" title="Download all selected files as zip" role="button">
 						<span class="glyphicon glyphicon-download-alt"></span> Download All</a>
-					<a id='delete-url' href="#" onclick="confirmDeleteSelected();"
-					class="btn btn-link" title="Delete all selected datasets" role="button">
-						<span class="glyphicon glyphicon-trash"></span> Delete All</a>
-					<a id='tag-url' href="#" onclick="showTagPopup();"
-					class="btn btn-link" title="Apply a tag to all selected datasets" role="button">
-						<span class="glyphicon glyphicon-tag"></span> Tag All</a>
+					@if(user.get.status != UserStatus.ReadOnly) {
+						<a id='delete-url' href="#" onclick="confirmDeleteSelected();"
+						class="btn btn-link" title="Delete all selected datasets" role="button">
+							<span class="glyphicon glyphicon-trash"></span> Delete All</a>
+						<a id='tag-url' href="#" onclick="showTagPopup();"
+						class="btn btn-link" title="Apply a tag to all selected datasets" role="button">
+							<span class="glyphicon glyphicon-tag"></span> Tag All</a>
+					}
 				}
 				<a id='clear-url' href="#" onclick="clearSelections();"
 				class="btn btn-link" title="Clear all selected datasets" role="button">
diff --git a/app/views/spaces/listSpaces.scala.html b/app/views/spaces/listSpaces.scala.html
index 1518df0ae..9f62efefb 100644
--- a/app/views/spaces/listSpaces.scala.html
+++ b/app/views/spaces/listSpaces.scala.html
@@ -33,16 +33,19 @@ <h1>@title</h1>
 
         <div class="row">
             <div class="btn-toolbar pagination-toolbar pull-right">
-                @(user, owner) match {
-                    case (Some(u), Some(o)) => {
-                        @if(o.equalsIgnoreCase(u.id.stringify)) {
-                            <a class="btn btn-primary btn-sm" id="create-space" href="@routes.Spaces.newSpace" title="Create a new @Messages("space.title")"><span class="glyphicon glyphicon-ok"></span> @Messages("create.title", "")</a>
+                @if(user.isDefined && user.get.status != UserStatus.ReadOnly) {
+                    @(user, owner) match {
+                        case (Some(u), Some(o)) => {
+                            @if(o.equalsIgnoreCase(u.id.stringify)) {
+                                <a class="btn btn-primary btn-sm" id="create-space" href="@routes.Spaces.newSpace" title="Create a new @Messages("space.title")"><span class="glyphicon glyphicon-ok"></span> @Messages("create.title", "")</a>
+                            }
                         }
+                        case (Some(u), _) => {
+                            <a class="btn btn-primary btn-sm" id="create-space" href="@routes.Spaces.newSpace" title="Create a new @Messages("space.title")"><span class="glyphicon glyphicon-ok"></span>
+                                @Messages("create.title", "")</a>
+                        }
+                        case (_, _) => {}
                     }
-                    case (Some(u), _) => {
-                        <a class="btn btn-primary btn-sm" id="create-space" href="@routes.Spaces.newSpace" title="Create a new @Messages("space.title")"><span class="glyphicon glyphicon-ok"></span> @Messages("create.title", "")</a>
-                    }
-                    case (_,_) => {}
                 }
                 <div class="btn-group btn-group-sm" id="spacesOwnership-displayed-dropdown">
                     <select id="spacesOwnership" class="form-control" onchange="getSpaceType('@user.get.id')">
diff --git a/app/views/spaces/miniList.scala.html b/app/views/spaces/miniList.scala.html
index 16915aeab..87bd2876e 100644
--- a/app/views/spaces/miniList.scala.html
+++ b/app/views/spaces/miniList.scala.html
@@ -23,7 +23,9 @@
     <div class="text-center">
         @if(ownProfile) {
             <div>@Messages("home.empty.message", Messages("spaces.title").toLowerCase)</div>
-            <div><a class="btn-link" href="@routes.Spaces.newSpace()" title="Create a new @Messages("space.title")">@Messages("create.title", Messages("space.title"))</a></div>
+            @if(user.get.status != UserStatus.ReadOnly) {
+                <div><a class="btn-link" href="@routes.Spaces.newSpace()" title="Create a new @Messages("space.title")">@Messages("create.title", Messages("space.title"))</a></div>
+            }
         } else {
             <div>@Messages("profile.empty.message", Messages("spaces.title").toLowerCase)</div>
         }

From 3e058a286c678c3655c30cf17e9036f9fac12a20 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 11 Sep 2023 15:56:54 -0500
Subject: [PATCH 15/24] Change how nested objects are indexed (#404)

* modifications to indexing

* remove old comment

* Update CHANGELOG.md

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                           |  1 +
 app/services/ElasticsearchPlugin.scala | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 16e57e087..75d070e10 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 ## Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
+- Fixed bug where complex JSON metadata objects using arrays were not being indexed properly for search.
 
 
 ## 1.21.0 - 2022-08-23
diff --git a/app/services/ElasticsearchPlugin.scala b/app/services/ElasticsearchPlugin.scala
index 25b7c8d9d..e6e5e6ce9 100644
--- a/app/services/ElasticsearchPlugin.scala
+++ b/app/services/ElasticsearchPlugin.scala
@@ -527,7 +527,7 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
                 case jv: JsArray => {
                   builder.startArray(clean_k)
                   jv.value.foreach(subv => {
-                    builder.value(subv.toString.replace("\"",""))
+                    builder.value(convertJsObjectToBuilder(builder, subv.asInstanceOf[JsObject]))
                   })
                   builder.endArray()
                 }
@@ -598,11 +598,9 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
           // Elasticsearch 2 does not allow periods in field names
           builder.startArray(k.toString.replace(".", "_"))
           v.value.foreach(jv => {
-            // Try to interpret numeric value from each String if possible
-            parseDouble(jv.toString) match {
-              case Some(d) => builder.value(d)
-              case None => builder.value(jv)
-            }
+            builder.startObject()
+            convertJsObjectToBuilder(builder, jv.asInstanceOf[JsObject])
+            builder.endObject()
           })
           builder.endArray()
         }

From 00ed4558b3d7d4dec738e57c09fe573e9c6a50bc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Sep 2023 15:57:59 -0500
Subject: [PATCH 16/24] Bump requests from 2.21.0 to 2.31.0 in
 /scripts/rmq-error-shovel (#422)

Bumps [requests](https://github.com/psf/requests) from 2.21.0 to 2.31.0.
- [Release notes](https://github.com/psf/requests/releases)
- [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md)
- [Commits](https://github.com/psf/requests/compare/v2.21.0...v2.31.0)

---
updated-dependencies:
- dependency-name: requests
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 scripts/rmq-error-shovel/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/rmq-error-shovel/requirements.txt b/scripts/rmq-error-shovel/requirements.txt
index a7e215d09..ea4e61e6b 100644
--- a/scripts/rmq-error-shovel/requirements.txt
+++ b/scripts/rmq-error-shovel/requirements.txt
@@ -1,2 +1,2 @@
 pika==1.0.0
-requests==2.21.0
\ No newline at end of file
+requests==2.31.0
\ No newline at end of file

From 1cce40de63b283b61369b333bc796269a3ce86ec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Sep 2023 15:58:24 -0500
Subject: [PATCH 17/24] Bump requests from 2.21.0 to 2.31.0 in /scripts/monitor
 (#421)

Bumps [requests](https://github.com/psf/requests) from 2.21.0 to 2.31.0.
- [Release notes](https://github.com/psf/requests/releases)
- [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md)
- [Commits](https://github.com/psf/requests/compare/v2.21.0...v2.31.0)

---
updated-dependencies:
- dependency-name: requests
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 scripts/monitor/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/monitor/requirements.txt b/scripts/monitor/requirements.txt
index 57b914834..c1e1847e3 100644
--- a/scripts/monitor/requirements.txt
+++ b/scripts/monitor/requirements.txt
@@ -1,3 +1,3 @@
 pika==1.0.0
-requests==2.21.0
+requests==2.31.0
 python-dateutil==2.8.0

From a515dacd271749b0f25044dd2f6a70302d72aa0a Mon Sep 17 00:00:00 2001
From: Dipannita <ddey2@illinois.edu>
Date: Mon, 11 Sep 2023 15:59:04 -0500
Subject: [PATCH 18/24] ddey2/375 (#388)

* setting viewer as default

* adding to change log

---------

Co-authored-by: Rob Kooper <kooper@illinois.edu>
Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                       | 3 ++-
 app/views/spaces/invite.scala.html | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75d070e10..d5d811d50 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,9 +21,10 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 - Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
 - Added a citation.cff file
 
-## Fixed
+### Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
+- Changed default to 'Viewer' while inviting users to new spaces [#375](https://github.com/clowder-framework/clowder/issues/375)
 - Fixed bug where complex JSON metadata objects using arrays were not being indexed properly for search.
 
 
diff --git a/app/views/spaces/invite.scala.html b/app/views/spaces/invite.scala.html
index c5f425e02..5000c1117 100644
--- a/app/views/spaces/invite.scala.html
+++ b/app/views/spaces/invite.scala.html
@@ -13,7 +13,7 @@ <h3>Invite new users</h3>
                     @inputText(myForm("addressList"),'class->"form-control", 'id -> "addressList", '_label -> "Email addresses (comma-separated)", 'type -> "Email", 'multiple -> "true")
                     <span class="error hiddencomplete" id="addresserror"> Please enter at least one email address.<br></span>
 
-                    @select(myForm("role"), options(roleList), 'id -> "role_list", '_label -> "Role", 'placeholder -> "Select a default role for all recipients")
+                    @select(myForm("role"), options(roleList.reverse), 'id -> "role_list", '_label -> "Role", 'placeholder -> "Select a default role for all recipients")
                     @textarea(myForm("message"), 'class->"form-control", '_label -> "Message")
 
 

From 4f6a6e393a9cd5064977d2efc0042212ac85478f Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 12 Sep 2023 10:37:55 -0500
Subject: [PATCH 19/24] Delete elasticsearch index on deletion, omit trash
 search results (#390)

* Add trash checks for Datasets, Collections

* Update CHANGELOG.md

* Remove entries from Elasticsearch

* check if files are in trash

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                                   |  5 +++++
 app/api/Collections.scala                      |  4 ++++
 app/api/Datasets.scala                         |  5 +++++
 app/api/Files.scala                            |  3 +++
 app/services/CollectionService.scala           |  2 ++
 app/services/DatasetService.scala              |  2 ++
 app/services/ElasticsearchPlugin.scala         | 18 ++++++++++++------
 app/services/FileService.scala                 |  2 ++
 .../mongodb/MongoDBCollectionService.scala     |  4 ++++
 .../mongodb/MongoDBDatasetService.scala        |  4 ++++
 app/services/mongodb/MongoDBFileService.scala  |  9 +++++++++
 11 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d5d811d50..3d373849f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+### Fixed
+
+- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377)
 
 ## Unreleased
 
diff --git a/app/api/Collections.scala b/app/api/Collections.scala
index bbfb8ba29..f917b91b4 100644
--- a/app/api/Collections.scala
+++ b/app/api/Collections.scala
@@ -157,6 +157,10 @@ class Collections @Inject() (datasets: DatasetService,
       case Some(collection) => {
         val useTrash = play.api.Play.configuration.getBoolean("useTrash").getOrElse(false)
         if (!useTrash || (useTrash && collection.trash)){
+          Logger.debug("Deleting collection from indexes " + collectionId)
+          current.plugin[ElasticsearchPlugin].foreach {
+            _.delete(collectionId.stringify)
+          }
           events.addObjectEvent(request.user , collection.id, collection.name, EventType.DELETE_COLLECTION.toString)
           collections.delete(collectionId)
           current.plugin[AdminsNotifierPlugin].foreach {
diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala
index edb7b3ca6..b59e67e8a 100644
--- a/app/api/Datasets.scala
+++ b/app/api/Datasets.scala
@@ -2041,6 +2041,11 @@ class  Datasets @Inject()(
   def deleteDatasetHelper(id: UUID, request: UserRequest[AnyContent]) = {
     datasets.get(id) match {
       case Some(dataset) => {
+        Logger.debug("Deleting dataset from indexes " + id)
+        current.plugin[ElasticsearchPlugin].foreach {
+          _.delete(id.stringify)
+        }
+
         //remove dataset from RDF triple store if triple store is used
         configuration.getString("userdfSPARQLStore").getOrElse("no") match {
           case "yes" => rdfsparql.removeDatasetFromGraphs(id)
diff --git a/app/api/Files.scala b/app/api/Files.scala
index 18c0602b8..db737dfde 100644
--- a/app/api/Files.scala
+++ b/app/api/Files.scala
@@ -1697,6 +1697,9 @@ class Files @Inject()(
         current.plugin[VersusPlugin].foreach {
           _.removeFromIndexes(id)
         }
+        current.plugin[ElasticsearchPlugin].foreach {
+          _.delete(id.stringify)
+        }
         Logger.debug("Deleting file: " + file.filename)
         files.removeFile(id, Utils.baseUrl(request), request.apiKey, request.user)
 
diff --git a/app/services/CollectionService.scala b/app/services/CollectionService.scala
index 5f3c3971e..3715921ee 100644
--- a/app/services/CollectionService.scala
+++ b/app/services/CollectionService.scala
@@ -252,4 +252,6 @@ trait CollectionService {
 
   def getMetrics(): Iterator[Collection]
 
+  def isInTrash(id: UUID): Boolean
+
 }
diff --git a/app/services/DatasetService.scala b/app/services/DatasetService.scala
index 071a707bf..1949c3ae3 100644
--- a/app/services/DatasetService.scala
+++ b/app/services/DatasetService.scala
@@ -389,5 +389,7 @@ trait DatasetService {
 
   def getTrashedIds(): List[UUID]
 
+  def isInTrash(id: UUID): Boolean
+
   def recursiveArchive(dataset: Dataset, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User])
 }
diff --git a/app/services/ElasticsearchPlugin.scala b/app/services/ElasticsearchPlugin.scala
index e6e5e6ce9..5888c09f7 100644
--- a/app/services/ElasticsearchPlugin.scala
+++ b/app/services/ElasticsearchPlugin.scala
@@ -228,12 +228,18 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
     // Check permissions for each resource
     results.foreach(resource => {
       resource.resourceType match {
-        case ResourceRef.file => if (Permission.checkPermission(user, Permission.ViewFile, resource))
-          filesFound += resource.id
-        case ResourceRef.dataset => if (Permission.checkPermission(user, Permission.ViewDataset, resource))
-          datasetsFound += resource.id
-        case ResourceRef.collection => if (Permission.checkPermission(user, Permission.ViewDataset, resource))
-          collectionsFound += resource.id
+        case ResourceRef.file => {
+          if (Permission.checkPermission(user, Permission.ViewFile, resource) && !files.isInTrash(resource.id))
+            filesFound += resource.id
+        }
+        case ResourceRef.dataset => {
+          if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !datasets.isInTrash(resource.id))
+            datasetsFound += resource.id
+        }
+        case ResourceRef.collection => {
+          if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !collections.isInTrash(resource.id))
+            collectionsFound += resource.id
+        }
         case _ => {}
       }
     })
diff --git a/app/services/FileService.scala b/app/services/FileService.scala
index 9558e7daf..6d474379d 100644
--- a/app/services/FileService.scala
+++ b/app/services/FileService.scala
@@ -248,4 +248,6 @@ trait FileService {
 
   def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[File]
 
+  def isInTrash(id: UUID): Boolean
+
 }
diff --git a/app/services/mongodb/MongoDBCollectionService.scala b/app/services/mongodb/MongoDBCollectionService.scala
index 6c9c58b70..340e86a8b 100644
--- a/app/services/mongodb/MongoDBCollectionService.scala
+++ b/app/services/mongodb/MongoDBCollectionService.scala
@@ -1116,6 +1116,10 @@ class MongoDBCollectionService @Inject() (
     Collection.find(MongoDBObject("trash" -> false)).toIterator
   }
 
+  def isInTrash(id: UUID): Boolean = {
+    Collection.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined
+  }
+
   private def isSubCollectionIdInCollection(subCollectionId: UUID, collection: Collection) : Boolean = {
     if (collection.child_collection_ids.contains(subCollectionId)){
       return true
diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala
index f4c3b1d66..a02677c23 100644
--- a/app/services/mongodb/MongoDBDatasetService.scala
+++ b/app/services/mongodb/MongoDBDatasetService.scala
@@ -1686,6 +1686,10 @@ class MongoDBDatasetService @Inject() (
     trashedIds.toList
   }
 
+  def isInTrash(id: UUID): Boolean = {
+    Dataset.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined
+  }
+
   /**
   * Recursively submit requests to archive or unarchive the contents of the given dataset.
   * NOTE: "parameters" includes "operation", which supports both archiving and unarchiving
diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala
index 52daf75ed..6d2766cf4 100644
--- a/app/services/mongodb/MongoDBFileService.scala
+++ b/app/services/mongodb/MongoDBFileService.scala
@@ -1237,6 +1237,15 @@ class MongoDBFileService @Inject() (
     until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t)))
     FileDAO.find(query)
   }
+
+  def isInTrash(id: UUID): Boolean = {
+    var foundTrash = false
+    datasets.findByFileIdAllContain(id).foreach(ds => {
+      if (ds.trash)
+        foundTrash = true
+    })
+    foundTrash
+  }
 }
 
 object FileDAO extends ModelCompanion[File, ObjectId] {

From 1e94e143686e3f457e1b7adb7b73fcf7876bf600 Mon Sep 17 00:00:00 2001
From: Chrysovalantis Constantinou <41321803+cconsta1@users.noreply.github.com>
Date: Tue, 12 Sep 2023 18:43:16 +0300
Subject: [PATCH 20/24] Added Google's model viewer within viewer_three.js
 (#372)

* New IFC previewer

* Added an IFC previewer and a GLTF previewer

* Changelog update

* Fix positioning issues of three.js previewer

* Added changes to CHANGELOG.md

* Improving the 3D model viewer

* Remove the console.log

---------

Co-authored-by: Chrysovalantis Constantinou <cconsta1@Chrysovalantiss-MacBook-Pro.local>
Co-authored-by: Luigi Marini <lmarini@users.noreply.github.com>
Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                                  |   2 +
 .../previewers/ifc_previewer/package.json     |   2 +-
 .../previewers/three_js/package.json          |   4 +-
 .../{loading_screen.css => style.css}         |   5 +
 .../previewers/three_js/viewer_three.js       | 515 +++++++++---------
 5 files changed, 281 insertions(+), 247 deletions(-)
 rename public/javascripts/previewers/three_js/{loading_screen.css => style.css} (97%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d373849f..25fd9e4b6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,12 +25,14 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 - Documentation on how to do easy testing of pull requests
 - Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
 - Added a citation.cff file
+- Added Google's model viewer within viewer_three.js previewer
 
 ### Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
 - Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
 - Changed default to 'Viewer' while inviting users to new spaces [#375](https://github.com/clowder-framework/clowder/issues/375)
 - Fixed bug where complex JSON metadata objects using arrays were not being indexed properly for search.
+- Fixed positioning problems related to how the 3D models appear on the screen
 
 
 ## 1.21.0 - 2022-08-23
diff --git a/public/javascripts/previewers/ifc_previewer/package.json b/public/javascripts/previewers/ifc_previewer/package.json
index de59e479f..17e354593 100644
--- a/public/javascripts/previewers/ifc_previewer/package.json
+++ b/public/javascripts/previewers/ifc_previewer/package.json
@@ -2,5 +2,5 @@
   "name": "IFC-Viewer",
   "main": "ifc_viewer.js",
   "file": true,
-  "contentType": ["model/ifc", "model/IFC", "application/octet-stream"]
+  "contentType": ["model/ifc", "model/IFC"]
 }
\ No newline at end of file
diff --git a/public/javascripts/previewers/three_js/package.json b/public/javascripts/previewers/three_js/package.json
index a4b9bf6a2..4cb94b49a 100644
--- a/public/javascripts/previewers/three_js/package.json
+++ b/public/javascripts/previewers/three_js/package.json
@@ -1,6 +1,6 @@
 {
-  "name": "Three-JS",
+  "name": "3D Viewer",
   "main": "viewer_three.js",
   "file": true,
-  "contentType": [ "model/fbx", "model/FBX","model/gltf","model/GLTF", "model/glb","model/GLB", "application/octet-stream"]
+  "contentType": [ "model/fbx", "model/FBX","model/gltf","model/GLTF", "model/glb","model/GLB"]
 }
\ No newline at end of file
diff --git a/public/javascripts/previewers/three_js/loading_screen.css b/public/javascripts/previewers/three_js/style.css
similarity index 97%
rename from public/javascripts/previewers/three_js/loading_screen.css
rename to public/javascripts/previewers/three_js/style.css
index c4a6cfe6c..4d48d161e 100644
--- a/public/javascripts/previewers/three_js/loading_screen.css
+++ b/public/javascripts/previewers/three_js/style.css
@@ -84,4 +84,9 @@ html, body {
         -ms-transform: rotate(360deg);
         transform: rotate(360deg);
     }
+}
+
+model-viewer {
+    width: 640px;
+    height: 480px;
 }
\ No newline at end of file
diff --git a/public/javascripts/previewers/three_js/viewer_three.js b/public/javascripts/previewers/three_js/viewer_three.js
index 30bfa1488..523e21451 100644
--- a/public/javascripts/previewers/three_js/viewer_three.js
+++ b/public/javascripts/previewers/three_js/viewer_three.js
@@ -15,6 +15,8 @@
  *                  First functional version
  * + 11/08/21 (cc): Added loading screen and ability
  *                  to load gltf files
+ * 
+ * + 07/05/22 (cc): Bug fixes
 
  ****************************************************************/
 
@@ -42,247 +44,272 @@
 
  ****************************************************************/
 
- let fileNameExtension;
- let loader;
- 
- (function ($, Configuration) {
-     let useTab = Configuration.tab;
-     let referenceUrl = Configuration.url;
-     let previewer = Configuration.previewer;
- 
-     $(useTab).append('<section id="loading-screen">\n' +
-         '\n' +
-         '\t<div id="loader"></div>\n' +
-         '\n' +
-         '</section>')
- 
-     $(useTab).append('<link rel="stylesheet" type="text/css" href = "/assets/javascripts/previewers/three_js/loading_screen.css" >');
- 
-     let fileName = $('#file-name-title').text().trim();
-     fileNameExtension = fileName.split('.').pop();
- 
-     let scripts = [
-         "three.min.js",
-         "stats.min.js",
-         "OrbitControls.js",
-         "FBXLoader.js",
-         "GLTFLoader.js",
-         "DRACOLoader.js",
-         "fflate.min.js"
-     ];
- 
-     for (let index = 0; index < scripts.length; index++) {
-         let s = document.createElement("script");
-         s.type = "text/javascript";
-         s.src = previewer + "/js/" + scripts[index];
-         $(useTab).append(s);
-     }
- 
-     $(document).ready(function () {
-         init(referenceUrl);
-         animate();
-     });
- 
- }(jQuery, Configuration));
- 
- let camera, scene, renderer, stats;
- const clock = new THREE.Clock();
- let mixer;
- let previewerWidth = 640;
- let previewerHeight = 480;
- 
- function init(urlAddress) {
-     const container = document.getElementById(Configuration.tab.replace("#", ""));
- 
-     renderer = new THREE.WebGLRenderer({antialias: false, powerPreference: "high-performance"});
-     renderer.setPixelRatio(window.devicePixelRatio);
-     renderer.setSize(previewerWidth, previewerHeight);
-     renderer.shadowMap.enabled = true;
- 
-     camera = new THREE.PerspectiveCamera(30, window.innerWidth / window.innerHeight, 1, 2000000);
-     camera.position.set(0, 100, 1250);
- 
-     scene = new THREE.Scene();
-     scene.background = new THREE.Color(0x444444);
-     //scene.fog = new THREE.Fog( 0xa0a0a0, 200, 1000 );
- 
-     const hemiLight = new THREE.HemisphereLight(0xffffff, 0x444444);
-     hemiLight.position.set(0, 200, 0);
-     scene.add(hemiLight);
- 
-     const dirLight = new THREE.DirectionalLight(0xffffff);
-     dirLight.position.set(0, 200, 100);
-     dirLight.castShadow = true;
-     dirLight.shadow.camera.top = 180;
-     dirLight.shadow.camera.bottom = -100;
-     dirLight.shadow.camera.left = -120;
-     dirLight.shadow.camera.right = 120;
-     scene.add(dirLight);
- 
-     const controls = new THREE.OrbitControls(camera, renderer.domElement);
-     controls.target.set(0, 100, 0);
-     controls.update();
-     container.appendChild(renderer.domElement);
- 
-     // scene.add( new THREE.CameraHelper( dirLight.shadow.camera ) );
- 
-     // ground
-     // const mesh = new THREE.Mesh(
-     //     new THREE.PlaneGeometry( 4000, 4000 ),
-     //     new THREE.MeshPhongMaterial( { color: 0x999999, depthWrite: false } )
-     // );
-     //
-     // mesh.rotation.x = - Math.PI / 2;
-     // mesh.receiveShadow = true;
-     // scene.add( mesh );
-     //
-     // const grid = new THREE.GridHelper( 4000, 20, 0x000000, 0x000000 );
-     // grid.material.opacity = 0.2;
-     // grid.material.transparent = true;
-     // scene.add( grid );
- 
-     const loadingManager = new THREE.LoadingManager(() => {
-         const loadingScreen = document.getElementById('loading-screen');
-         loadingScreen.classList.add('fade-out');
-         loadingScreen.addEventListener('transitionend', onTransitionEnd);
-     });
- 
-     if (fileNameExtension === 'fbx') {
-         loader = new THREE.FBXLoader(loadingManager);
-     }
- 
-     if (fileNameExtension === 'gltf' || fileNameExtension === 'glb') {
-         let dracopath = "/assets/javascripts/previewers/three_js/draco/"
-
-         loader = new THREE.GLTFLoader(loadingManager);
-
-         const dracoLoader = new THREE.DRACOLoader(loadingManager);
-
-         dracoLoader.setDecoderPath(dracopath);
-
-         dracoLoader.setDecoderConfig({ type: 'js' });
-
-         loader.setDRACOLoader(dracoLoader);
-     }
- 
-     loader.setPath(urlAddress);
- 
-     //loader.setPath("/assets/javascripts/previewers/three_js/models/chimpanzee/source/");
- 
-     //const texture = new THREE.TextureLoader().setPath('/assets/javascripts/previewers/three_js/models/chimpanzee/source/');
- 
-     loader.load('', function (object) {
- 
-         if (fileNameExtension === 'gltf' || fileNameExtension === 'glb') {
-             object = object.scene;
-         }
- 
-         object.traverse(function (child) {
- 
-             if (child.isMesh) {
- 
-                 child.castShadow = true;
-                 child.receiveShadow = true;
- 
-             }
- 
-         });
- 
-         if (object.animations[0]) {
-             mixer = new THREE.AnimationMixer(object);
- 
-             const action = mixer.clipAction(object.animations[0]);
-             action.play();
-         }
- 
-         const box = new THREE.Box3().setFromObject(object);
-         const size = box.getSize(new THREE.Vector3()).length();
-         const center = box.getCenter(new THREE.Vector3());
- 
-         controls.reset();
- 
-         object.position.x += (object.position.x - center.x);
-         object.position.y += (object.position.y - center.y);
-         object.position.z += (object.position.z - center.z);
- 
-         controls.maxDistance = size * 10;
-         camera.near = size / 100;
-         camera.far = size * 100;
- 
-         camera.updateProjectionMatrix();
-         scene.add(object);
-
-         // object.traverse( function ( child ) {
-         //
-         //     if ( child.isMesh ) {
-         //         child.castShadow = true;
-         //         child.receiveShadow = true;
-         //
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('EYESLP_defaultMat_BaseColor.png')});
-         //     }
-         //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('EYESLP_defaultMat_Normal.png')});
-         //     }
-         //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('LOW_POLY_defaultMat_AO.png')});
-         //     }
-         //     //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('LOW_POLY_defaultMat_BaseColor.png')});
-         //     }
-         //     //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('LOW_POLY_defaultMat_Metallic.png')});
-         //     }
-         //     //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('LOW_POLY_defaultMat_Roughness.png')});
-         //     }
-         //
-         //     if ( child.isMesh ) {
-         //         child.material = new THREE.MeshBasicMaterial({map: texture.load('Pruyebanm.jpg')});
-         //     }
-         //
-         // } );
-         //
-         // scene.add( object );
- 
-     });
- 
- 
-     window.addEventListener('resize', onWindowResize);
- 
-     // stats
-     stats = new Stats();
-     //container.appendChild( stats.dom );
- 
- }
- 
- function onWindowResize() {
- 
-     camera.aspect = window.innerWidth / window.innerHeight;
-     camera.updateProjectionMatrix();
- 
-     renderer.setSize(window.innerWidth, window.innerHeight);
- 
- }
- 
- function onTransitionEnd(event) {
- 
-     event.target.remove();
- 
- }
- 
- function animate() {
-     requestAnimationFrame(animate);
- 
-     const delta = clock.getDelta();
- 
-     if (mixer) mixer.update(delta);
- 
-     renderer.render(scene, camera);
- 
-     stats.update();
- }
\ No newline at end of file
+/**
+ * Jquery 
+ */
+
+let loader;
+let useTab;
+let referenceUrl;
+let previewer;
+let fileName;
+let fileNameExtension;
+
+let scripts = [
+    "three.min.js",
+    "stats.min.js",
+    "OrbitControls.js",
+    "FBXLoader.js",
+    // "GLTFLoader.js",
+    // "DRACOLoader.js",
+    "fflate.min.js"
+];
+
+(function ($, Configuration) {
+    useTab = Configuration.tab;
+    referenceUrl = Configuration.url;
+    previewer = Configuration.previewer;
+
+    fileName = $('#file-name-title').text().trim();
+    fileNameExtension = fileName.split('.').pop();
+
+    $(useTab).append('<link rel="stylesheet" type="text/css" href = "/assets/javascripts/previewers/three_js/style.css" >');
+
+    /**
+     * Decipher whether the uploaded file is an 
+     * FBX or a GLTF (GLB) file and load the appropriate model loader 
+     * For the special case of GLTF/GLB model usel Google's
+     * model viewer 
+     */
+
+    for (let index = 0; index < scripts.length; index++) {
+        let s = document.createElement("script");
+        s.type = "text/javascript";
+        s.src = previewer + "/js/" + scripts[index];
+        $(useTab).append(s);
+    }
+
+    if (fileNameExtension === 'fbx') {
+
+        $(useTab).append('<section id="loading-screen">\n' +
+            '\n' +
+            '\t<div id="loader"></div>\n' +
+            '\n' +
+            '</section>')
+
+        $(document).ready(function () {
+            init(referenceUrl);
+            animate();
+        });
+    }
+
+    if (fileNameExtension === 'gltf' || fileNameExtension === 'glb') {
+
+        $(useTab).append('<script nomodule src="https://www.unpkg.com/@google/model-viewer@1.4.1/dist/model-viewer-legacy.js"></script>')
+
+
+        $(useTab).append("<model-viewer id='mv-demo' ar ar-modes='scene-viewer webxr' shadow-intensity='1'  auto-rotate camera-controls enable-pan src='" + referenceUrl + "'></model-viewer>")
+    }
+
+}(jQuery, Configuration));
+
+/**
+ * Three.js
+ */
+
+let camera, scene, renderer, stats;
+let mixer;
+let previewerWidth = 640;
+let previewerHeight = 480;
+const clock = new THREE.Clock();
+
+function init(urlAddress) {
+    const container = document.getElementById(Configuration.tab.replace("#", ""));
+
+
+    /**
+     * Renderer
+     */
+
+    renderer = new THREE.WebGLRenderer(
+        {
+            antialias: false
+        }
+    );
+
+    //renderer.physicallyCorrectLights = true;
+    renderer.outputEncoding = THREE.sRGBEncoding;
+    renderer.setClearColor(0xcccccc);
+    renderer.setPixelRatio(window.devicePixelRatio);
+    renderer.setSize(previewerWidth, previewerHeight);
+    renderer.shadowMap.enabled = true;
+
+    /**
+     * Camera
+     */
+
+    camera = new THREE.PerspectiveCamera(55, previewerWidth / previewerHeight, 0.1, 1000);
+    //camera.position.set(0, 20, 100);
+
+    /**
+     * Scene
+     */
+
+    scene = new THREE.Scene();
+    scene.background = new THREE.Color("White");
+    //scene.fog = new THREE.Fog( 0xa0a0a0, 200, 1000 );
+
+    /**
+     * Lights
+     */
+
+    const hemiLight1 = new THREE.HemisphereLight(0xffffff, 0x444444);
+    const hemiLight2 = new THREE.HemisphereLight(0xffffff, 1);
+    //hemiLight.position.set(0, -20, 0);
+    scene.add(hemiLight1, hemiLight2);
+
+    const ambientLight = new THREE.AmbientLight(0xffffff, 1);
+    //ambientLight.position.set(0, 200, 0);
+    scene.add(ambientLight);
+
+    const dirLight = new THREE.DirectionalLight(0xffffff, 0.8);
+    //dirLight.position.set(0, 4, 4);
+
+    // const helper = new THREE.DirectionalLightHelper(dirLight, 2);
+    // scene.add(helper);
+    dirLight.castShadow = true;
+    // dirLight.shadow.camera.top = 180;
+    // dirLight.shadow.camera.bottom = -100;
+    // dirLight.shadow.camera.left = -120;
+    // dirLight.shadow.camera.right = 120;
+    scene.add(dirLight);
+
+    /**
+     * Orbit controls
+     */
+
+    const controls = new THREE.OrbitControls(camera, renderer.domElement);
+    //controls.target.set(0, 100, 0);
+    controls.update();
+    container.appendChild(renderer.domElement);
+
+    // scene.add( new THREE.CameraHelper( dirLight.shadow.camera ) );
+
+    // ground
+    // const mesh = new THREE.Mesh(
+    //     new THREE.PlaneGeometry(1000, 1000),
+    //     new THREE.MeshPhongMaterial({ color: 0x999999, depthWrite: false })
+    // );
+    // mesh.rotation.x = - Math.PI / 2;
+    // mesh.receiveShadow = true;
+    // mesh.material.transparent = false;
+    // scene.add(mesh);
+
+    // const grid = new THREE.GridHelper(1000, 100, 0x000000, 0x000000);
+    // grid.material.opacity = 0.3;
+    // grid.material.transparent = true;
+    // scene.add(grid);
+
+    /**
+     * Loading manager
+     */
+
+    // The loading manager is needed for the loading screen
+
+    const loadingManager = new THREE.LoadingManager(() => {
+        const loadingScreen = document.getElementById('loading-screen');
+        loadingScreen.classList.add('fade-out');
+        loadingScreen.addEventListener('transitionend', onTransitionEnd);
+    });
+
+    loader = new THREE.FBXLoader(loadingManager);
+
+    // Set loader's path to the url address of the file
+
+    loader.setPath(urlAddress);
+
+    //loader.setPath("/assets/javascripts/previewers/three_js/models/chimpanzee/source/");
+
+    loader.load('', function (object) {
+
+        if (fileNameExtension === 'gltf' || fileNameExtension === 'glb') {
+            object = object.scene;
+        }
+
+        object.traverse(function (child) {
+
+            if (child.isMesh) {
+
+                child.castShadow = true;
+                child.receiveShadow = true;
+
+            }
+
+        });
+
+        if (object.animations[0]) {
+            mixer = new THREE.AnimationMixer(object);
+
+            const action = mixer.clipAction(object.animations[0]);
+            action.play();
+        }
+
+        const box = new THREE.Box3().setFromObject(object);
+        const size = box.getSize(new THREE.Vector3()).length();
+        const center = box.getCenter(new THREE.Vector3());
+
+        //controls.reset();
+
+        object.position.x += (object.position.x - center.x);
+        object.position.y += (object.position.y - center.y);
+        object.position.z += (object.position.z - center.z);
+
+        controls.maxDistance = size * 10;
+        controls.minDistance = size;
+
+        camera.near = size / 100;
+        camera.far = size * 100;
+        camera.updateProjectionMatrix();
+        camera.position.copy(center);
+        camera.position.x += size / 2.0;
+        camera.position.y += size / 5.0;
+        camera.position.z += size / 2.0;
+        camera.lookAt(center);
+        controls.update();
+
+        scene.add(object);
+    });
+
+    window.addEventListener('resize', onWindowResize);
+
+    // stats
+    stats = new Stats();
+    //container.appendChild( stats.dom );
+}
+
+function onWindowResize() {
+
+    camera.aspect = window.innerWidth / window.innerHeight;
+    camera.updateProjectionMatrix();
+
+    renderer.setSize(window.innerWidth, window.innerHeight);
+
+}
+
+function onTransitionEnd(event) {
+
+    event.target.remove();
+
+}
+
+function animate() {
+    requestAnimationFrame(animate);
+
+    const delta = clock.getDelta();
+
+    if (mixer) mixer.update(delta);
+
+    renderer.render(scene, camera);
+
+    stats.update();
+}
\ No newline at end of file

From 15bd2e723a1c841402a63038c53a676d95de4377 Mon Sep 17 00:00:00 2001
From: LDeakin <ljdgit@gmail.com>
Date: Wed, 13 Sep 2023 01:46:53 +1000
Subject: [PATCH 21/24] Add some missing sections endpoints (#410)

* Add some missing sections endpoints

* rename getSections endpoint to sections

* Fix name clash in 61f451d50

---------

Co-authored-by: Luigi Marini <lmarini@illinois.edu>
---
 CHANGELOG.md                      |  1 +
 app/api/Files.scala               | 13 +++++++++++++
 app/api/Sections.scala            | 10 ++++++++--
 app/controllers/Application.scala |  3 +++
 app/models/Section.scala          | 12 ++++++++++++
 conf/routes                       |  1 +
 public/swagger.yml                | 26 ++++++++++++++++++++++++++
 7 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 25fd9e4b6..ba5525252 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ registration or heartbeat to Clowder that will restrict use of that extractor to
 - Documentation on how to do easy testing of pull requests
 - Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
 - Added a citation.cff file
+- Add sections endpoint to file API and fix missing section routes in javascriptRoutes [#410](https://github.com/clowder-framework/clowder/pull/410)
 - Added Google's model viewer within viewer_three.js previewer
 
 ### Fixed
diff --git a/app/api/Files.scala b/app/api/Files.scala
index db737dfde..69d8a3d70 100644
--- a/app/api/Files.scala
+++ b/app/api/Files.scala
@@ -32,6 +32,7 @@ class Files @Inject()(
   collections: CollectionService,
   queries: MultimediaQueryService,
   tags: TagService,
+  sections_service: SectionService,
   comments: CommentService,
   extractions: ExtractionService,
   dtsrequests:ExtractionRequestsService,
@@ -207,6 +208,18 @@ class Files @Inject()(
       }
     }
 
+  def sections(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
+    implicit val user = request.user
+    files.get(id) match {
+      case Some(file) => {
+        val sectionList = sections_service.findByFileId(id)
+        val sectionIds = sectionList.map { section => section.id }
+        Ok(toJson(sectionIds))
+      }
+      case None => NotFound(toJson("The requested file does not exist"))
+    }
+  }
+
   def getMetadataDefinitions(id: UUID, space: Option[String]) = PermissionAction(Permission.AddMetadata, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
     implicit val user = request.user
     files.get(id) match {
diff --git a/app/api/Sections.scala b/app/api/Sections.scala
index 0b7abf8c9..0e15460f4 100644
--- a/app/api/Sections.scala
+++ b/app/api/Sections.scala
@@ -67,8 +67,14 @@ class Sections @Inject()(
       Logger.debug("Getting info for section with id " + id)
       sections.get(id) match {
         case Some(section) =>
-          Ok(Json.obj("id" -> section.id.toString, "file_id" -> section.file_id.toString,
-            "startTime" -> section.startTime.getOrElse(-1).toString, "tags" -> Json.toJson(section.tags.map(_.name))))
+          Ok(Json.obj(
+              "id" -> section.id.toString,
+              "file_id" -> section.file_id.toString,
+              "startTime" -> section.startTime.getOrElse(-1).toString,
+              "tags" -> Json.toJson(section.tags.map(_.name)),
+              "area" -> Json.toJson(section.area.getOrElse(null)),
+              "description" -> Json.toJson(section.description.getOrElse(""))
+          ))
         case None => Logger.error("Section not found " + id); NotFound(toJson("Section not found, id: " + id))
       }
   }
diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala
index 344474f35..34cd2a0fa 100644
--- a/app/controllers/Application.scala
+++ b/app/controllers/Application.scala
@@ -369,6 +369,7 @@ class Application @Inject()(files: FileService, collections: CollectionService,
         api.routes.javascript.Files.unfollow,
         api.routes.javascript.Files.getTechnicalMetadataJSON,
         api.routes.javascript.Files.filePreviewsList,
+        api.routes.javascript.Files.sections,
         api.routes.javascript.Files.updateMetadata,
         api.routes.javascript.Files.addMetadata,
         api.routes.javascript.Files.getMetadataDefinitions,
@@ -382,7 +383,9 @@ class Application @Inject()(files: FileService, collections: CollectionService,
         api.routes.javascript.Search.search,
         api.routes.javascript.Search.searchJson,
         api.routes.javascript.Sections.add,
+        api.routes.javascript.Sections.get,
         api.routes.javascript.Sections.delete,
+        api.routes.javascript.Sections.description,
         api.routes.javascript.Sections.comment,
         api.routes.javascript.Sections.getTags,
         api.routes.javascript.Sections.addTags,
diff --git a/app/models/Section.scala b/app/models/Section.scala
index 59f40c85c..8bf5da309 100644
--- a/app/models/Section.scala
+++ b/app/models/Section.scala
@@ -33,3 +33,15 @@ case class Rectangle(
   h: Double) {
   override def toString() = f"x: $x%.2f, y: $y%.2f, width: $w%.2f, height: $h%.2f"
 }
+
+object Rectangle {
+  implicit object RectangleWrites extends Writes[Rectangle] {
+    def writes(rectangle: Rectangle): JsObject = {
+      Json.obj(
+        "x" -> rectangle.x,
+        "y" -> rectangle.y,
+        "w" -> rectangle.w,
+        "h" -> rectangle.h)
+    }
+  }
+}
diff --git a/conf/routes b/conf/routes
index 9ab4f1b56..786757f97 100644
--- a/conf/routes
+++ b/conf/routes
@@ -390,6 +390,7 @@ GET            /api/files/:id/blob
 POST           /api/files/:id/remove                                                    @api.Files.removeFile(id: UUID)
 POST           /api/files/bulkRemove                                                    @api.Files.bulkDeleteFiles()
 GET            /api/files/:id/metadata                                                  @api.Files.get(id: UUID)
+GET            /api/files/:id/sections                                                  @api.Files.sections(id: UUID)
 POST           /api/files/:id/metadata                                                  @api.Files.addMetadata(id: UUID)
 GET            /api/files/:id/metadataDefinitions                                       @api.Files.getMetadataDefinitions(id: UUID, space: Option[String] ?= None)
 POST           /api/files/:id/updateMetadata                                            @api.Files.updateMetadata(id: UUID, extractor_id: String)
diff --git a/public/swagger.yml b/public/swagger.yml
index b76ba7c2a..c312d4462 100644
--- a/public/swagger.yml
+++ b/public/swagger.yml
@@ -351,6 +351,32 @@ paths:
         404:
           $ref: '#/components/responses/NotFound'
 
+  /files/{id}/sections:
+    get:
+      tags:
+        - files
+      summary: Get the sections of a file
+      description: |
+        Get the sections of a file.
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/UUID'
+        404:
+          $ref: '#/components/responses/NotFound'
+      
+
   /files/{id}/paths:
     get:
       tags:

From e8801421bd2118abfb1c8985b2b25adf58c20da3 Mon Sep 17 00:00:00 2001
From: Luigi Marini <lmarini@illinois.edu>
Date: Tue, 12 Sep 2023 10:52:56 -0500
Subject: [PATCH 22/24] Added Lachlan Deakin to CONTRIBUTORS.md.

---
 CONTRIBUTORS.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 380b4e1ce..43a6c68d6 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -20,6 +20,7 @@ Following is a list of contributors in alphabetical order:
 - Kastan Day
 - Kaveh Karimi-Asli
 - Kenton McHenry
+- Lachlan Deakin
 - Luigi Marini
 - Maria-Spyridoula Tzima
 - Mario Felarca

From 5e0e97977bbac02737bf4282c8f31b50738dc745 Mon Sep 17 00:00:00 2001
From: Luigi Marini <lmarini@illinois.edu>
Date: Tue, 12 Sep 2023 13:57:17 -0500
Subject: [PATCH 23/24] Updated version and changelog for 1.22.0 release.

---
 CHANGELOG.md           | 30 ++++++++++++------------------
 doc/src/sphinx/conf.py |  2 +-
 project/Build.scala    |  2 +-
 public/swagger.yml     |  2 +-
 version.sh             |  2 +-
 5 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba5525252..cc2f90afb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,36 +5,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## Unreleased
-
-### Fixed
-
-- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377)
-
-## Unreleased
+## 1.22.0 - 2023-09-12
 
 ### Added
 - Users can be marked as ReadOnly [#405](https://github.com/clowder-framework/clowder/issues/405)
-- Added Trash button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
-- Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Delete button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
+- When parameter on API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
 - Extractors can now specify an extractor_key and an owner (email address) when sending a
 registration or heartbeat to Clowder that will restrict use of that extractor to them.
-- Added a dropdown menu to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
-- Add SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417)
+- Added a dropdown menu to space listing to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
+- Support for SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417)
 - Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419)
 - Documentation on how to do easy testing of pull requests
-- Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
-- Added a citation.cff file
-- Add sections endpoint to file API and fix missing section routes in javascriptRoutes [#410](https://github.com/clowder-framework/clowder/pull/410)
-- Added Google's model viewer within viewer_three.js previewer
+- citation.cff
+- New GET sections endpoint to file API and fix missing section routes in javascriptRoutes [#410](https://github.com/clowder-framework/clowder/pull/410)
+- Google's model viewer within viewer_three.js previewer
 
 ### Fixed
 - Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
-- Disabled button while create dataset ajax call is still going on [#311](https://github.com/clowder-framework/clowder/issues/311)
+- Disabled button after creating new dataset [#311](https://github.com/clowder-framework/clowder/issues/311)
 - Changed default to 'Viewer' while inviting users to new spaces [#375](https://github.com/clowder-framework/clowder/issues/375)
-- Fixed bug where complex JSON metadata objects using arrays were not being indexed properly for search.
+- Complex extracted JSON metadata objects using arrays are now being indexed properly for search.
 - Fixed positioning problems related to how the 3D models appear on the screen
-
+- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377)
+- Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
 
 ## 1.21.0 - 2022-08-23
 
diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py
index d0e4e68f9..0997b9b1d 100644
--- a/doc/src/sphinx/conf.py
+++ b/doc/src/sphinx/conf.py
@@ -22,7 +22,7 @@
 author = 'Luigi Marini'
 
 # The full version, including alpha/beta/rc tags
-release = '1.21.0'
+release = '1.22.0'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index 46f9eec06..cebdf2b27 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -13,7 +13,7 @@ import NativePackagerKeys._
 object ApplicationBuild extends Build {
 
   val appName = "clowder"
-  val version = "1.21.0"
+  val version = "1.22.0"
   val jvm = "1.7"
 
   def appVersion: String = {
diff --git a/public/swagger.yml b/public/swagger.yml
index c312d4462..5e221cda4 100644
--- a/public/swagger.yml
+++ b/public/swagger.yml
@@ -9,7 +9,7 @@ info:
     Clowder is a customizable and scalable data management system to support any
     data format and multiple research domains. It is under active development
     and deployed for a variety of research projects.
-  version: 1.21.0
+  version: 1.22.0
   termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
   contact:
     name: Clowder
diff --git a/version.sh b/version.sh
index dbdcfeb8e..349b1c28a 100755
--- a/version.sh
+++ b/version.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION="1.19.2"
+VERSION="1.22.0"
 
 sed -i~ "s#^  val version = \".*\"\$#  val version = \"${VERSION}\"#" project/Build.scala
 sed -i~ "s#^  version: .*\$#  version: ${VERSION}#" public/swagger.yml

From c5d8ace5dd7dd67638ee9b877eca93b07de6adf1 Mon Sep 17 00:00:00 2001
From: Rob Kooper <kooper@illinois.edu>
Date: Tue, 26 Sep 2023 10:35:45 -0500
Subject: [PATCH 24/24] make sure to bump citation

---
 .gitignore   | 1 +
 citation.cff | 2 +-
 version.sh   | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 3dd6fdf62..e49cb504f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,4 @@ doc/src/sphinx/_build
 .DS_Store
 /doc/src/sphinx/_build
 **/.c9/*
+*~
diff --git a/citation.cff b/citation.cff
index e0affaf67..5a092f829 100644
--- a/citation.cff
+++ b/citation.cff
@@ -3,7 +3,7 @@ message: If you use this software, please cite it using these metadata.
 title: "Clowder: Open Source Data Management for Long Tail Data"
 abstract: "A customizable and scalable data management system you can install in the cloud or on your own hardware."
 type: software
-version: "1.21.0"
+version: "1.22.0"
 license: "NCSA"
 repository-code: "https://github.com/clowder-framework/clowder"
 keywords:
diff --git a/version.sh b/version.sh
index 349b1c28a..7cbc0cf95 100755
--- a/version.sh
+++ b/version.sh
@@ -3,6 +3,7 @@
 VERSION="1.22.0"
 
 sed -i~ "s#^  val version = \".*\"\$#  val version = \"${VERSION}\"#" project/Build.scala
+sed -i~ "s#^version: .*\$#version: \"${VERSION}\"#" citation.cff
 sed -i~ "s#^  version: .*\$#  version: ${VERSION}#" public/swagger.yml
 sed -i~ "s#^release = '.*'\$#release = '${VERSION}'#" doc/src/sphinx/conf.py
 sed -i~ "s/^##.*unreleased.*$/## ${VERSION} - $(date +'%Y-%m-%d')/i" CHANGELOG.md