Merge pull request #423 from clowder-framework/release/1.22.0

Release/1.22.0

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -46,9 +46,10 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
         uses: actions/cache@v1
         with:
@@ -84,7 +85,7 @@ jobs:
         ports:
           - 27017:27017
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -100,16 +101,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
@@ -128,7 +130,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -144,16 +146,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
@@ -204,7 +207,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: github branch
         run: |
           if [ "${{ github.event.release.target_commitish }}" != "" ]; then
@@ -220,16 +223,17 @@ jobs:
           else
             echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v3
         with:
-          java-version: 1.8
+          distribution: 'zulu'
+          java-version: 8
       - name: Cache SBT ivy cache
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.ivy2/cache
           key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }}
       - name: Cache SBT
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/.sbt
           key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}

.github/workflows/swagger.yml

@@ -20,7 +20,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: openapi-lint 
         uses: mbowman100/swagger-validator-action@master

.gitignore

@@ -30,3 +30,4 @@ doc/src/sphinx/_build
 .DS_Store
 /doc/src/sphinx/_build
 **/.c9/*
+*~

CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.22.0 - 2023-09-12
+
+### Added
+- Users can be marked as ReadOnly [#405](https://github.com/clowder-framework/clowder/issues/405)
+- Delete button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347)
+- When parameter on API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Extractors can now specify an extractor_key and an owner (email address) when sending a
+registration or heartbeat to Clowder that will restrict use of that extractor to them.
+- Added a dropdown menu to space listing to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374)
+- Support for SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417)
+- Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419)
+- Documentation on how to do easy testing of pull requests
+- citation.cff
+- New GET sections endpoint to file API and fix missing section routes in javascriptRoutes [#410](https://github.com/clowder-framework/clowder/pull/410)
+- Google's model viewer within viewer_three.js previewer
+
+### Fixed
+- Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI  [386](https://github.com/clowder-framework/clowder/issues/386)
+- Disabled button after creating new dataset [#311](https://github.com/clowder-framework/clowder/issues/311)
+- Changed default to 'Viewer' while inviting users to new spaces [#375](https://github.com/clowder-framework/clowder/issues/375)
+- Complex extracted JSON metadata objects using arrays are now being indexed properly for search.
+- Fixed positioning problems related to how the 3D models appear on the screen
+- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377)
+- Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395)
 
 ## 1.21.0 - 2022-08-23
 

CONTRIBUTORS.md

@@ -20,6 +20,7 @@ Following is a list of contributors in alphabetical order:
 - Kastan Day
 - Kaveh Karimi-Asli
 - Kenton McHenry
+- Lachlan Deakin
 - Luigi Marini
 - Maria-Spyridoula Tzima
 - Mario Felarca

TESTING.md

@@ -0,0 +1,82 @@
+# Testing Clowder PR
+
+Download the [docker-compose.yml](https://raw.githubusercontent.com/clowder-framework/clowder/develop/docker-compose.yml) file in a new folder. Next create a .env file with the following data:
+
+```ini
+COMPOSE_PROJECT_NAME=clowder
+TRAEFIK_HOST=Host:yourmachine.ncsa.illinois.edu;
+TRAEFIK_HTTP_PORT=80
+TRAEFIK_HTTPS_PORT=443
+TRAEFIK_HTTPS_OPTIONS=TLS
+TRAEFIK_ACME_ENABLE=true
+TRAEFIK_ACME_EMAIL[email protected]
+TRAEFIK_HTTP_REDIRECT=Redirect.EntryPoint:https
+CLOWDER_SSL=true
+CLOWDER_ADMINS[email protected]
+```
+
+Next create a docker-compose.override.yml file:
+
+```yaml
+version: '3.5'
+
+services:
+  # point to the PR image (in this case PR-404)
+  clowder:
+    image: ghcr.io/clowder-framework/clowder:PR-404
+
+  # add any more extractors if you want
+  # extract preview image
+  imagepreview:
+    image: clowder/extractors-image-preview:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+
+  # extract image metadata
+  imagemetadata:
+    image: clowder/extractors-image-metadata:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+
+  # digest
+  digest:
+    image: clowder/extractors-digest:latest
+    restart: unless-stopped
+    networks:
+      - clowder
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F}
+```
+
+It is best practice to start with a `docker-compose pull` to make sure you have all the latest versions of the containers, followed by a  `docker-compose up -d`. This will start all containers. You should be able to go to https://yourmachine.ncsa.illinois.edu.
+
+If this is the first time running the stack (or if you removed the mongo database), you will need to create the initial user again:
+
+```bash
+docker run --rm -it \
+    --network clowder_clowder \
+    -e "FIRSTNAME=Admin" \
+    -e "LASTNAME=User" \
+    -e "ADMIN=true" \
+    -e "PASSWORD=areallygoodpassword" \
+    -e "[email protected]" \
+    -e "MONGO_URI=mongodb://mongo:27017/clowder" \
+    clowder/mongo-init
+```
+
+

app/api/Admin.scala

@@ -125,10 +125,10 @@ class Admin @Inject() (userService: UserService,
       list.foreach(id =>
         userService.findById(UUID(id)) match {
           case Some(u: ClowderUser) => {
-            if (u.status == UserStatus.Inactive) {
+            if (u.status != UserStatus.Active) {
               userService.update(u.copy(status = UserStatus.Active))
-              val subject = s"[${AppConfiguration.getDisplayName}] account activated"
-              val body = views.html.emails.userActivated(u, active = true)(request)
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now active"
+              val body = views.html.emails.userChanged(u, "activated")(request)
               util.Mail.sendEmail(subject, request.user, u, body)
             }
           }
@@ -138,10 +138,10 @@ class Admin @Inject() (userService: UserService,
       list.foreach(id =>
         userService.findById(UUID(id)) match {
           case Some(u: ClowderUser) => {
-            if (!(u.status == UserStatus.Inactive)) {
+            if (u.status != UserStatus.Inactive) {
               userService.update(u.copy(status = UserStatus.Inactive))
-              val subject = s"[${AppConfiguration.getDisplayName}] account deactivated"
-              val body = views.html.emails.userActivated(u, active = false)(request)
+              val subject = s"[${AppConfiguration.getDisplayName}] account is deactivated"
+              val body = views.html.emails.userChanged(u, "deactivated")(request)
               util.Mail.sendEmail(subject, request.user, u, body)
             }
           }
@@ -150,26 +150,27 @@ class Admin @Inject() (userService: UserService,
     (request.body \ "admin").asOpt[List[String]].foreach(list =>
       list.foreach(id =>
         userService.findById(UUID(id)) match {
-          case Some(u: ClowderUser) if (u.status == UserStatus.Active) => {
-
-            userService.update(u.copy(status = UserStatus.Admin))
-            val subject = s"[${AppConfiguration.getDisplayName}] admin access granted"
-            val body = views.html.emails.userAdmin(u, admin = true)(request)
-            util.Mail.sendEmail(subject, request.user, u, body)
-
+          case Some(u: ClowderUser) => {
+            if (u.status != UserStatus.Admin) {
+              userService.update(u.copy(status = UserStatus.Admin))
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now an admin"
+              val body = views.html.emails.userChanged(u, "an admin account")(request)
+              util.Mail.sendEmail(subject, request.user, u, body)
+            }
           }
           case _ => Logger.error(s"Could not update user with id=${id}")
         }))
-    (request.body \ "unadmin").asOpt[List[String]].foreach(list =>
+    (request.body \ "readonly").asOpt[List[String]].foreach(list =>
       list.foreach(id =>
         userService.findById(UUID(id)) match {
-          case Some(u: ClowderUser) if (u.status == UserStatus.Admin) => {
-            userService.update(u.copy(status = UserStatus.Active))
-            val subject = s"[${AppConfiguration.getDisplayName}] admin access revoked"
-            val body = views.html.emails.userAdmin(u, admin = false)(request)
-            util.Mail.sendEmail(subject, request.user, u, body)
+          case Some(u: ClowderUser) => {
+            if (u.status != UserStatus.ReadOnly) {
+              userService.update(u.copy(status = UserStatus.ReadOnly))
+              val subject = s"[${AppConfiguration.getDisplayName}] account is now an read-only"
+              val body = views.html.emails.userChanged(u, "read-only")(request)
+              util.Mail.sendEmail(subject, request.user, u, body)
+            }
           }
-
           case _ => Logger.error(s"Could not update user with id=${id}")
         }))
     Ok(toJson(Map("status" -> "success")))

app/api/ApiController.scala

@@ -88,12 +88,13 @@ trait ApiController extends Controller {
       userRequest.user match {
         case Some(u) if !AppConfiguration.acceptedTermsOfServices(u.termsOfServices) => Future.successful(Unauthorized("Terms of Service not accepted"))
         case Some(u) if (u.status == UserStatus.Inactive) => Future.successful(Unauthorized("Account is not activated"))
+        case Some(u) if (u.status == UserStatus.ReadOnly && !api.Permission.READONLY.contains(permission) && permission != Permission.DownloadFiles) => Future.successful(Unauthorized("Account is ReadOnly"))
         case Some(u) if u.superAdminMode || Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
         case Some(u) => {
           affectedResource match {
             case Some(resource) if Permission.checkOwner(u, resource) => block(userRequest)
             case _ => Future.successful(Unauthorized("Not authorized"))
-           }
+          }
         }
         case None if Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest)
         case _ => Future.successful(Unauthorized("Not authorized"))

app/api/Collections.scala

@@ -157,6 +157,10 @@ class Collections @Inject() (datasets: DatasetService,
       case Some(collection) => {
         val useTrash = play.api.Play.configuration.getBoolean("useTrash").getOrElse(false)
         if (!useTrash || (useTrash && collection.trash)){
+          Logger.debug("Deleting collection from indexes " + collectionId)
+          current.plugin[ElasticsearchPlugin].foreach {
+            _.delete(collectionId.stringify)
+          }
           events.addObjectEvent(request.user , collection.id, collection.name, EventType.DELETE_COLLECTION.toString)
           collections.delete(collectionId)
           current.plugin[AdminsNotifierPlugin].foreach {

app/api/Datasets.scala

@@ -882,6 +882,7 @@ class  Datasets @Inject()(
 
         datasets.index(id)
         Ok(toJson(Map("status" -> "success")))
+
       }
       case None => Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id"))
     }
@@ -928,6 +929,7 @@ class  Datasets @Inject()(
                 events.addObjectEvent(request.user, id, x.name, EventType.ADD_METADATA_DATASET.toString)
 
                 datasets.index(id)
+
                 Ok(toJson("Metadata successfully added to db"))
               }
               case e: JsError => {
@@ -2039,6 +2041,11 @@ class  Datasets @Inject()(
   def deleteDatasetHelper(id: UUID, request: UserRequest[AnyContent]) = {
     datasets.get(id) match {
       case Some(dataset) => {
+        Logger.debug("Deleting dataset from indexes " + id)
+        current.plugin[ElasticsearchPlugin].foreach {
+          _.delete(id.stringify)
+        }
+
         //remove dataset from RDF triple store if triple store is used
         configuration.getString("userdfSPARQLStore").getOrElse("no") match {
           case "yes" => rdfsparql.removeDatasetFromGraphs(id)