diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0c5d8d13..a77d911b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: github branch run: | if [ "${{ github.event.release.target_commitish }}" != "" ]; then @@ -46,9 +46,10 @@ jobs: else echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'zulu' + java-version: 8 - name: Cache SBT ivy cache uses: actions/cache@v1 with: @@ -84,7 +85,7 @@ jobs: ports: - 27017:27017 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: github branch run: | if [ "${{ github.event.release.target_commitish }}" != "" ]; then @@ -100,16 +101,17 @@ jobs: else echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'zulu' + java-version: 8 - name: Cache SBT ivy cache - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.ivy2/cache key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }} - name: Cache SBT - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.sbt key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }} @@ -128,7 +130,7 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: github branch run: | if [ "${{ github.event.release.target_commitish }}" != "" ]; then @@ -144,16 +146,17 @@ jobs: else echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'zulu' + java-version: 8 - name: Cache SBT ivy cache - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.ivy2/cache key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }} - name: Cache SBT - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.sbt key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }} @@ -204,7 +207,7 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: github branch run: | if [ "${{ github.event.release.target_commitish }}" != "" ]; then @@ -220,16 +223,17 @@ jobs: else echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v3 with: - java-version: 1.8 + distribution: 'zulu' + java-version: 8 - name: Cache SBT ivy cache - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.ivy2/cache key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('project/Build.scala') }} - name: Cache SBT - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.sbt key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }} diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index b0f328a67..b55dfb26f 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -20,7 +20,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: openapi-lint uses: mbowman100/swagger-validator-action@master diff --git a/.gitignore b/.gitignore index 3dd6fdf62..e49cb504f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ doc/src/sphinx/_build .DS_Store /doc/src/sphinx/_build **/.c9/* +*~ diff --git a/CHANGELOG.md b/CHANGELOG.md index 85bd19a86..cc2f90afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.22.0 - 2023-09-12 + +### Added +- Users can be marked as ReadOnly [#405](https://github.com/clowder-framework/clowder/issues/405) +- Delete button to delete section [#347](https://github.com/clowder-framework/clowder/issues/347) +- When parameter on API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266) +- Extractors can now specify an extractor_key and an owner (email address) when sending a +registration or heartbeat to Clowder that will restrict use of that extractor to them. +- Added a dropdown menu to space listing to select all spaces, your spaces and also the spaces you have access to. [#374](https://github.com/clowder-framework/clowder/issues/374) +- Support for SMTP_FROM in docker-compose yml file. [#417](https://github.com/clowder-framework/clowder/issues/417) +- Keycloak provider with secure social [#419](https://github.com/clowder-framework/clowder/issues/419) +- Documentation on how to do easy testing of pull requests +- citation.cff +- New GET sections endpoint to file API and fix missing section routes in javascriptRoutes [#410](https://github.com/clowder-framework/clowder/pull/410) +- Google's model viewer within viewer_three.js previewer + +### Fixed +- Updated lastModifiesDate when updating file or metadata to a dataset, added lastModified to UI [386](https://github.com/clowder-framework/clowder/issues/386) +- Disabled button after creating new dataset [#311](https://github.com/clowder-framework/clowder/issues/311) +- Changed default to 'Viewer' while inviting users to new spaces [#375](https://github.com/clowder-framework/clowder/issues/375) +- Complex extracted JSON metadata objects using arrays are now being indexed properly for search. +- Fixed positioning problems related to how the 3D models appear on the screen +- Search results are checked to verify nothing has been put in trash before display [#377](https://github.com/clowder-framework/clowder/issues/377) +- Previewer source URL in the documentation to point to the Clowder GitHub repo. [#395](https://github.com/clowder-framework/clowder/issues/395) ## 1.21.0 - 2022-08-23 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 380b4e1ce..43a6c68d6 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -20,6 +20,7 @@ Following is a list of contributors in alphabetical order: - Kastan Day - Kaveh Karimi-Asli - Kenton McHenry +- Lachlan Deakin - Luigi Marini - Maria-Spyridoula Tzima - Mario Felarca diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 000000000..fdfc36690 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,82 @@ +# Testing Clowder PR + +Download the [docker-compose.yml](https://raw.githubusercontent.com/clowder-framework/clowder/develop/docker-compose.yml) file in a new folder. Next create a .env file with the following data: + +```ini +COMPOSE_PROJECT_NAME=clowder +TRAEFIK_HOST=Host:yourmachine.ncsa.illinois.edu; +TRAEFIK_HTTP_PORT=80 +TRAEFIK_HTTPS_PORT=443 +TRAEFIK_HTTPS_OPTIONS=TLS +TRAEFIK_ACME_ENABLE=true +TRAEFIK_ACME_EMAIL=youremail@ncsa.illinois.edu +TRAEFIK_HTTP_REDIRECT=Redirect.EntryPoint:https +CLOWDER_SSL=true +CLOWDER_ADMINS=youremail@ncsa.illinois.edu +``` + +Next create a docker-compose.override.yml file: + +```yaml +version: '3.5' + +services: + # point to the PR image (in this case PR-404) + clowder: + image: ghcr.io/clowder-framework/clowder:PR-404 + + # add any more extractors if you want + # extract preview image + imagepreview: + image: clowder/extractors-image-preview:latest + restart: unless-stopped + networks: + - clowder + depends_on: + rabbitmq: + condition: service_started + environment: + - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} + + # extract image metadata + imagemetadata: + image: clowder/extractors-image-metadata:latest + restart: unless-stopped + networks: + - clowder + depends_on: + rabbitmq: + condition: service_started + environment: + - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} + + # digest + digest: + image: clowder/extractors-digest:latest + restart: unless-stopped + networks: + - clowder + depends_on: + rabbitmq: + condition: service_started + environment: + - RABBITMQ_URI=${RABBITMQ_URI:-amqp://guest:guest@rabbitmq/%2F} +``` + +It is best practice to start with a `docker-compose pull` to make sure you have all the latest versions of the containers, followed by a `docker-compose up -d`. This will start all containers. You should be able to go to https://yourmachine.ncsa.illinois.edu. + +If this is the first time running the stack (or if you removed the mongo database), you will need to create the initial user again: + +```bash +docker run --rm -it \ + --network clowder_clowder \ + -e "FIRSTNAME=Admin" \ + -e "LASTNAME=User" \ + -e "ADMIN=true" \ + -e "PASSWORD=areallygoodpassword" \ + -e "EMAIL_ADDRESS=youremail@ncsa.illinois.edu" \ + -e "MONGO_URI=mongodb://mongo:27017/clowder" \ + clowder/mongo-init +``` + + diff --git a/app/api/Admin.scala b/app/api/Admin.scala index d8c878df0..27bc8f482 100644 --- a/app/api/Admin.scala +++ b/app/api/Admin.scala @@ -125,10 +125,10 @@ class Admin @Inject() (userService: UserService, list.foreach(id => userService.findById(UUID(id)) match { case Some(u: ClowderUser) => { - if (u.status == UserStatus.Inactive) { + if (u.status != UserStatus.Active) { userService.update(u.copy(status = UserStatus.Active)) - val subject = s"[${AppConfiguration.getDisplayName}] account activated" - val body = views.html.emails.userActivated(u, active = true)(request) + val subject = s"[${AppConfiguration.getDisplayName}] account is now active" + val body = views.html.emails.userChanged(u, "activated")(request) util.Mail.sendEmail(subject, request.user, u, body) } } @@ -138,10 +138,10 @@ class Admin @Inject() (userService: UserService, list.foreach(id => userService.findById(UUID(id)) match { case Some(u: ClowderUser) => { - if (!(u.status == UserStatus.Inactive)) { + if (u.status != UserStatus.Inactive) { userService.update(u.copy(status = UserStatus.Inactive)) - val subject = s"[${AppConfiguration.getDisplayName}] account deactivated" - val body = views.html.emails.userActivated(u, active = false)(request) + val subject = s"[${AppConfiguration.getDisplayName}] account is deactivated" + val body = views.html.emails.userChanged(u, "deactivated")(request) util.Mail.sendEmail(subject, request.user, u, body) } } @@ -150,26 +150,27 @@ class Admin @Inject() (userService: UserService, (request.body \ "admin").asOpt[List[String]].foreach(list => list.foreach(id => userService.findById(UUID(id)) match { - case Some(u: ClowderUser) if (u.status == UserStatus.Active) => { - - userService.update(u.copy(status = UserStatus.Admin)) - val subject = s"[${AppConfiguration.getDisplayName}] admin access granted" - val body = views.html.emails.userAdmin(u, admin = true)(request) - util.Mail.sendEmail(subject, request.user, u, body) - + case Some(u: ClowderUser) => { + if (u.status != UserStatus.Admin) { + userService.update(u.copy(status = UserStatus.Admin)) + val subject = s"[${AppConfiguration.getDisplayName}] account is now an admin" + val body = views.html.emails.userChanged(u, "an admin account")(request) + util.Mail.sendEmail(subject, request.user, u, body) + } } case _ => Logger.error(s"Could not update user with id=${id}") })) - (request.body \ "unadmin").asOpt[List[String]].foreach(list => + (request.body \ "readonly").asOpt[List[String]].foreach(list => list.foreach(id => userService.findById(UUID(id)) match { - case Some(u: ClowderUser) if (u.status == UserStatus.Admin) => { - userService.update(u.copy(status = UserStatus.Active)) - val subject = s"[${AppConfiguration.getDisplayName}] admin access revoked" - val body = views.html.emails.userAdmin(u, admin = false)(request) - util.Mail.sendEmail(subject, request.user, u, body) + case Some(u: ClowderUser) => { + if (u.status != UserStatus.ReadOnly) { + userService.update(u.copy(status = UserStatus.ReadOnly)) + val subject = s"[${AppConfiguration.getDisplayName}] account is now an read-only" + val body = views.html.emails.userChanged(u, "read-only")(request) + util.Mail.sendEmail(subject, request.user, u, body) + } } - case _ => Logger.error(s"Could not update user with id=${id}") })) Ok(toJson(Map("status" -> "success"))) diff --git a/app/api/ApiController.scala b/app/api/ApiController.scala index d996b6786..2a5c435b1 100644 --- a/app/api/ApiController.scala +++ b/app/api/ApiController.scala @@ -88,12 +88,13 @@ trait ApiController extends Controller { userRequest.user match { case Some(u) if !AppConfiguration.acceptedTermsOfServices(u.termsOfServices) => Future.successful(Unauthorized("Terms of Service not accepted")) case Some(u) if (u.status == UserStatus.Inactive) => Future.successful(Unauthorized("Account is not activated")) + case Some(u) if (u.status == UserStatus.ReadOnly && !api.Permission.READONLY.contains(permission) && permission != Permission.DownloadFiles) => Future.successful(Unauthorized("Account is ReadOnly")) case Some(u) if u.superAdminMode || Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest) case Some(u) => { affectedResource match { case Some(resource) if Permission.checkOwner(u, resource) => block(userRequest) case _ => Future.successful(Unauthorized("Not authorized")) - } + } } case None if Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest) case _ => Future.successful(Unauthorized("Not authorized")) diff --git a/app/api/Collections.scala b/app/api/Collections.scala index bbfb8ba29..f917b91b4 100644 --- a/app/api/Collections.scala +++ b/app/api/Collections.scala @@ -157,6 +157,10 @@ class Collections @Inject() (datasets: DatasetService, case Some(collection) => { val useTrash = play.api.Play.configuration.getBoolean("useTrash").getOrElse(false) if (!useTrash || (useTrash && collection.trash)){ + Logger.debug("Deleting collection from indexes " + collectionId) + current.plugin[ElasticsearchPlugin].foreach { + _.delete(collectionId.stringify) + } events.addObjectEvent(request.user , collection.id, collection.name, EventType.DELETE_COLLECTION.toString) collections.delete(collectionId) current.plugin[AdminsNotifierPlugin].foreach { diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala index c0fe3fcbc..b59e67e8a 100644 --- a/app/api/Datasets.scala +++ b/app/api/Datasets.scala @@ -882,6 +882,7 @@ class Datasets @Inject()( datasets.index(id) Ok(toJson(Map("status" -> "success"))) + } case None => Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id")) } @@ -928,6 +929,7 @@ class Datasets @Inject()( events.addObjectEvent(request.user, id, x.name, EventType.ADD_METADATA_DATASET.toString) datasets.index(id) + Ok(toJson("Metadata successfully added to db")) } case e: JsError => { @@ -2039,6 +2041,11 @@ class Datasets @Inject()( def deleteDatasetHelper(id: UUID, request: UserRequest[AnyContent]) = { datasets.get(id) match { case Some(dataset) => { + Logger.debug("Deleting dataset from indexes " + id) + current.plugin[ElasticsearchPlugin].foreach { + _.delete(id.stringify) + } + //remove dataset from RDF triple store if triple store is used configuration.getString("userdfSPARQLStore").getOrElse("no") match { case "yes" => rdfsparql.removeDatasetFromGraphs(id) diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala index 0ee8701c9..120b02472 100644 --- a/app/api/Extractions.scala +++ b/app/api/Extractions.scala @@ -127,36 +127,6 @@ class Extractions @Inject()( } } - /** - * - * Given a file id (UUID), submit this file for extraction - */ - def submitExtraction(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id)))(parse.json) { implicit request => - if (UUID.isValid(id.stringify)) { - files.get(id) match { - case Some(file) => { - // FIXME dataset not available? - routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { - case Some(jobId) => { - Ok(Json.obj("status" -> "OK", "job_id" -> jobId)) - } - case None => { - val message = "No jobId found for Extraction" - Logger.error(message) - InternalServerError(toJson(Map("status" -> "KO", "message" -> message))) - } - } - } - case None => { - Logger.error("Could not retrieve file that was just saved.") - InternalServerError("Error uploading file") - } - } //file match - } else { - BadRequest("Not valid id") - } - } - /** * For a given file id, checks for the status of all extractors processing that file. * REST endpoint GET /api/extractions/:id/status @@ -404,24 +374,24 @@ class Extractions @Inject()( Ok(jarr) } - def listExtractors(categories: List[String]) = AuthenticatedAction { implicit request => - Ok(Json.toJson(extractors.listExtractorsInfo(categories))) + def listExtractors(categories: List[String], space: Option[UUID]) = AuthenticatedAction { implicit request => + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + Ok(Json.toJson(extractors.listExtractorsInfo(categories, userid))) } - def getExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => - extractors.getExtractorInfo(extractorName) match { + def getExtractorInfo(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => + extractors.getExtractorInfo(extractorName, extractor_key, request.user) match { case Some(info) => Ok(Json.toJson(info)) case None => NotFound(Json.obj("status" -> "KO", "message" -> "Extractor info not found")) } } - def deleteExtractor(extractorName: String) = ServerAdminAction { implicit request => - extractors.deleteExtractor(extractorName) + def deleteExtractor(extractorName: String, extractor_key: Option[String]) = ServerAdminAction { implicit request => + extractors.deleteExtractor(extractorName, extractor_key) Ok(toJson(Map("status" -> "success"))) } - def addExtractorInfo() = AuthenticatedAction(parse.json) { implicit request => - + def addExtractorInfo(extractor_key: Option[String], user: Option[String]) = AuthenticatedAction(parse.json) { implicit request => // If repository is of type object, change it into an array. // This is for backward compatibility with requests from existing extractors. var requestJson = request.body \ "repository" match { @@ -438,34 +408,66 @@ class Extractions @Inject()( BadRequest(Json.obj("status" -> "KO", "message" -> JsError.toFlatJson(errors))) }, info => { - extractors.updateExtractorInfo(info) match { - case Some(u) => { - // Create/assign any default labels for this extractor - u.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") + // Check private extractor flags + val submissionInfo: Option[ExtractorInfo] = extractor_key match { + case Some(ek) => { + user match { + case None => { + Logger.error("Extractors with a private key must also specify a user email.") + None } - extractors.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) - } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(u.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) - val updatedLabel = extractors.updateExtractorsLabel(label) + case Some(userEmail) => { + userservice.findByEmail(userEmail) match { + case Some(u) => { + val perms = List(new ResourceRef('user, u.id)) + Some(info.copy(unique_key=Some(ek), permissions=perms)) + } + case None => { + Logger.error("No user found with email "+userEmail) + None } } } - }) + } + } + case None => Some(info) + } + + // TODO: Check user permissions if the extractor_key has already been registered + + submissionInfo match { + case None => BadRequest("Extractors with a private key must also specify a non-anonymous user.") + case Some(subInfo) => { + extractors.updateExtractorInfo(subInfo) match { + case Some(u) => { + // Create/assign any default labels for this extractor + u.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") + } + extractors.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(u.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name)) + val updatedLabel = extractors.updateExtractorsLabel(label) + } + } + } + }) - Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id))) + } + case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) + } } - case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info")) } } ) @@ -518,11 +520,14 @@ class Extractions @Inject()( } // if extractor_id is not specified default to execution of all extractors matching mime type (request.body \ "extractor").asOpt[String] match { - case Some(extractorId) => + case Some(extractorId) => { + val extractorKey = (request.body \ "extractor").asOpt[String] + extractors.getExtractorInfo(extractorId, extractorKey, request.user) val job_id = routing.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra, datasetId, newFlags, request.apiKey, request.user) sink.logSubmitFileToExtractorEvent(file, extractorId, request.user) Ok(Json.obj("status" -> "OK", "job_id" -> job_id)) + } case None => { routing.fileCreated(file, None, Utils.baseUrl(request).toString, request.apiKey) match { case Some(job_id) => { diff --git a/app/api/Files.scala b/app/api/Files.scala index 18c0602b8..69d8a3d70 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -32,6 +32,7 @@ class Files @Inject()( collections: CollectionService, queries: MultimediaQueryService, tags: TagService, + sections_service: SectionService, comments: CommentService, extractions: ExtractionService, dtsrequests:ExtractionRequestsService, @@ -207,6 +208,18 @@ class Files @Inject()( } } + def sections(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request => + implicit val user = request.user + files.get(id) match { + case Some(file) => { + val sectionList = sections_service.findByFileId(id) + val sectionIds = sectionList.map { section => section.id } + Ok(toJson(sectionIds)) + } + case None => NotFound(toJson("The requested file does not exist")) + } + } + def getMetadataDefinitions(id: UUID, space: Option[String]) = PermissionAction(Permission.AddMetadata, Some(ResourceRef(ResourceRef.file, id))) { implicit request => implicit val user = request.user files.get(id) match { @@ -1697,6 +1710,9 @@ class Files @Inject()( current.plugin[VersusPlugin].foreach { _.removeFromIndexes(id) } + current.plugin[ElasticsearchPlugin].foreach { + _.delete(id.stringify) + } Logger.debug("Deleting file: " + file.filename) files.removeFile(id, Utils.baseUrl(request), request.apiKey, request.user) diff --git a/app/api/Metadata.scala b/app/api/Metadata.scala index 1c2be48fb..1dec8f676 100644 --- a/app/api/Metadata.scala +++ b/app/api/Metadata.scala @@ -257,7 +257,7 @@ class Metadata @Inject() ( // Given a list of terms, create a new standard vocabulary from the list // Expects a JSON array of Strings as the request body - def createVocabulary() = AuthenticatedAction(parse.json) { + def createVocabulary() = PermissionAction(Permission.CreateVocabulary)(parse.json) { implicit request => request.user match { case None => BadRequest(toJson("Invalid user")) @@ -278,7 +278,7 @@ class Metadata @Inject() ( // Given an ID, replace the entire terms list of a standard vocabulary // Expects a JSON array of Strings as the request body - def updateVocabulary(id: UUID) = AuthenticatedAction(parse.json) { + def updateVocabulary(id: UUID) = PermissionAction(Permission.EditVocabulary)(parse.json) { implicit request => request.user match { case None => BadRequest(toJson("Invalid user")) @@ -304,7 +304,7 @@ class Metadata @Inject() ( } // Given an ID, delete the standard vocabulary with that ID - def deleteVocabulary(id: UUID) = AuthenticatedAction(parse.empty) { + def deleteVocabulary(id: UUID) = PermissionAction(Permission.DeleteVocabulary)(parse.empty) { implicit request => request.user match { case None => BadRequest(toJson("Invalid user")) @@ -341,7 +341,7 @@ class Metadata @Inject() ( } } - def editDefinition(id: UUID, spaceId: Option[String]) = AuthenticatedAction(parse.json) { + def editDefinition(id: UUID, spaceId: Option[String]) = PermissionAction(Permission.EditVocabulary)(parse.json) { implicit request => request.user match { case Some(user) => { @@ -387,7 +387,7 @@ class Metadata @Inject() ( } } - def deleteDefinition(id: UUID) = AuthenticatedAction { implicit request => + def deleteDefinition(id: UUID) = PermissionAction(Permission.CreateVocabulary) { implicit request => implicit val user = request.user user match { case Some(user) => { diff --git a/app/api/Permissions.scala b/app/api/Permissions.scala index ac902a31d..e3c6ca5c3 100644 --- a/app/api/Permissions.scala +++ b/app/api/Permissions.scala @@ -426,6 +426,7 @@ object Permission extends Enumeration { def checkPermission(user: User, permission: Permission, resourceRef: ResourceRef): Boolean = { // check if user is owner, in that case they can do what they want. if (user.superAdminMode) return true + if (user.status == UserStatus.ReadOnly && !READONLY.contains(permission) && permission != Permission.DownloadFiles) return false if (checkOwner(users.findByIdentity(user), resourceRef)) return true resourceRef match { diff --git a/app/api/Sections.scala b/app/api/Sections.scala index 0b7abf8c9..0e15460f4 100644 --- a/app/api/Sections.scala +++ b/app/api/Sections.scala @@ -67,8 +67,14 @@ class Sections @Inject()( Logger.debug("Getting info for section with id " + id) sections.get(id) match { case Some(section) => - Ok(Json.obj("id" -> section.id.toString, "file_id" -> section.file_id.toString, - "startTime" -> section.startTime.getOrElse(-1).toString, "tags" -> Json.toJson(section.tags.map(_.name)))) + Ok(Json.obj( + "id" -> section.id.toString, + "file_id" -> section.file_id.toString, + "startTime" -> section.startTime.getOrElse(-1).toString, + "tags" -> Json.toJson(section.tags.map(_.name)), + "area" -> Json.toJson(section.area.getOrElse(null)), + "description" -> Json.toJson(section.description.getOrElse("")) + )) case None => Logger.error("Section not found " + id); NotFound(toJson("Section not found, id: " + id)) } } diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala index 8dd34edda..e48de645b 100644 --- a/app/api/Spaces.scala +++ b/app/api/Spaces.scala @@ -32,35 +32,39 @@ class Spaces @Inject()(spaces: SpaceService, val spaceTitle: String = Messages("space.title") //TODO- Minimal Space created with Name and description. URLs are not yet put in - def createSpace() = AuthenticatedAction(parse.json) { implicit request => + def createSpace() = PermissionAction(Permission.CreateSpace)(parse.json) { implicit request => Logger.debug("Creating new space") - val nameOpt = (request.body \ "name").asOpt[String] - val descOpt = (request.body \ "description").asOpt[String] - (nameOpt, descOpt) match { - case (Some(name), Some(description)) => { - // TODO: add creator - val userId = request.user.get.id - val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, - homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, - datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) - spaces.insert(c) match { - case Some(id) => { - appConfig.incrementCount('spaces, 1) - events.addObjectEvent(request.user, c.id, c.name, "create_space") - userService.findRoleByName("Admin") match { - case Some(realRole) => { - spaces.addUser(userId, realRole, UUID(id)) - } - case None => Logger.info("No admin role found") + if(request.user.get.status == UserStatus.ReadOnly) { + BadRequest(toJson("User is Read-Only")) + } else { + val nameOpt = (request.body \ "name").asOpt[String] + val descOpt = (request.body \ "description").asOpt[String] + (nameOpt, descOpt) match { + case (Some(name), Some(description)) => { + // TODO: add creator + val userId = request.user.get.id + val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, + homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, + datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) + spaces.insert(c) match { + case Some(id) => { + appConfig.incrementCount('spaces, 1) + events.addObjectEvent(request.user, c.id, c.name, "create_space") + userService.findRoleByName("Admin") match { + case Some(realRole) => { + spaces.addUser(userId, realRole, UUID(id)) + } + case None => Logger.info("No admin role found") + } + Ok(toJson(Map("id" -> id))) } - Ok(toJson(Map("id" -> id))) + case None => Ok(toJson(Map("status" -> "error"))) } - case None => Ok(toJson(Map("status" -> "error"))) - } + } + case (_, _) => BadRequest(toJson("Missing required parameters")) } - case (_, _) => BadRequest(toJson("Missing required parameters")) } } diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 344474f35..34cd2a0fa 100644 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -369,6 +369,7 @@ class Application @Inject()(files: FileService, collections: CollectionService, api.routes.javascript.Files.unfollow, api.routes.javascript.Files.getTechnicalMetadataJSON, api.routes.javascript.Files.filePreviewsList, + api.routes.javascript.Files.sections, api.routes.javascript.Files.updateMetadata, api.routes.javascript.Files.addMetadata, api.routes.javascript.Files.getMetadataDefinitions, @@ -382,7 +383,9 @@ class Application @Inject()(files: FileService, collections: CollectionService, api.routes.javascript.Search.search, api.routes.javascript.Search.searchJson, api.routes.javascript.Sections.add, + api.routes.javascript.Sections.get, api.routes.javascript.Sections.delete, + api.routes.javascript.Sections.description, api.routes.javascript.Sections.comment, api.routes.javascript.Sections.getTags, api.routes.javascript.Sections.addTags, diff --git a/app/controllers/Datasets.scala b/app/controllers/Datasets.scala index 4f639a4cd..4304d4be5 100644 --- a/app/controllers/Datasets.scala +++ b/app/controllers/Datasets.scala @@ -8,7 +8,8 @@ import play.api.Logger import play.api.Play.current import play.api.libs.json.Json._ import services._ -import util.{FileUtils, Formatters, RequiredFieldsConfig, SortingUtils } +import util.{FileUtils, Formatters, RequiredFieldsConfig, SortingUtils} + import scala.collection.immutable._ import scala.collection.mutable.ListBuffer import play.api.i18n.Messages @@ -728,7 +729,6 @@ class Datasets @Inject() ( implicit val user = request.user Logger.debug("------- in Datasets.submit ---------") - val folder = folderId.flatMap(id => folders.get(UUID(id))) val retMap = request.body.asFormUrlEncoded.get("datasetid").flatMap(_.headOption) match { case Some(ds) => { diff --git a/app/controllers/Extractors.scala b/app/controllers/Extractors.scala index b3e63e948..664fcbc3a 100644 --- a/app/controllers/Extractors.scala +++ b/app/controllers/Extractors.scala @@ -39,9 +39,9 @@ class Extractors @Inject() (extractions: ExtractionService, /** * Gets a map of all updates from all jobs given to this extractor. */ - def showJobHistory(extractorName: String) = AuthenticatedAction { implicit request => + def showJobHistory(extractorName: String, extractor_key: Option[String]) = AuthenticatedAction { implicit request => implicit val user = request.user - extractorService.getExtractorInfo(extractorName) match { + extractorService.getExtractorInfo(extractorName, extractor_key, user) match { case None => NotFound(s"No extractor found with name=${extractorName}") case Some(info) => { val allExtractions = extractions.findAll() @@ -56,9 +56,10 @@ class Extractors @Inject() (extractions: ExtractionService, */ def selectExtractors() = AuthenticatedAction { implicit request => implicit val user = request.user - + val userid = request.user.map(u => Some(u.id)).getOrElse(None) // Filter extractors by user filters necessary - var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty) + // TODO: Filter by multiple spaces + var runningExtractors: List[ExtractorInfo] = extractorService.listExtractorsInfo(List.empty, userid) val selectedExtractors: List[String] = extractorService.getEnabledExtractors() val groups = extractions.groupByType(extractions.findAll()) val allLabels = extractorService.listExtractorsLabels() @@ -166,7 +167,7 @@ class Extractors @Inject() (extractions: ExtractionService, def manageLabels = ServerAdminAction { implicit request => implicit val user = request.user val categories = List[String]("EXTRACT") - val extractors = extractorService.listExtractorsInfo(categories) + val extractors = extractorService.listExtractorsInfo(categories, None) val labels = extractorService.listExtractorsLabels() Ok(views.html.extractorLabels(labels, extractors)) @@ -211,7 +212,8 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorInfo(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => { val labels = extractorService.getLabelsForExtractor(extractor.name) @@ -223,6 +225,7 @@ class Extractors @Inject() (extractions: ExtractionService, def showExtractorMetrics(extractorName: String) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) val dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS") val todaydate = dateFormatter.format(new java.util.Date()) @@ -299,7 +302,7 @@ class Extractors @Inject() (extractions: ExtractionService, } Logger.warn("last 10 average: " + lastTenAverage) - val targetExtractor = extractorService.listExtractorsInfo(List.empty).find(p => p.name == extractorName) + val targetExtractor = extractorService.listExtractorsInfo(List.empty, userid).find(p => p.name == extractorName) targetExtractor match { case Some(extractor) => Ok(views.html.extractorMetrics(extractorName, average.toString, lastTenAverage.toString, lastweeksubmitted, lastmonthsubmitted)) case None => InternalServerError("Extractor Info not found: " + extractorName) @@ -308,11 +311,19 @@ class Extractors @Inject() (extractions: ExtractionService, def submitFileExtraction(file_id: UUID) = PermissionAction(Permission.EditFile, Some(ResourceRef(ResourceRef.file, file_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.file.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) fileService.get(file_id) match { - case Some(file) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + var extractors = all_extractors.filter(!_.process.file.isEmpty) + + val user_extra = userid match { + case Some(uid) => all_extractors.filter(_.permissions.contains(ResourceRef('user, uid))) + case None => List.empty + } + + extractors = (extractors ++ user_extra).distinct + val foldersContainingFile = folders.findByFileId(file.id).sortBy(_.name) var folderHierarchy = new ListBuffer[Folder]() if(foldersContainingFile.length > 0) { @@ -352,7 +363,8 @@ class Extractors @Inject() (extractions: ExtractionService, def submitSelectedExtractions(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) val extractors = all_extractors.filter(!_.process.file.isEmpty) datasets.get(ds_id) match { case Some(dataset) => { @@ -372,10 +384,13 @@ class Extractors @Inject() (extractions: ExtractionService, def submitDatasetExtraction(ds_id: UUID) = PermissionAction(Permission.EditDataset, Some(ResourceRef(ResourceRef.dataset, ds_id))) { implicit request => implicit val user = request.user - val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT")) - val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + val userid = request.user.map(u => Some(u.id)).getOrElse(None) datasetService.get(ds_id) match { - case Some(ds) => Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + case Some(ds) => { + val all_extractors = extractorService.listExtractorsInfo(List("EXTRACT", "CONVERT"), userid) + val extractors = all_extractors.filter(!_.process.dataset.isEmpty) + Ok(views.html.extractions.submitDatasetExtraction(extractors, ds)) + } case None => InternalServerError("Dataset not found") } } diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala index 0cbedeec3..ce4dae8c6 100644 --- a/app/controllers/SecuredController.scala +++ b/app/controllers/SecuredController.scala @@ -104,6 +104,9 @@ trait SecuredController extends Controller { userRequest.user match { case Some(u) if !AppConfiguration.acceptedTermsOfServices(u.termsOfServices) => Future.successful(Results.Redirect(routes.Application.tos(Some(request.uri)))) case Some(u) if (u.status==UserStatus.Inactive) => Future.successful(Results.Redirect(routes.Error.notActivated())) + case Some(u) if (u.status==UserStatus.ReadOnly && !api.Permission.READONLY.contains(permission) && permission != Permission.DownloadFiles) => { + Future.successful(Results.Redirect(routes.Error.notAuthorized("Account is ReadOnly", "", ""))) + } case Some(u) if u.superAdminMode || Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest) case Some(u) => notAuthorizedMessage(userRequest.user, resourceRef) case None if Permission.checkPermission(userRequest.user, permission, resourceRef) => block(userRequest) diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index a8388812f..1c197060a 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -83,10 +83,11 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS def selectExtractors(id: UUID) = AuthenticatedAction { implicit request => implicit val user = request.user + val userid = request.user.map(u => Some(u.id)).getOrElse(None) spaces.get(id) match { case Some(s) => { // get list of registered extractors - val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty) + val runningExtractors: List[ExtractorInfo] = extractors.listExtractorsInfo(List.empty, userid) // list of extractors enabled globally val globalSelections: List[String] = extractors.getEnabledExtractors() // get list of extractors registered with a specific space @@ -221,7 +222,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS } } - def newSpace() = AuthenticatedAction { implicit request => + def newSpace() = PermissionAction(Permission.CreateSpace) { implicit request => implicit val user = request.user Ok(views.html.spaces.newSpace(spaceForm)) } @@ -395,10 +396,10 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS * Submit action for new or edit space */ // TODO this should check to see if user has editspace for specific space - def submit() = AuthenticatedAction { implicit request => + def submit() = PermissionAction(Permission.CreateSpace) { implicit request => implicit val user = request.user user match { - case Some(identity) => { + case Some(identity) if identity.status != UserStatus.ReadOnly => { val userId = request.user.get.id //need to get the submitValue before binding form data, in case of errors we want to trigger different forms request.body.asMultipartFormData.get.dataParts.get("submitValue").headOption match { @@ -482,7 +483,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => { BadRequest("Did not get any submit button value.") } } } //some identity - case None => Redirect(routes.Spaces.list()).flashing("error" -> "You are not authorized to create/edit $spaceTitle.") + case _ => Redirect(routes.Spaces.list()).flashing("error" -> "You are not authorized to create/edit $spaceTitle.") } } def followingSpaces(index: Int, limit: Int, mode: String) = PrivateServerAction { implicit request => diff --git a/app/models/Extraction.scala b/app/models/Extraction.scala index e6b35b503..d38fe3b34 100644 --- a/app/models/Extraction.scala +++ b/app/models/Extraction.scala @@ -78,6 +78,7 @@ case class ExtractorDetail( * * @param id id internal to the system * @param name lower case, no spaces, can use dashes + * @param uniqueName name+suffix to uniquely identify extractor for private use e.g. clowder.extractor.v2.johndoe123 * @param version the version, for example 1.3.5 * @param updated date when this information was last updated * @param description short description of what the extractor does @@ -117,7 +118,9 @@ case class ExtractorInfo( defaultLabels: List[String] = List[String](), process: ExtractorProcessTriggers = new ExtractorProcessTriggers(), categories: List[String] = List[String](ExtractorCategory.EXTRACT.toString), - parameters: JsValue = JsObject(Seq()) + parameters: JsValue = JsObject(Seq()), + unique_key: Option[String] = None, + permissions: List[ResourceRef] =List[ResourceRef]() ) /** what are the categories of the extractor? @@ -170,7 +173,9 @@ object ExtractorInfo { (JsPath \ "labels").read[List[String]].orElse(Reads.pure(List.empty)) and (JsPath \ "process").read[ExtractorProcessTriggers].orElse(Reads.pure(new ExtractorProcessTriggers())) and (JsPath \ "categories").read[List[String]].orElse(Reads.pure(List[String](ExtractorCategory.EXTRACT.toString))) and - (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) + (JsPath \ "parameters").read[JsValue].orElse(Reads.pure(JsObject(Seq()))) and + (JsPath \ "unique_key").read[Option[String]].orElse(Reads.pure(None)) and + (JsPath \ "permissions").read[List[ResourceRef]].orElse(Reads.pure(List.empty)) )(ExtractorInfo.apply _) } diff --git a/app/models/Section.scala b/app/models/Section.scala index 59f40c85c..8bf5da309 100644 --- a/app/models/Section.scala +++ b/app/models/Section.scala @@ -33,3 +33,15 @@ case class Rectangle( h: Double) { override def toString() = f"x: $x%.2f, y: $y%.2f, width: $w%.2f, height: $h%.2f" } + +object Rectangle { + implicit object RectangleWrites extends Writes[Rectangle] { + def writes(rectangle: Rectangle): JsObject = { + Json.obj( + "x" -> rectangle.x, + "y" -> rectangle.y, + "w" -> rectangle.w, + "h" -> rectangle.h) + } + } +} diff --git a/app/models/User.scala b/app/models/User.scala index 5daae5c46..d07fe8020 100644 --- a/app/models/User.scala +++ b/app/models/User.scala @@ -13,7 +13,7 @@ import play.api.libs.json._ object UserStatus extends Enumeration { type UserStatus = Value - val Inactive, Active, Admin = Value + val Inactive, Active, Admin, ReadOnly = Value } /** diff --git a/app/services/CollectionService.scala b/app/services/CollectionService.scala index 5f3c3971e..3715921ee 100644 --- a/app/services/CollectionService.scala +++ b/app/services/CollectionService.scala @@ -252,4 +252,6 @@ trait CollectionService { def getMetrics(): Iterator[Collection] + def isInTrash(id: UUID): Boolean + } diff --git a/app/services/DatasetService.scala b/app/services/DatasetService.scala index 071a707bf..1949c3ae3 100644 --- a/app/services/DatasetService.scala +++ b/app/services/DatasetService.scala @@ -389,5 +389,7 @@ trait DatasetService { def getTrashedIds(): List[UUID] + def isInTrash(id: UUID): Boolean + def recursiveArchive(dataset: Dataset, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User]) } diff --git a/app/services/ElasticsearchPlugin.scala b/app/services/ElasticsearchPlugin.scala index 25b7c8d9d..5888c09f7 100644 --- a/app/services/ElasticsearchPlugin.scala +++ b/app/services/ElasticsearchPlugin.scala @@ -228,12 +228,18 @@ class ElasticsearchPlugin(application: Application) extends Plugin { // Check permissions for each resource results.foreach(resource => { resource.resourceType match { - case ResourceRef.file => if (Permission.checkPermission(user, Permission.ViewFile, resource)) - filesFound += resource.id - case ResourceRef.dataset => if (Permission.checkPermission(user, Permission.ViewDataset, resource)) - datasetsFound += resource.id - case ResourceRef.collection => if (Permission.checkPermission(user, Permission.ViewDataset, resource)) - collectionsFound += resource.id + case ResourceRef.file => { + if (Permission.checkPermission(user, Permission.ViewFile, resource) && !files.isInTrash(resource.id)) + filesFound += resource.id + } + case ResourceRef.dataset => { + if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !datasets.isInTrash(resource.id)) + datasetsFound += resource.id + } + case ResourceRef.collection => { + if (Permission.checkPermission(user, Permission.ViewDataset, resource) && !collections.isInTrash(resource.id)) + collectionsFound += resource.id + } case _ => {} } }) @@ -527,7 +533,7 @@ class ElasticsearchPlugin(application: Application) extends Plugin { case jv: JsArray => { builder.startArray(clean_k) jv.value.foreach(subv => { - builder.value(subv.toString.replace("\"","")) + builder.value(convertJsObjectToBuilder(builder, subv.asInstanceOf[JsObject])) }) builder.endArray() } @@ -598,11 +604,9 @@ class ElasticsearchPlugin(application: Application) extends Plugin { // Elasticsearch 2 does not allow periods in field names builder.startArray(k.toString.replace(".", "_")) v.value.foreach(jv => { - // Try to interpret numeric value from each String if possible - parseDouble(jv.toString) match { - case Some(d) => builder.value(d) - case None => builder.value(jv) - } + builder.startObject() + convertJsObjectToBuilder(builder, jv.asInstanceOf[JsObject]) + builder.endObject() }) builder.endArray() } diff --git a/app/services/ExtractorRoutingService.scala b/app/services/ExtractorRoutingService.scala index 545eb4194..4915bb35a 100644 --- a/app/services/ExtractorRoutingService.scala +++ b/app/services/ExtractorRoutingService.scala @@ -72,12 +72,12 @@ class ExtractorRoutingService { * @param resourceType the type of resource to check * @return filtered list of extractors */ - private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value): List[String] = { + private def getMatchingExtractors(extractorIds: List[String], operation: String, resourceType: ResourceType.Value, user: Option[User] = None): List[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) extractorIds.flatMap(exId => - extractorsService.getExtractorInfo(exId)).filter(exInfo => - resourceType match { + extractorsService.getExtractorInfo(exId, None, None)).filter(exInfo => { + val processMatch = resourceType match { case ResourceType.dataset => containsOperation(exInfo.process.dataset, operation) case ResourceType.file => @@ -87,7 +87,17 @@ class ExtractorRoutingService { case _ => false } - ).map(_.name) + val permissionMatch = exInfo.unique_key match { + case Some(key) => { + user match { + case None => false // User must be provided for a key-protected extractor + case Some(u) => exInfo.permissions.contains(new ResourceRef('user,u.id)) + } + } + case None => true + } + processMatch && permissionMatch + }).map(_.name) } /** @@ -96,15 +106,15 @@ class ExtractorRoutingService { * @param operation The dataset operation requested. * @return A list of extractors IDs. */ - private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value): (List[String], List[String]) = { + private def getSpaceExtractorsByOperation(dataset: Dataset, operation: String, resourceType: ResourceType.Value, user: Option[User] = None): (List[String], List[String]) = { val spacesService = DI.injector.getInstance(classOf[SpaceService]) var enabledExtractors = new ListBuffer[String]() var disabledExtractors = new ListBuffer[String]() dataset.spaces.map(space => { spacesService.getAllExtractors(space).foreach { extractors => - enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType)) - disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType)) + enabledExtractors.appendAll(getMatchingExtractors(extractors.enabled, operation, resourceType, user)) + disabledExtractors.appendAll(getMatchingExtractors(extractors.disabled, operation, resourceType, user)) } }) (enabledExtractors.toList, disabledExtractors.toList) @@ -145,7 +155,7 @@ class ExtractorRoutingService { * @param contentType the content type of the file in the case of a file * @return a set of unique rabbitmq queues */ - private def getQueues(dataset: Dataset, routingKey: String, contentType: String): Set[String] = { + private def getQueues(dataset: Dataset, routingKey: String, contentType: String, user: Option[User] = None): Set[String] = { val extractorsService = DI.injector.getInstance(classOf[ExtractorService]) // drop the first fragment from the routing key and replace characters to create operation id @@ -160,9 +170,9 @@ class ExtractorRoutingService { else return Set.empty[String] // get extractors enabled at the global level - val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType) + val globalExtractors = getMatchingExtractors(extractorsService.getEnabledExtractors(), operation, resourceType, user) // get extractors enabled/disabled at the space level - val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType) + val (enabledExtractors, disabledExtractors) = getSpaceExtractorsByOperation(dataset, operation, resourceType, user) // get queues based on RabbitMQ bindings (old method). val queuesFromBindings = getQueuesFromBindings(routingKey) // take the union of queues so that we publish to a specific queue only once @@ -229,6 +239,7 @@ class ExtractorRoutingService { var jobId: Option[UUID] = None dataset match { case Some(d) => { + // TODO: Check private extractor behavior getQueues(d, routingKey, file.contentType).foreach { queue => val source = Entity(ResourceRef(ResourceRef.file, file.id), Some(file.contentType), sourceExtra) diff --git a/app/services/ExtractorService.scala b/app/services/ExtractorService.scala index 75acf38e1..d750eaf58 100644 --- a/app/services/ExtractorService.scala +++ b/app/services/ExtractorService.scala @@ -35,13 +35,13 @@ trait ExtractorService { def dropAllExtractorStatusCollection() - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] - def deleteExtractor(extractorName: String) + def deleteExtractor(extractorName: String, extractorKey: Option[String]) def listExtractorsLabels(): List[ExtractorsLabel] diff --git a/app/services/FileService.scala b/app/services/FileService.scala index 9558e7daf..6d474379d 100644 --- a/app/services/FileService.scala +++ b/app/services/FileService.scala @@ -248,4 +248,6 @@ trait FileService { def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[File] + def isInTrash(id: UUID): Boolean + } diff --git a/app/services/KeycloakProvider.scala b/app/services/KeycloakProvider.scala new file mode 100644 index 000000000..dab72cda6 --- /dev/null +++ b/app/services/KeycloakProvider.scala @@ -0,0 +1,103 @@ +package services + +import play.api.libs.ws.WS +import play.api.{Application, Logger} +import play.api.libs.json.JsObject +import securesocial.core._ +import scala.collection.JavaConverters._ + + +/** + * A Keycloak OAuth2 Provider + */ +class KeycloakProvider(application: Application) extends OAuth2Provider(application) { + val Error = "error" + val Message = "message" + val Type = "type" + val Sub = "sub" + val Name = "name" + val GivenName = "given_name" + val FamilyName = "family_name" + // todo: picture wont work + val Picture = "picture" + val Email = "email" + val Groups = "groups" + + override def id = KeycloakProvider.Keycloak + + def fillProfile(user: SocialUser): SocialUser = { + val UserInfoApi = loadProperty("userinfoUrl").getOrElse(throwMissingPropertiesException()) + val accessToken = user.oAuth2Info.get.accessToken + val promise = WS.url(UserInfoApi.toString).withHeaders(("Authorization", "Bearer " + accessToken)).get() + + try { + val response = awaitResult(promise) + val me = response.json + Logger.debug("Got back from Keycloak : " + me.toString()) + (me \ Error).asOpt[JsObject] match { + case Some(error) => + val message = (error \ Message).as[String] + val errorType = ( error \ Type).as[String] + Logger.error("[securesocial] error retrieving profile information from Keycloak. Error type = %s, message = %s" + .format(errorType,message)) + throw new AuthenticationException() + case _ => + val userId = (me \ Sub).as[String] + val firstName = (me \ GivenName).asOpt[String] + val lastName = (me \ FamilyName).asOpt[String] + val fullName = (me \ Name).asOpt[String] + val avatarUrl = ( me \ Picture).asOpt[String] + val email = ( me \ Email).asOpt[String] + val groups = ( me \ Groups).asOpt[List[String]] + val roles = ( me \ "resource_access" \ "account" \ "roles").asOpt[List[String]] + (application.configuration.getList("securesocial.keycloak.groups"), groups) match { + case (Some(conf), Some(keycloak)) => { + val conflist = conf.unwrapped().asScala.toList + if (keycloak.intersect(conflist).isEmpty) { + throw new AuthenticationException() + } + } + case (Some(_), None) => throw new AuthenticationException() + case (None, _) => Logger.debug("[securesocial] No check needed for groups") + } + (application.configuration.getList("securesocial.keycloak.roles"), roles) match { + case (Some(conf), Some(keycloak)) => { + val conflist = conf.unwrapped().asScala.toList + if (keycloak.intersect(conflist).isEmpty) { + throw new AuthenticationException() + } + } + case (Some(_), None) => throw new AuthenticationException() + case (None, _) => Logger.debug("[securesocial] No check needed for roles") + } + user.copy( + identityId = IdentityId(userId, id), + firstName = firstName.getOrElse(""), + lastName = lastName.getOrElse(""), + fullName = fullName.getOrElse({ + if (firstName.isDefined && lastName.isDefined) { + firstName.get + " " + lastName.get + } else if (firstName.isDefined) { + firstName.get + } else if (lastName.isDefined) { + lastName.get + } else { + "" + } + }), + avatarUrl = avatarUrl, + email = email + ) + } + } catch { + case e: Exception => { + Logger.error( "[securesocial] error retrieving profile information from Keycloak", e) + throw new AuthenticationException() + } + } + } +} + +object KeycloakProvider { + val Keycloak = "keycloak" +} diff --git a/app/services/MessageService.scala b/app/services/MessageService.scala index 5eb95d557..c6eda512a 100644 --- a/app/services/MessageService.scala +++ b/app/services/MessageService.scala @@ -298,6 +298,7 @@ class EventFilter(channel: Channel, queue: String) extends Actor { * @param queue */ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { + val users: UserService = DI.injector.getInstance(classOf[UserService]) val extractions: ExtractionService = DI.injector.getInstance(classOf[ExtractionService]) val extractorsService: ExtractorService = DI.injector.getInstance(classOf[ExtractorService]) @@ -305,6 +306,7 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { case statusBody: String => Logger.debug("Received extractor heartbeat: " + statusBody) val json = Json.parse(statusBody) + Logger.debug(json.toString) // TODO store running extractors ids val id = UUID((json \ "id").as[String]) val queue = (json \ "queue").as[String] @@ -313,52 +315,78 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor { // Validate document val extractionInfoResult = extractor_info.validate[ExtractorInfo] + // Determine if there is a user associated with this request + val owner = (json \ "owner").as[String] + val user: Option[User] = if (owner.length > 0) { + users.findByEmail(owner) + } else { + None + } + // Update database extractionInfoResult.fold( - errors => { - Logger.debug("Received extractor heartbeat with bad format: " + extractor_info) - }, + errors => Logger.debug("Received extractor heartbeat with bad format: " + extractor_info), info => { - extractorsService.getExtractorInfo(info.name) match { - case Some(infoFromDB) => { - // TODO only update if new semantic version is greater than old semantic version - if (infoFromDB.version != info.version) { - // TODO keep older versions of extractor info instead of just the latest one - extractorsService.updateExtractorInfo(info) - Logger.info("Updated extractor definition for " + info.name) + if (info.unique_key.isDefined && user.isEmpty) { + Logger.error("Extractor keys must have a user associated with them.") + } else { + extractorsService.getExtractorInfo(info.name, info.unique_key, user) match { + case Some(infoFromDB) => { + if (info.unique_key.isDefined) { + // Retain existing permissions + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=infoFromDB.permissions) + case None => info + } + extractorsService.updateExtractorInfo(registrationInfo) + Logger.info(s"Updated private extractor definition for ${info.name} - ${info.unique_key}") + } else { + // TODO only update if new semantic version is greater than old semantic version + if (infoFromDB.version != info.version) { + // TODO keep older versions of extractor info instead of just the latest one + extractorsService.updateExtractorInfo(info) + Logger.info(s"Updated extractor definition for ${info.name}") + } + } } - } - case None => { - extractorsService.updateExtractorInfo(info) match { - case None => {} - case Some(eInfo) => { - // Create (if needed) and assign default labels - eInfo.defaultLabels.foreach(labelStr => { - val segments = labelStr.split("/") - val (labelName, labelCategory) = if (segments.length > 1) { - (segments(1), segments(0)) - } else { - (segments(0), "Other") - } - extractorsService.getExtractorsLabel(labelName) match { - case None => { - // Label does not exist - create and assign it - val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + case None => { + // Inject user into permissions list if a key is given + val registrationInfo = info.unique_key match { + case Some(ek) => info.copy(permissions=List(ResourceRef('user, user.get.id))) + case None => info + } + extractorsService.updateExtractorInfo(registrationInfo) match { + case None => {} + case Some(eInfo) => { + // Create (if needed) and assign default labels + eInfo.defaultLabels.foreach(labelStr => { + val segments = labelStr.split("/") + val (labelName, labelCategory) = if (segments.length > 1) { + (segments(1), segments(0)) + } else { + (segments(0), "Other") } - case Some(lbl) => { - // Label already exists, assign it - if (!lbl.extractors.contains(eInfo.name)) { - val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) - val updatedLabel = extractorsService.updateExtractorsLabel(label) + extractorsService.getExtractorsLabel(labelName) match { + case None => { + // Label does not exist - create and assign it + val createdLabel = extractorsService.createExtractorsLabel(labelName, Some(labelCategory), List[String](eInfo.name)) + } + case Some(lbl) => { + // Label already exists, assign it + if (!lbl.extractors.contains(eInfo.name)) { + val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](eInfo.name)) + val updatedLabel = extractorsService.updateExtractorsLabel(label) + } } } - } - }) + }) + } } - } - Logger.info(s"New extractor ${info.name} registered from heartbeat") + Logger.info(s"New extractor ${info.name} registered from heartbeat with key "+info.unique_key.toString) + } } + } } ) diff --git a/app/services/mongodb/MongoDBCollectionService.scala b/app/services/mongodb/MongoDBCollectionService.scala index 6c9c58b70..340e86a8b 100644 --- a/app/services/mongodb/MongoDBCollectionService.scala +++ b/app/services/mongodb/MongoDBCollectionService.scala @@ -1116,6 +1116,10 @@ class MongoDBCollectionService @Inject() ( Collection.find(MongoDBObject("trash" -> false)).toIterator } + def isInTrash(id: UUID): Boolean = { + Collection.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined + } + private def isSubCollectionIdInCollection(subCollectionId: UUID, collection: Collection) : Boolean = { if (collection.child_collection_ids.contains(subCollectionId)){ return true diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index 15de0db8d..a02677c23 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -729,12 +729,12 @@ class MongoDBDatasetService @Inject() ( if (file.isInstanceOf[models.File]) { val theFile = file.asInstanceOf[models.File] if (!theFile.thumbnail_id.isEmpty) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) return } } } - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } case None => Logger.debug(s"Dataset $datasetId not found") } @@ -749,12 +749,12 @@ class MongoDBDatasetService @Inject() ( if (file.isInstanceOf[File]) { val theFile = file.asInstanceOf[File] if (!theFile.thumbnail_id.isEmpty) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) return } } } - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } case None => Logger.debug("No dataset found with id " + datasetId) } @@ -929,13 +929,13 @@ class MongoDBDatasetService @Inject() ( val md = JSON.parse(json).asInstanceOf[DBObject] Dataset.dao.collection.findOne(MongoDBObject("_id" -> new ObjectId(id.stringify)), MongoDBObject("metadata" -> 1)) match { case None => { - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> md), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> md, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } case Some(x) => { x.getAs[DBObject]("metadata") match { case Some(map) => { val union = map.asInstanceOf[DBObject] ++ md - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> union), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("metadata" -> union, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } case None => Map.empty } @@ -947,23 +947,25 @@ class MongoDBDatasetService @Inject() ( Logger.debug("Adding XML metadata to dataset " + id + " from file " + fileId + ": " + json) val md = JsonUtil.parseJSON(json).asInstanceOf[java.util.LinkedHashMap[String, Any]].toMap Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $addToSet("datasetXmlMetadata" -> DatasetXMLMetadata.toDBObject(models.DatasetXMLMetadata(md, fileId.stringify))), false, false, WriteConcern.Safe) + $addToSet("datasetXmlMetadata" -> DatasetXMLMetadata.toDBObject(models.DatasetXMLMetadata(md, fileId.stringify))) + ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def removeXMLMetadata(id: UUID, fileId: UUID) { Logger.debug("Removing XML metadata belonging to file " + fileId + " from dataset " + id + ".") - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("datasetXmlMetadata" -> MongoDBObject("fileId" -> fileId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("datasetXmlMetadata" -> MongoDBObject("fileId" -> fileId.stringify)) + ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def addUserMetadata(id: UUID, json: String) { Logger.debug("Adding/modifying user metadata to dataset " + id + " : " + json) val md = com.mongodb.util.JSON.parse(json).asInstanceOf[DBObject] - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadata" -> md), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadata" -> md, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } /** Change the metadataCount field for a dataset */ def incrementMetadataCount(id: UUID, count: Long) = { - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $inc("metadataCount" -> count), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $inc("metadataCount" -> count) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } /** @@ -971,20 +973,20 @@ class MongoDBDatasetService @Inject() ( */ def updateInformation(id: UUID, description: String, name: String) { val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $set("description" -> description, "name" -> name), + $set("description" -> description, "name" -> name, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def updateName(id: UUID, name: String) { events.updateObjectName(id, name) val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $set("name" -> name), + $set("name" -> name, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def updateDescription(id: UUID, description: String){ val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $set("description" -> description), + $set("description" -> description, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } @@ -995,7 +997,7 @@ class MongoDBDatasetService @Inject() ( //Don't allow duplicates if (Dataset.dao.find(MongoDBObject("_id" -> new ObjectId(id.stringify)) ++ MongoDBObject("creators" -> creator)).length == 0) { val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $push("creators" -> creator), + $push("creators" -> creator) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } } @@ -1005,7 +1007,7 @@ class MongoDBDatasetService @Inject() ( */ def removeCreator(id: UUID, creator: String) { Dataset.dao.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $pull("creators" -> creator), false, false, WriteConcern.Safe) + $pull("creators" -> creator) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } /** @@ -1016,8 +1018,8 @@ class MongoDBDatasetService @Inject() ( if (Dataset.dao.find(MongoDBObject("_id" -> new ObjectId(id.stringify)) ++ MongoDBObject("creators" -> creator)).length != 0) { removeCreator(id, creator); Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $push("creators" -> MongoDBObject("$each" -> MongoDBList(creator), "$position" -> position)), - false, false, WriteConcern.Safe) + $push("creators" -> MongoDBObject("$each" -> MongoDBList(creator), "$position" -> position)) + ++ $set("lastModifiedDate" -> new Date()),false, false, WriteConcern.Safe) } } @@ -1032,7 +1034,7 @@ class MongoDBDatasetService @Inject() ( def updateLicense(id: UUID, licenseType: String, rightsHolder: String, licenseText: String, licenseUrl: String, allowDownload: String) { val licenseData = models.LicenseData(m_licenseType = licenseType, m_rightsHolder = rightsHolder, m_licenseText = licenseText, m_licenseUrl = licenseUrl, m_allowDownload = allowDownload.toBoolean) val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $set("licenseData" -> LicenseData.toDBObject(licenseData)), + $set("licenseData" -> LicenseData.toDBObject(licenseData), "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } @@ -1056,14 +1058,14 @@ class MongoDBDatasetService @Inject() ( if (!existingTags.contains(shortTag)) { val tagObj = models.Tag(name = shortTag, userId = userIdStr, extractor_id = eid, created = createdDate) tagsAdded += tagObj - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("tags" -> Tag.toDBObject(tagObj)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("tags" -> Tag.toDBObject(tagObj)) ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } }) tagsAdded.toList } def setUserMetadataWasModified(id: UUID, wasModified: Boolean) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadataWasModified" -> Some(wasModified)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadataWasModified" -> Some(wasModified), "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def findMetadataChangedDatasets(): List[Dataset] = { @@ -1083,7 +1085,8 @@ class MongoDBDatasetService @Inject() ( def removeTag(id: UUID, tagId: UUID) { Logger.debug("Removing tag " + tagId) - val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe) + val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))) + ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def removeTags(id: UUID, tags: List[String]) { @@ -1094,12 +1097,13 @@ class MongoDBDatasetService @Inject() ( // Only remove existing tags. tags.intersect(existingTags).map { tag => - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("name" -> tag)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("name" -> tag)) + ++ $set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } } def removeAllTags(id: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("tags" -> List()), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("tags" -> List(), "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } // ---------- Tags related code ends ------------------ @@ -1349,28 +1353,28 @@ class MongoDBDatasetService @Inject() ( } def addFile(datasetId: UUID, file: File) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("files" -> new ObjectId(file.id.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("files" -> new ObjectId(file.id.stringify)) ++$set("lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def addFolder(datasetId: UUID, folderId: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("folders" -> new ObjectId(folderId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("folders" -> new ObjectId(folderId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def addCollection(datasetId: UUID, collectionId: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("collections" -> new ObjectId(collectionId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $addToSet("collections" -> new ObjectId(collectionId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def removeCollection(datasetId: UUID, collectionId: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("collections" -> new ObjectId(collectionId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("collections" -> new ObjectId(collectionId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def removeFile(datasetId: UUID, fileId: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("files" -> new ObjectId(fileId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("files" -> new ObjectId(fileId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) removeXMLMetadata(datasetId, fileId) } def removeFolder(datasetId: UUID, folderId: UUID) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("folders" -> new ObjectId(folderId.stringify)), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $pull("folders" -> new ObjectId(folderId.stringify)) ++$set( "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } def newThumbnail(datasetId: UUID) { @@ -1381,12 +1385,12 @@ class MongoDBDatasetService @Inject() ( if (file.isInstanceOf[models.File]) { val theFile = file.asInstanceOf[models.File] if (!theFile.thumbnail_id.isEmpty) { - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> theFile.thumbnail_id.get, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) return } } } - Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None), false, false, WriteConcern.Safe) + Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), $set("thumbnail_id" -> None, "lastModifiedDate" -> new Date()), false, false, WriteConcern.Safe) } case None => } @@ -1452,11 +1456,11 @@ class MongoDBDatasetService @Inject() ( def addToSpace(datasetId: UUID, spaceId: UUID): Unit = { val result = Dataset.update( MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), - $addToSet("spaces" -> Some(new ObjectId(spaceId.stringify))), + $addToSet("spaces" -> Some(new ObjectId(spaceId.stringify))) ++$set( "lastModifiedDate" -> new Date()), false, false) if (get(datasetId).exists(_.isTRIAL == true) && spaces.get(spaceId).exists(_.isTrial == false)) { Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), - $set("status" -> DatasetStatus.DEFAULT.toString), + $set("status" -> DatasetStatus.DEFAULT.toString, "lastModifiedDate" -> new Date()), false, false) } } @@ -1464,7 +1468,7 @@ class MongoDBDatasetService @Inject() ( def removeFromSpace(datasetId: UUID, spaceId: UUID): Unit = { val result = Dataset.update( MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), - $pull("spaces" -> Some(new ObjectId(spaceId.stringify))), + $pull("spaces" -> Some(new ObjectId(spaceId.stringify))) ++$set("lastModifiedDate" -> new Date()), false, false) if (play.Play.application().configuration().getBoolean("verifySpaces")) { @@ -1472,7 +1476,7 @@ class MongoDBDatasetService @Inject() ( get(datasetId) match { case Some(d) if !d.spaces.map(s => spaces.get(s)).flatten.exists(_.isTrial == false) => Dataset.update(MongoDBObject("_id" -> new ObjectId(datasetId.stringify)), - $set("status" -> DatasetStatus.TRIAL.toString), + $set("status" -> DatasetStatus.TRIAL.toString, "lastModifiedDate" -> new Date()), false, false) case _ => } @@ -1646,13 +1650,13 @@ class MongoDBDatasetService @Inject() ( def incrementDownloads(id: UUID, user: Option[User]) = { Logger.debug("updating downloads for dataset "+id.toString) Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), - $inc("stats.downloads" -> 1) ++ $set("stats.last_downloaded" -> new Date), true, false, WriteConcern.Safe) + $inc("stats.downloads" -> 1) ++ $set("stats.last_downloaded" -> new Date, "lastModifiedDate" -> new Date()), true, false, WriteConcern.Safe) user match { case Some(u) => { Logger.debug("updating downloads for user "+u.toString) DatasetStats.update(MongoDBObject("user_id" -> new ObjectId(u.id.stringify), "resource_id" -> new ObjectId(id.stringify), "resource_type" -> "dataset"), - $inc("downloads" -> 1) ++ $set("last_downloaded" -> new Date), true, false, WriteConcern.Safe) + $inc("downloads" -> 1) ++ $set("last_downloaded" -> new Date, "lastModifiedDate" -> new Date()), true, false, WriteConcern.Safe) } case None => {} } @@ -1682,6 +1686,10 @@ class MongoDBDatasetService @Inject() ( trashedIds.toList } + def isInTrash(id: UUID): Boolean = { + Dataset.findOne(MongoDBObject("trash" -> true, "_id" -> new ObjectId(id.stringify))).isDefined + } + /** * Recursively submit requests to archive or unarchive the contents of the given dataset. * NOTE: "parameters" includes "operation", which supports both archiving and unarchiving diff --git a/app/services/mongodb/MongoDBExtractorService.scala b/app/services/mongodb/MongoDBExtractorService.scala index 039d7df06..3938e34bb 100644 --- a/app/services/mongodb/MongoDBExtractorService.scala +++ b/app/services/mongodb/MongoDBExtractorService.scala @@ -1,6 +1,6 @@ package services.mongodb -import javax.inject.Singleton +import javax.inject.{Inject, Singleton} import com.mongodb.casbah.Imports._ import com.mongodb.casbah.WriteConcern import com.mongodb.casbah.commons.MongoDBObject @@ -12,11 +12,12 @@ import play.api.Play.current import play.api.libs.json.{JsArray, JsNumber, JsObject, JsString, JsValue, Json} import services._ import services.mongodb.MongoContext.context - import org.bson.types.ObjectId @Singleton -class MongoDBExtractorService extends ExtractorService { +class MongoDBExtractorService @Inject() ( + users: MongoDBUserService + ) extends ExtractorService { def getExtractorServerIPList() = { var listServersIPs = List[String]() @@ -169,51 +170,106 @@ class MongoDBExtractorService extends ExtractorService { } } - def listExtractorsInfo(categories: List[String]): List[ExtractorInfo] = { + def listExtractorsInfo(categories: List[String], user: Option[UUID]): List[ExtractorInfo] = { + Logger.info("listing: "+categories.toString) var list_queue = List[ExtractorInfo]() val allDocs = ExtractorInfoDAO.findAll().sort(orderBy = MongoDBObject("name" -> -1)) for (doc <- allDocs) { - // If no categories are specified, return all extractor names - var category_match = categories.isEmpty - if (!category_match) { + // If no filters are specified, return all extractor names + var filter_match = (categories.isEmpty && doc.permissions.isEmpty) + if (!filter_match) { // Otherwise check if any extractor categories overlap requested categories (force uppercase) + val user_match = user match { + case Some(u) => { + val rr = new ResourceRef('user, u) + doc.permissions.contains(rr) || doc.permissions.isEmpty + } + case None => doc.permissions.isEmpty // If no user filter in registered extractor, everyone can see + } val upper_categories = categories.map(cat => cat.toUpperCase) - category_match = doc.categories.intersect(upper_categories).length > 0 + val category_match = categories.length == 0 || doc.categories.intersect(upper_categories).length > 0 + filter_match = (category_match && user_match) } - if (category_match) + if (filter_match) list_queue = doc :: list_queue } list_queue } - def getExtractorInfo(extractorName: String): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) + def getExtractorInfo(extractorName: String, extractorKey: Option[String], user: Option[User]): Option[ExtractorInfo] = { + extractorKey match { + case Some(ek) => { + user match { + case None => { + Logger.error("User authentication required to view extractor info with a unique key.") + None + } + case Some(u) => { + val userRef = new ResourceRef('user, u.id) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek, "permissions" -> userRef)) + } + } + } + case None => ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) + } } def updateExtractorInfo(e: ExtractorInfo): Option[ExtractorInfo] = { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name)) match { - case Some(old) => { - val updated = e.copy(id = old.id) - ExtractorInfoDAO.update(MongoDBObject("name" -> e.name), updated, false, false, WriteConcern.Safe) - Some(updated) - } + // TODO: Make this account for version as well + e.unique_key match { case None => { - ExtractorInfoDAO.save(e) - Some(e) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false))) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> MongoDBObject("$exists" -> false)), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } + } + case Some(ek) => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> e.name, "unique_key" -> ek)) match { + case Some(old) => { + val updated = e.copy(id = old.id) + ExtractorInfoDAO.update(MongoDBObject("name" -> e.name, "unique_key" -> ek), updated, false, false, WriteConcern.Safe) + Some(updated) + } + case None => { + ExtractorInfoDAO.save(e) + Some(e) + } + } } } } - def deleteExtractor(extractorName: String) { - ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName)) match { - case Some(extractor) => { - ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name)) + def deleteExtractor(extractorName: String, extractorKey: Option[String]) { + extractorKey match { + case Some(ek) => { + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> ek)) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> ek)) + } + case None => { + Logger.error(s"No extractor found with name ${extractorName} and key ${ek}") + } + } } case None => { - Logger.info("No extractor found with name: " + extractorName) + ExtractorInfoDAO.findOne(MongoDBObject("name" -> extractorName, "unique_key" -> MongoDBObject("$exists" -> false))) match { + case Some(extractor) => { + ExtractorInfoDAO.remove(MongoDBObject("name" -> extractor.name, "unique_key" -> MongoDBObject("$exists" -> false))) + } + case None => { + Logger.error("No extractor found with name: " + extractorName) + } + } } } } @@ -246,15 +302,11 @@ class MongoDBExtractorService extends ExtractorService { def getLabelsForExtractor(extractorName: String): List[ExtractorsLabel] = { var results = List[ExtractorsLabel]() - ExtractorInfoDAO.findOne(MongoDBObject("name"->extractorName)) match { - case Some(info) => { - ExtractorsLabelDAO.findAll().foreach(label => { - if (label.extractors.contains(extractorName)) { - results = results ++ List[ExtractorsLabel](label) - } - }) + ExtractorsLabelDAO.findAll().foreach(label => { + if (label.extractors.contains(extractorName) && !results.contains(label)) { + results = results ++ List[ExtractorsLabel](label) } - } + }) results } } diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index 52daf75ed..6d2766cf4 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -1237,6 +1237,15 @@ class MongoDBFileService @Inject() ( until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t))) FileDAO.find(query) } + + def isInTrash(id: UUID): Boolean = { + var foundTrash = false + datasets.findByFileIdAllContain(id).foreach(ds => { + if (ds.trash) + foundTrash = true + }) + foundTrash + } } object FileDAO extends ModelCompanion[File, ObjectId] { diff --git a/app/views/admin/users.scala.html b/app/views/admin/users.scala.html index a5b0ed626..046a47c1a 100644 --- a/app/views/admin/users.scala.html +++ b/app/views/admin/users.scala.html @@ -18,32 +18,32 @@ never } } - @if(user.fold("")(_.id.stringify) == u.id.stringify) { - @if(!(u.status==UserStatus.Inactive)) { - + @if(user.fold("")(_.id.stringify) == u.id.stringify || configAdmins.contains(u.email.getOrElse(""))) { + + } + @if(u.status==UserStatus.Inactive) { + } else { - + } - } else { - @if(!(u.status==UserStatus.Inactive)) { - + @if(u.status==UserStatus.Active) { + } else { - + } - } - @if(user.fold("")(_.id.stringify) == u.id.stringify) { - - } else { - @if(configAdmins.contains(u.email.getOrElse(""))) { - + @if(u.status==UserStatus.ReadOnly) { + } else { - @if(u.status==UserStatus.Admin) { - - } else { - - } + } - } + @if(u.status==UserStatus.Admin) { + + } else { + + } + } @@ -66,12 +66,11 @@

- + - - + @@ -95,12 +94,11 @@

FullnameFullname Email Provider Last LoginActiveAdminStatus
- + - - + @@ -124,12 +122,11 @@

FullnameFullname Email Provider Last LoginActiveAdminStatus
- + - - + @@ -140,6 +137,34 @@

+
+ +
+
+

FullnameFullname Email Provider Last LoginActiveAdminStatus
+ + + + + + + + + + + @users.filter(u => u.status==UserStatus.ReadOnly).map(printRow) + +
FullnameEmailProviderLast LoginStatus
+ + + + @@ -148,45 +173,17 @@


') + + + $(useTab).append("") + } + +}(jQuery, Configuration)); + +/** + * Three.js + */ + +let camera, scene, renderer, stats; +let mixer; +let previewerWidth = 640; +let previewerHeight = 480; +const clock = new THREE.Clock(); + +function init(urlAddress) { + const container = document.getElementById(Configuration.tab.replace("#", "")); + + + /** + * Renderer + */ + + renderer = new THREE.WebGLRenderer( + { + antialias: false + } + ); + + //renderer.physicallyCorrectLights = true; + renderer.outputEncoding = THREE.sRGBEncoding; + renderer.setClearColor(0xcccccc); + renderer.setPixelRatio(window.devicePixelRatio); + renderer.setSize(previewerWidth, previewerHeight); + renderer.shadowMap.enabled = true; + + /** + * Camera + */ + + camera = new THREE.PerspectiveCamera(55, previewerWidth / previewerHeight, 0.1, 1000); + //camera.position.set(0, 20, 100); + + /** + * Scene + */ + + scene = new THREE.Scene(); + scene.background = new THREE.Color("White"); + //scene.fog = new THREE.Fog( 0xa0a0a0, 200, 1000 ); + + /** + * Lights + */ + + const hemiLight1 = new THREE.HemisphereLight(0xffffff, 0x444444); + const hemiLight2 = new THREE.HemisphereLight(0xffffff, 1); + //hemiLight.position.set(0, -20, 0); + scene.add(hemiLight1, hemiLight2); + + const ambientLight = new THREE.AmbientLight(0xffffff, 1); + //ambientLight.position.set(0, 200, 0); + scene.add(ambientLight); + + const dirLight = new THREE.DirectionalLight(0xffffff, 0.8); + //dirLight.position.set(0, 4, 4); + + // const helper = new THREE.DirectionalLightHelper(dirLight, 2); + // scene.add(helper); + dirLight.castShadow = true; + // dirLight.shadow.camera.top = 180; + // dirLight.shadow.camera.bottom = -100; + // dirLight.shadow.camera.left = -120; + // dirLight.shadow.camera.right = 120; + scene.add(dirLight); + + /** + * Orbit controls + */ + + const controls = new THREE.OrbitControls(camera, renderer.domElement); + //controls.target.set(0, 100, 0); + controls.update(); + container.appendChild(renderer.domElement); + + // scene.add( new THREE.CameraHelper( dirLight.shadow.camera ) ); + + // ground + // const mesh = new THREE.Mesh( + // new THREE.PlaneGeometry(1000, 1000), + // new THREE.MeshPhongMaterial({ color: 0x999999, depthWrite: false }) + // ); + // mesh.rotation.x = - Math.PI / 2; + // mesh.receiveShadow = true; + // mesh.material.transparent = false; + // scene.add(mesh); + + // const grid = new THREE.GridHelper(1000, 100, 0x000000, 0x000000); + // grid.material.opacity = 0.3; + // grid.material.transparent = true; + // scene.add(grid); + + /** + * Loading manager + */ + + // The loading manager is needed for the loading screen + + const loadingManager = new THREE.LoadingManager(() => { + const loadingScreen = document.getElementById('loading-screen'); + loadingScreen.classList.add('fade-out'); + loadingScreen.addEventListener('transitionend', onTransitionEnd); + }); + + loader = new THREE.FBXLoader(loadingManager); + + // Set loader's path to the url address of the file + + loader.setPath(urlAddress); + + //loader.setPath("/assets/javascripts/previewers/three_js/models/chimpanzee/source/"); + + loader.load('', function (object) { + + if (fileNameExtension === 'gltf' || fileNameExtension === 'glb') { + object = object.scene; + } + + object.traverse(function (child) { + + if (child.isMesh) { + + child.castShadow = true; + child.receiveShadow = true; + + } + + }); + + if (object.animations[0]) { + mixer = new THREE.AnimationMixer(object); + + const action = mixer.clipAction(object.animations[0]); + action.play(); + } + + const box = new THREE.Box3().setFromObject(object); + const size = box.getSize(new THREE.Vector3()).length(); + const center = box.getCenter(new THREE.Vector3()); + + //controls.reset(); + + object.position.x += (object.position.x - center.x); + object.position.y += (object.position.y - center.y); + object.position.z += (object.position.z - center.z); + + controls.maxDistance = size * 10; + controls.minDistance = size; + + camera.near = size / 100; + camera.far = size * 100; + camera.updateProjectionMatrix(); + camera.position.copy(center); + camera.position.x += size / 2.0; + camera.position.y += size / 5.0; + camera.position.z += size / 2.0; + camera.lookAt(center); + controls.update(); + + scene.add(object); + }); + + window.addEventListener('resize', onWindowResize); + + // stats + stats = new Stats(); + //container.appendChild( stats.dom ); +} + +function onWindowResize() { + + camera.aspect = window.innerWidth / window.innerHeight; + camera.updateProjectionMatrix(); + + renderer.setSize(window.innerWidth, window.innerHeight); + +} + +function onTransitionEnd(event) { + + event.target.remove(); + +} + +function animate() { + requestAnimationFrame(animate); + + const delta = clock.getDelta(); + + if (mixer) mixer.update(delta); + + renderer.render(scene, camera); + + stats.update(); +} \ No newline at end of file diff --git a/public/securesocial/images/providers/keycloak.png b/public/securesocial/images/providers/keycloak.png new file mode 100644 index 000000000..7387557f7 Binary files /dev/null and b/public/securesocial/images/providers/keycloak.png differ diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css index 774cd1737..906bc9a46 100644 --- a/public/stylesheets/main.css +++ b/public/stylesheets/main.css @@ -760,11 +760,15 @@ table#sensorsListTable>thead>tr>th.headerSortDown { } #scrollTableGalleria .galleriaDescr { - width: 35%; + width: 30%; } #scrollTableGalleria .galleriaFindSimilar { - width: 20%; + width: 15%; +} + +#scrollTableGalleria .galleriaDelete { + width: 10%; } #scrollTableGalleria thead { diff --git a/public/swagger.yml b/public/swagger.yml index b76ba7c2a..5e221cda4 100644 --- a/public/swagger.yml +++ b/public/swagger.yml @@ -9,7 +9,7 @@ info: Clowder is a customizable and scalable data management system to support any data format and multiple research domains. It is under active development and deployed for a variety of research projects. - version: 1.21.0 + version: 1.22.0 termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos contact: name: Clowder @@ -351,6 +351,32 @@ paths: 404: $ref: '#/components/responses/NotFound' + /files/{id}/sections: + get: + tags: + - files + summary: Get the sections of a file + description: | + Get the sections of a file. + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + 200: + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/UUID' + 404: + $ref: '#/components/responses/NotFound' + + /files/{id}/paths: get: tags: diff --git a/scripts/monitor/requirements.txt b/scripts/monitor/requirements.txt index 57b914834..c1e1847e3 100644 --- a/scripts/monitor/requirements.txt +++ b/scripts/monitor/requirements.txt @@ -1,3 +1,3 @@ pika==1.0.0 -requests==2.21.0 +requests==2.31.0 python-dateutil==2.8.0 diff --git a/scripts/rmq-error-shovel/requirements.txt b/scripts/rmq-error-shovel/requirements.txt index a7e215d09..ea4e61e6b 100644 --- a/scripts/rmq-error-shovel/requirements.txt +++ b/scripts/rmq-error-shovel/requirements.txt @@ -1,2 +1,2 @@ pika==1.0.0 -requests==2.21.0 \ No newline at end of file +requests==2.31.0 \ No newline at end of file diff --git a/version.sh b/version.sh index dbdcfeb8e..7cbc0cf95 100755 --- a/version.sh +++ b/version.sh @@ -1,8 +1,9 @@ #!/bin/bash -VERSION="1.19.2" +VERSION="1.22.0" sed -i~ "s#^ val version = \".*\"\$# val version = \"${VERSION}\"#" project/Build.scala +sed -i~ "s#^version: .*\$#version: \"${VERSION}\"#" citation.cff sed -i~ "s#^ version: .*\$# version: ${VERSION}#" public/swagger.yml sed -i~ "s#^release = '.*'\$#release = '${VERSION}'#" doc/src/sphinx/conf.py sed -i~ "s/^##.*unreleased.*$/## ${VERSION} - $(date +'%Y-%m-%d')/i" CHANGELOG.md