From 8910b1bafa5231732f4e16491a32b66da08dad13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 22 Jul 2024 19:00:44 +0200 Subject: [PATCH 01/22] WIP add zarr3 streaming - Added mag/zarr.json & mag/coords route for datasets (not annotations) --- .../Zarr3StreamingController.scala | 523 ++++++++++++++++++ .../zarr/ZarrCoordinatesParser.scala | 12 + ....scalableminds.webknossos.datastore.routes | 7 +- 3 files changed, 541 insertions(+), 1 deletion(-) create mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala new file mode 100644 index 00000000000..2b0a77fc100 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala @@ -0,0 +1,523 @@ +package com.scalableminds.webknossos.datastore.controllers + +import com.google.inject.Inject +import com.scalableminds.util.geometry.Vec3Int +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.dataformats.MagLocator +import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} +import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser +import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, BloscCompressor, StringCompressionSetting} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BloscCodecConfiguration, + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader +} +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.datastore.models.datasource._ +import com.scalableminds.webknossos.datastore.models.requests.{ + Cuboid, + DataServiceDataRequest, + DataServiceRequestSettings +} +import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} +import com.scalableminds.webknossos.datastore.services._ +import net.liftweb.common.Box.tryo +import play.api.i18n.{Messages, MessagesProvider} +import play.api.libs.json.{JsObject, JsValue, Json} +import play.api.mvc._ + +import scala.concurrent.ExecutionContext + +class Zarr3StreamingController @Inject()( + dataSourceRepository: DataSourceRepository, + accessTokenService: DataStoreAccessTokenService, + binaryDataServiceHolder: BinaryDataServiceHolder, + remoteWebknossosClient: DSRemoteWebknossosClient, + remoteTracingstoreClient: DSRemoteTracingstoreClient, +)(implicit ec: ExecutionContext) + extends Controller { + + override def defaultErrorCode: Int = NOT_FOUND + + val binaryDataService: BinaryDataService = binaryDataServiceHolder.binaryDataService + + override def allowRemoteOrigin: Boolean = true + + /** + * Serve .zattrs file for a dataset + * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) + */ + def requestZAttrs( + token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String = "", + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, + datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + omeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) + } yield Ok(Json.toJson(omeNgffHeader)) + } + } + + def zAttrsWithAnnotationPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String = ""): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) + omeNgffHeader <- annotationLayer match { + case Some(layer) => + remoteTracingstoreClient.getOmeNgffHeader(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + case None => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( + annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions) + } yield dataSourceOmeNgffHeader + } + } yield Ok(Json.toJson(omeNgffHeader)) + } + + /** + * Zarr-specific datasource-properties.json file for a datasource. + * Note that the result here is not necessarily equal to the file used in the underlying storage. + */ + def requestDataSource( + token: Option[String], + organizationName: String, + datasetName: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + for { + dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ~> NOT_FOUND + dataLayers = dataSource.dataLayers + zarrLayers = dataLayers.map(convertLayerToZarrLayer) + zarrSource = GenericDataSource[DataLayer](dataSource.id, zarrLayers, dataSource.scale) + zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) + } yield Ok(Json.toJson(zarrSourceJson)) + } + } + + private def replaceVoxelSizeByLegacyFormat(jsValue: JsValue): Fox[JsValue] = { + val jsObject = jsValue.as[JsObject] + val voxelSizeOpt = (jsObject \ "scale").asOpt[VoxelSize] + voxelSizeOpt match { + case None => Fox.successful(jsObject) + case Some(voxelSize) => + val inNanometer = voxelSize.toNanometer + for { + newDataSource <- tryo(jsObject - "scale" + ("scale" -> Json.toJson(inNanometer))) + } yield newDataSource + } + } + + private def convertLayerToZarrLayer(layer: DataLayer): ZarrLayer = + layer match { + case s: SegmentationLayer => + ZarrSegmentationLayer( + s.name, + s.boundingBox, + s.elementClass, + s.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), + mappings = s.mappings, + largestSegmentId = s.largestSegmentId, + numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1) + ) + case d: DataLayer => + ZarrDataLayer( + d.name, + d.category, + d.boundingBox, + d.elementClass, + d.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), + numChannels = Some(if (d.elementClass == ElementClass.uint24) 3 else 1), + additionalAxes = None + ) + } + + def dataSourceWithAnnotationPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + volumeAnnotationLayers = annotationSource.annotationLayers.filter(_.typ == AnnotationLayerType.Volume) + dataSource <- dataSourceRepository + .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationName)) + .toFox ~> NOT_FOUND + dataSourceLayers = dataSource.dataLayers + .filter(dL => !volumeAnnotationLayers.exists(_.name == dL.name)) + .map(convertLayerToZarrLayer) + annotationLayers <- Fox.serialCombined(volumeAnnotationLayers)( + l => + remoteTracingstoreClient + .getVolumeLayerAsZarrLayer(l.tracingId, Some(l.name), annotationSource.tracingStoreUrl, relevantToken)) + allLayer = dataSourceLayers ++ annotationLayers + zarrSource = GenericDataSource[DataLayer](dataSource.id, allLayer, dataSource.scale) + zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) + } yield Ok(Json.toJson(zarrSourceJson)) + } + + def requestRawZarr3Cube( + token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String, + coordinates: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + rawZarr3Cube(organizationName, datasetName, dataLayerName, mag, coordinates) + } + } + + def rawZarrCubePrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String, + coordinates: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + layer = annotationSource.getAnnotationLayer(dataLayerName) + + // ensures access to volume layers if fallback layer with equal name exists + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getRawZarrCube(annotationLayer.tracingId, + mag, + coordinates, + annotationSource.tracingStoreUrl, + relevantToken) + .map(Ok(_)) + case None => + rawZarr3Cube(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + mag, + coordinates) + } + } yield result + } + + private def rawZarr3Cube( + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String, + coordinates: String, + )(implicit m: MessagesProvider): Fox[Result] = + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, + datasetName, + dataLayerName) ~> NOT_FOUND + // (c, x, y, z) + parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + _ <- bool2Fox(parsedCoordinates.head == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND + (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), + parsedCoordinates(parsedCoordinates.length - 2), + parsedCoordinates(parsedCoordinates.length - 1)) + additionalCoordinates = Some( + parsedCoordinates + .slice(1, parsedCoordinates.length - 3) + .zipWithIndex + .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) + .toList) + cubeSize = DataLayer.bucketLength + request = DataServiceDataRequest( + dataSource, + dataLayer, + Cuboid( + topLeft = VoxelPosition(x * cubeSize * magParsed.x, + y * cubeSize * magParsed.y, + z * cubeSize * magParsed.z, + magParsed), + width = cubeSize, + height = cubeSize, + depth = cubeSize + ), + DataServiceRequestSettings(halfByte = false, additionalCoordinates = additionalCoordinates) + ) + (data, notFoundIndices) <- binaryDataService.handleDataRequests(List(request)) + _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND + } yield Ok(data) + + def requestZarrJsonForMag(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + zarrJsonForMag(organizationName, datasetName, dataLayerName, mag) + } + } + + private def zarrJsonForMag(organizationName: String, datasetName: String, dataLayerName: String, mag: String)( + implicit m: MessagesProvider): Fox[Result] = + for { + (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + additionalAxes = dataLayer.additionalAxes.getOrElse(Seq.empty) + zarrHeader = Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, + data_type = Left(dataLayer.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + ) + } yield Ok(Json.toJson(zarrHeader)) + + def zArrayPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient + .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) + layer = annotationSource.getAnnotationLayer(dataLayerName) + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))) + case None => + zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + } + } yield result + } + + def requestDataLayerMagFolderContents(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag) + } + } + + private def dataLayerMagFolderContents(organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String)(implicit m: MessagesProvider): Fox[Result] = + for { + (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Datastore", + "%s/%s/%s/%s".format(organizationName, datasetName, dataLayerName, mag), + List(".zarray") + )).withHeaders() + + def dataLayerMagFolderContentsPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + layer = annotationSource.getAnnotationLayer(dataLayerName) + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getDataLayerMagFolderContents(annotationLayer.tracingId, + mag, + annotationSource.tracingStoreUrl, + relevantToken) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Combined Annotation Route", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()) + case None => + dataLayerMagFolderContents(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + mag) + } + } yield result + } + + def requestDataLayerFolderContents(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + dataLayerFolderContents(organizationName, datasetName, dataLayerName) + } + } + + private def dataLayerFolderContents(organizationName: String, datasetName: String, dataLayerName: String)( + implicit m: MessagesProvider): Fox[Result] = + for { + (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + mags = dataLayer.resolutions + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Datastore", + "%s/%s/%s".format(organizationName, datasetName, dataLayerName), + List(".zattrs", ".zgroup") ++ mags.map(_.toMagLiteral(allowScalar = true)) + )).withHeaders() + + def dataLayerFolderContentsPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + layer = annotationSource.getAnnotationLayer(dataLayerName) + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Tracingstore", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()) + case None => + dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) + } + } yield result + } + + def requestDataSourceFolderContents(token: Option[String], + organizationName: String, + datasetName: String): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + for { + dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name) + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Datastore", + s"$organizationName/$datasetName", + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + )) + } + } + + def dataSourceFolderContentsPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) + dataSource <- dataSourceRepository + .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationName)) + .toFox ?~> Messages("dataSource.notFound") ~> NOT_FOUND + annotationLayerNames = annotationSource.annotationLayers.filter(_.typ == AnnotationLayerType.Volume).map(_.name) + dataSourceLayerNames = dataSource.dataLayers + .map((dataLayer: DataLayer) => dataLayer.name) + .filter(!annotationLayerNames.contains(_)) + layerNames = annotationLayerNames ++ dataSourceLayerNames + } yield + Ok( + views.html.datastoreZarrDatasourceDir( + "Combined datastore and tracingstore directory", + s"$accessToken", + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + )) + } + + def requestZGroup(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccessForSyncBlock( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + Ok(zGroupJson) + } + } + + private def zGroupJson: JsValue = Json.toJson(NgffGroupHeader(zarr_format = 2)) + + def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + layer = annotationSource.getAnnotationLayer(dataLayerName) + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(Ok(_)) + case None => + Fox.successful(Ok(zGroupJson)) + } + } yield result + } +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index b17d5ed1ad0..94e2c8b1f29 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -12,4 +12,16 @@ object ZarrCoordinatesParser { case _ => None } } + + def parseNDimensionalDotCoordinates( + coordinates: String, + ): Option[Array[Int]] = { + val ndCoordinatesRx = "\\s*([0-9]+).([0-9]+).([0-9]+)(.([0-9]+))+\\s*".r + + coordinates match { + case ndCoordinatesRx(coordString) => + Some(coordString.split('.').map(coord => Integer.parseInt(coord))) + case _ => None + } + } } diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index e796fc871d3..8945fe87a12 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -15,7 +15,7 @@ GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/his # Knossos compatible routes GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) -# Zarr compatible routes +# Zarr2 compatible routes GET /zarr/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) GET /zarr/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) GET /zarr/:organizationName/:datasetName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationName: String, datasetName: String, dataLayerName="") @@ -42,6 +42,11 @@ GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, cxyz: String) +# Zarr3 compatible routes +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestRawZarr3Cube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) + + # Segmentation mappings GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mappingName: String) GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) From acab8c0ddfeb80fa1f4d623246f27fdb2d7e84c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 24 Jul 2024 11:21:47 +0200 Subject: [PATCH 02/22] manage minimal zarr3 support for viewing a dataset and viewing an annotation --- .../Zarr3StreamingController.scala | 32 ++--- .../zarr/ZarrCoordinatesParser.scala | 11 +- .../services/DSRemoteTracingstoreClient.scala | 15 +++ ....scalableminds.webknossos.datastore.routes | 3 + ...VolumeTracingZarrStreamingController.scala | 116 +++++++++++++++++- ...alableminds.webknossos.tracingstore.routes | 3 + 6 files changed, 153 insertions(+), 27 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala index 2b0a77fc100..d5eb8c151d1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala @@ -193,11 +193,11 @@ class Zarr3StreamingController @Inject()( } } - def rawZarrCubePrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String, - coordinates: String): Action[AnyContent] = + def rawZarr3CubePrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String, + coordinates: String): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -209,11 +209,11 @@ class Zarr3StreamingController @Inject()( result <- layer match { case Some(annotationLayer) => remoteTracingstoreClient - .getRawZarrCube(annotationLayer.tracingId, - mag, - coordinates, - annotationSource.tracingStoreUrl, - relevantToken) + .getRawZarr3Cube(annotationLayer.tracingId, + mag, + coordinates, + annotationSource.tracingStoreUrl, + relevantToken) .map(Ok(_)) case None => rawZarr3Cube(annotationSource.organizationName, @@ -237,7 +237,7 @@ class Zarr3StreamingController @Inject()( datasetName, dataLayerName) ~> NOT_FOUND // (c, x, y, z) - parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND + parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND // TODO: change error message magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND _ <- bool2Fox(parsedCoordinates.head == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND @@ -316,10 +316,10 @@ class Zarr3StreamingController @Inject()( ) } yield Ok(Json.toJson(zarrHeader)) - def zArrayPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = Action.async { implicit request => + def zArray3PrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -328,7 +328,7 @@ class Zarr3StreamingController @Inject()( result <- layer match { case Some(annotationLayer) => remoteTracingstoreClient - .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) .map(z => Ok(Json.toJson(z))) case None => zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 94e2c8b1f29..43b9b77ec26 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -14,14 +14,9 @@ object ZarrCoordinatesParser { } def parseNDimensionalDotCoordinates( - coordinates: String, - ): Option[Array[Int]] = { + coordinates: String, + ): Option[Array[Int]] = { val ndCoordinatesRx = "\\s*([0-9]+).([0-9]+).([0-9]+)(.([0-9]+))+\\s*".r - - coordinates match { - case ndCoordinatesRx(coordString) => - Some(coordString.split('.').map(coord => Integer.parseInt(coord))) - case _ => None - } + ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 038812c2bf1..b1a931ecca1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -5,6 +5,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader import com.scalableminds.webknossos.datastore.rpc.RPC import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle @@ -29,6 +30,11 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithJsonResponse[ZarrHeader] + def getZarrJson(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[Zarr3ArrayHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/zarr.json") + .addQueryStringOptional("token", token) + .getWithJsonResponse[Zarr3ArrayHeader] + def getVolumeLayerAsZarrLayer(tracingId: String, tracingName: Option[String], tracingStoreUri: String, @@ -52,6 +58,15 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithBytesResponse + def getRawZarr3Cube(tracingId: String, + mag: String, + coordinates: String, + tracingStoreUri: String, + token: Option[String]): Fox[Array[Byte]] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/$coordinates").silent + .addQueryStringOptional("token", token) + .getWithBytesResponse + def getDataLayerMagFolderContents(tracingId: String, mag: String, tracingStoreUri: String, diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index 8945fe87a12..ef9f51e86fb 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -46,6 +46,9 @@ GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestRawZarr3Cube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.rawZarr3CubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) + # Segmentation mappings GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mappingName: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 38a4c05f7ee..1fc19c77cca 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -9,10 +9,20 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader +} import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import com.scalableminds.webknossos.datastore.models.WebknossosDataRequest -import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass} +import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} +import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer, ElementClass} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService @@ -133,6 +143,53 @@ class VolumeTracingZarrStreamingController @Inject()( } } + def zarrJsonForMag(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { + implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int + .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + + additionalAxes = AdditionalAxis.fromProtos(tracing.additionalAxes) + dimNames = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + + zarrHeader = Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array( + (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x, + (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y, + (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z + ), + data_type = Left(tracing.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = dimNames + ) + } yield Ok(Json.toJson(zarrHeader)) + } + } + def zGroup(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) @@ -213,6 +270,58 @@ class VolumeTracingZarrStreamingController @Inject()( magParsed, Vec3Int(x, y, z), cubeSize, + None, + urlOrHeaderToken(token, request)) ~> NOT_FOUND + } yield Ok(dataWithFallback) + } + } + } + + def rawZarr3Cube(token: Option[String], tracingId: String, mag: String, coordinates: String): Action[AnyContent] = + Action.async { implicit request => + { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + + parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> Messages( + "zarr.invalidChunkCoordinates") ~> NOT_FOUND + (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), + parsedCoordinates(parsedCoordinates.length - 2), + parsedCoordinates(parsedCoordinates.length - 1)) + additionalCoordinates = Some( + parsedCoordinates + .slice(1, parsedCoordinates.length - 3) + .zipWithIndex + .map(coordWithIndex => + new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) + .toList) + _ <- bool2Fox(parsedCoordinates.head == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND + cubeSize = DataLayer.bucketLength + wkRequest = WebknossosDataRequest( + position = Vec3Int(x, y, z) * cubeSize * magParsed, + mag = magParsed, + cubeSize = cubeSize, + fourBit = Some(false), + applyAgglomerate = None, + version = None, + additionalCoordinates = additionalCoordinates + ) + (data, missingBucketIndices) <- if (tracing.getMappingIsEditable) + editableMappingService.volumeData(tracing, tracingId, List(wkRequest), urlOrHeaderToken(token, request)) + else tracingService.data(tracingId, tracing, List(wkRequest)) + dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, + tracingId, + data, + missingBucketIndices, + magParsed, + Vec3Int(x, y, z), + cubeSize, + additionalCoordinates, urlOrHeaderToken(token, request)) ~> NOT_FOUND } yield Ok(dataWithFallback) } @@ -226,6 +335,7 @@ class VolumeTracingZarrStreamingController @Inject()( mag: Vec3Int, position: Vec3Int, cubeSize: Int, + additionalCoordinates: Option[Seq[AdditionalCoordinate]], urlToken: Option[String]): Fox[Array[Byte]] = if (missingBucketIndices.nonEmpty) { for { @@ -237,7 +347,7 @@ class VolumeTracingZarrStreamingController @Inject()( fourBit = Some(false), applyAgglomerate = tracing.mappingName, version = None, - additionalCoordinates = None + additionalCoordinates = additionalCoordinates ) (fallbackData, fallbackMissingBucketIndices) <- remoteDataStoreClient.getData(remoteFallbackLayer, List(request), diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index b10d4276af0..199ac159ec4 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -52,6 +52,9 @@ GET /volume/zarr/:tracingId/:mag/ @com.scalablemin GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) +GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarr3Cube(token: Option[String], tracingId: String, mag: String, coordinates: String) + # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String]) POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple(token: Option[String]) From 229a3748f6c787d610f63c3d486c62f361412aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 26 Jul 2024 15:31:41 +0200 Subject: [PATCH 03/22] add basic zarr 3 ngff v2 group header route --- .../Zarr3StreamingController.scala | 30 ++++------- .../datareaders/zarr/NgffMetadata.scala | 46 ++++++++++++++++ .../datareaders/zarr3/Zarr3GroupHeader.scala | 52 +++++++++++++++++++ ....scalableminds.webknossos.datastore.routes | 2 + 4 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala index d5eb8c151d1..25bff384774 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala @@ -7,25 +7,11 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, BloscCompressor, StringCompressionSetting} -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ - BloscCodecConfiguration, - BytesCodecConfiguration, - ChunkGridConfiguration, - ChunkGridSpecification, - ChunkKeyEncoding, - ChunkKeyEncodingConfiguration, - TransposeCodecConfiguration, - TransposeSetting, - Zarr3ArrayHeader -} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BloscCodecConfiguration, BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{ - Cuboid, - DataServiceDataRequest, - DataServiceRequestSettings -} +import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo @@ -54,7 +40,7 @@ class Zarr3StreamingController @Inject()( * Serve .zattrs file for a dataset * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) */ - def requestZAttrs( + def requestZarrJson( token: Option[String], organizationName: String, datasetName: String, @@ -67,8 +53,9 @@ class Zarr3StreamingController @Inject()( datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - omeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) - } yield Ok(Json.toJson(omeNgffHeader)) + omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) + } yield Ok(Json.toJson(zarr3GroupHeader)) } } @@ -490,7 +477,8 @@ class Zarr3StreamingController @Inject()( )) } - def requestZGroup(token: Option[String], + // TODOM + def requestZarrJsonbla(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala index cb55fbaf3ec..b1b9d66d761 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala @@ -57,6 +57,22 @@ object NgffMultiscalesItem { implicit val jsonFormat: OFormat[NgffMultiscalesItem] = Json.format[NgffMultiscalesItem] } +case class NgffMultiscalesItemV2( + // Ngff V2 no longer has the version inside the multiscale field. + name: Option[String], + axes: List[NgffAxis] = List( + NgffAxis(name = "c", `type` = "channel"), + NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), + ), + datasets: List[NgffDataset] +) + +object NgffMultiscalesItemV2 { + implicit val jsonFormat: OFormat[NgffMultiscalesItemV2] = Json.format[NgffMultiscalesItemV2] +} + case class NgffMetadata(multiscales: List[NgffMultiscalesItem], omero: Option[NgffOmeroMetadata]) object NgffMetadata { @@ -86,6 +102,36 @@ object NgffMetadata { val FILENAME_DOT_ZATTRS = ".zattrs" } +case class NgffMetadataV2(version: String, multiscales: List[NgffMultiscalesItemV2], omero: Option[NgffOmeroMetadata]) + +object NgffMetadataV2 { + def fromNameVoxelSizeAndMags(dataLayerName: String, + dataSourceVoxelSize: VoxelSize, + mags: List[Vec3Int], + version: String = "0.5"): NgffMetadataV2 = { + val datasets = mags.map( + mag => + NgffDataset( + path = mag.toMagLiteral(allowScalar = true), + List(NgffCoordinateTransformation( + scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) + )) + val lengthUnitStr = dataSourceVoxelSize.unit.toString + val axes = List( + NgffAxis(name = "c", `type` = "channel"), + NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), + ) + NgffMetadataV2(version, + multiscales = + List(NgffMultiscalesItemV2(name = Some(dataLayerName), datasets = datasets, axes = axes)), + None) + } + + implicit val jsonFormat: OFormat[NgffMetadataV2] = Json.format[NgffMetadataV2] +} + case class NgffLabelsGroup(labels: List[String]) object NgffLabelsGroup { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala new file mode 100644 index 00000000000..0dfed9d62c1 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -0,0 +1,52 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr3 + +import com.scalableminds.util.tools.BoxImplicits +import com.scalableminds.webknossos.datastore.datareaders.ArrayDataType.ArrayDataType +import com.scalableminds.webknossos.datastore.datareaders.ArrayOrder.ArrayOrder +import com.scalableminds.webknossos.datastore.datareaders.DimensionSeparator.DimensionSeparator +import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3DataType.{Zarr3DataType, raw} +import com.scalableminds.webknossos.datastore.datareaders._ +import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV2.jsonFormat +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, NgffMetadataV2} +import com.scalableminds.webknossos.datastore.helpers.JsonImplicits +import net.liftweb.common.Box.tryo +import net.liftweb.common.{Box, Full} +import play.api.libs.json._ + +import java.nio.ByteOrder + +case class Zarr3GroupHeader( + zarr_format: Int, // must be 3 + node_type: String, // must be "group" + ngffMetadata: Option[NgffMetadataV2], +) + +object Zarr3GroupHeader extends JsonImplicits { + + def FILENAME_ZARR_JSON = "zarr.json" + implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { + override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = + for { + zarr_format <- (json \ "zarr_format").validate[Int] + node_type <- (json \ "node_type").validate[String] + ngffMetadata = (json \ "attributes" \ "ome").validate[NgffMetadataV2].asOpt + } yield + Zarr3GroupHeader( + zarr_format, + node_type, + ngffMetadata, + ) + + override def writes(zarr3GroupHeader: Zarr3GroupHeader): JsValue = { + val groupHeaderBuilder = Json.newBuilder + groupHeaderBuilder ++= Seq( + "zarr_format" -> zarr3GroupHeader.zarr_format, + "node_type" -> zarr3GroupHeader.node_type, + ) + if (zarr3GroupHeader.ngffMetadata.isDefined) { + groupHeaderBuilder += ("attributes" -> Json.obj("ome" -> zarr3GroupHeader.ngffMetadata.get)) + } + groupHeaderBuilder.result() + } + } +} diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index ef9f51e86fb..ec2ebb9bf0a 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -43,9 +43,11 @@ GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, cxyz: String) # Zarr3 compatible routes +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestRawZarr3Cube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) + GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.rawZarr3CubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) From 05c8480da6fe39592faeb391f7793409fe9f27d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 26 Jul 2024 16:19:27 +0200 Subject: [PATCH 04/22] add zarr 3 ngff v2 group header route for annotations --- .../Zarr3StreamingController.scala | 13 ++++--- .../services/DSRemoteTracingstoreClient.scala | 16 ++++++-- ....scalableminds.webknossos.datastore.routes | 1 + ...VolumeTracingZarrStreamingController.scala | 37 +++++++++++-------- ...alableminds.webknossos.tracingstore.routes | 1 + 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala index 25bff384774..ea79fa15ce1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala @@ -59,7 +59,7 @@ class Zarr3StreamingController @Inject()( } } - def zAttrsWithAnnotationPrivateLink(token: Option[String], + def zarrJsonWithAnnotationPrivateLink(token: Option[String], accessToken: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => @@ -68,21 +68,22 @@ class Zarr3StreamingController @Inject()( relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) - omeNgffHeader <- annotationLayer match { + zarr3GroupHeader <- annotationLayer match { case Some(layer) => - remoteTracingstoreClient.getOmeNgffHeader(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) case None => for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( annotationSource.organizationName, annotationSource.datasetName, dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, + dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) - } yield dataSourceOmeNgffHeader + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) + } yield zarr3GroupHeader } - } yield Ok(Json.toJson(omeNgffHeader)) + } yield Ok(Json.toJson(zarr3GroupHeader)) } /** diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index b1a931ecca1..29c97f117e0 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -4,8 +4,8 @@ import com.google.inject.Inject import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, NgffMetadataV2, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.rpc.RPC import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle @@ -30,7 +30,10 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithJsonResponse[ZarrHeader] - def getZarrJson(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[Zarr3ArrayHeader] = + def getZarrJson(tracingId: String, + mag: String, + tracingStoreUri: String, + token: Option[String]): Fox[Zarr3ArrayHeader] = rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/zarr.json") .addQueryStringOptional("token", token) .getWithJsonResponse[Zarr3ArrayHeader] @@ -49,6 +52,13 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithJsonResponse[NgffMetadata] + def getZarrJsonGroupHeaderWithNgff(tracingId: String, + tracingStoreUri: String, + token: Option[String]): Fox[Zarr3GroupHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/zarr.json") + .addQueryStringOptional("token", token) + .getWithJsonResponse[Zarr3GroupHeader] + def getRawZarrCube(tracingId: String, mag: String, cxyz: String, diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index ec2ebb9bf0a..7fc5ff43099 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -48,6 +48,7 @@ GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/ GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestRawZarr3Cube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.rawZarr3CubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 1fc19c77cca..2cbaf19e620 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -8,17 +8,8 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ - BytesCodecConfiguration, - ChunkGridConfiguration, - ChunkGridSpecification, - ChunkKeyEncoding, - ChunkKeyEncodingConfiguration, - TransposeCodecConfiguration, - TransposeSetting, - Zarr3ArrayHeader -} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} @@ -26,11 +17,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService -import com.scalableminds.webknossos.tracingstore.{ - TSRemoteDatastoreClient, - TSRemoteWebknossosClient, - TracingStoreAccessTokenService -} +import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService} import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent} @@ -218,6 +205,24 @@ class VolumeTracingZarrStreamingController @Inject()( } } + def zarrJson( + token: Option[String], + tracingId: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND + omeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(tracingId, + dataSourceVoxelSize = dataSource.scale, + mags = existingMags.toList) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeader)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + } + } + def zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 199ac159ec4..8967fbcfd7e 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -52,6 +52,7 @@ GET /volume/zarr/:tracingId/:mag/ @com.scalablemin GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) +GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarr3Cube(token: Option[String], tracingId: String, mag: String, coordinates: String) From 36286737362b43da8d8c820dcf3b9c676add597a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 26 Jul 2024 16:39:48 +0200 Subject: [PATCH 05/22] merge Zarr3StreamController into ZarrStreamingController --- .../controllers/ZarrStreamingController.scala | 170 +++++++++++++++--- ....scalableminds.webknossos.datastore.routes | 12 +- 2 files changed, 156 insertions(+), 26 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index b6ef82ef9fc..faf25c19c0a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -7,15 +7,12 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.AxisOrder -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} -import com.scalableminds.webknossos.datastore.models.{VoxelPosition, VoxelSize} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} +import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{ - Cuboid, - DataServiceDataRequest, - DataServiceRequestSettings -} +import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo import play.api.i18n.{Messages, MessagesProvider} @@ -61,6 +58,25 @@ class ZarrStreamingController @Inject()( } } + def requestZarrJson( + token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String = "", + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, + datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + } + } + def zAttrsWithAnnotationPrivateLink(token: Option[String], accessToken: String, dataLayerName: String = ""): Action[AnyContent] = @@ -87,6 +103,34 @@ class ZarrStreamingController @Inject()( } yield Ok(Json.toJson(omeNgffHeader)) } + + def zarrJsonWithAnnotationPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String = ""): Action[AnyContent] = + Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) + annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) + zarr3GroupHeader <- annotationLayer match { + case Some(layer) => + remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + case None => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( + annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) + } yield zarr3GroupHeader + } + } yield Ok(Json.toJson(zarr3GroupHeader)) + } + /** * Zarr-specific datasource-properties.json file for a datasource. * Note that the result here is not necessarily equal to the file used in the underlying storage. @@ -174,19 +218,19 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String, mag: String, - cxyz: String, + coordinates: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - rawZarrCube(organizationName, datasetName, dataLayerName, mag, cxyz) + rawZarrCube(organizationName, datasetName, dataLayerName, mag, coordinates) } } def rawZarrCubePrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String, - cxyz: String): Action[AnyContent] = + accessToken: String, + dataLayerName: String, + mag: String, + coordinates: String): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -198,29 +242,47 @@ class ZarrStreamingController @Inject()( result <- layer match { case Some(annotationLayer) => remoteTracingstoreClient - .getRawZarrCube(annotationLayer.tracingId, mag, cxyz, annotationSource.tracingStoreUrl, relevantToken) + .getRawZarrCube(annotationLayer.tracingId, + mag, + coordinates, + annotationSource.tracingStoreUrl, + relevantToken) .map(Ok(_)) case None => - rawZarrCube(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag, cxyz) + rawZarrCube(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + mag, + coordinates) } } yield result } - private def rawZarrCube( + private def rawZarrCube( organizationName: String, datasetName: String, dataLayerName: String, mag: String, - cxyz: String, + coordinates: String, )(implicit m: MessagesProvider): Fox[Result] = for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND - (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND + // (c, x, y, z) + parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND // TODO: change error message magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - _ <- bool2Fox(c == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND + _ <- bool2Fox(parsedCoordinates.head == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND + (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), + parsedCoordinates(parsedCoordinates.length - 2), + parsedCoordinates(parsedCoordinates.length - 1)) + additionalCoordinates = Some( + parsedCoordinates + .slice(1, parsedCoordinates.length - 3) + .zipWithIndex + .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) + .toList) cubeSize = DataLayer.bucketLength request = DataServiceDataRequest( dataSource, @@ -234,7 +296,7 @@ class ZarrStreamingController @Inject()( height = cubeSize, depth = cubeSize ), - DataServiceRequestSettings(halfByte = false) + DataServiceRequestSettings(halfByte = false, additionalCoordinates = additionalCoordinates) ) (data, notFoundIndices) <- binaryDataService.handleDataRequests(List(request)) _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND @@ -262,6 +324,54 @@ class ZarrStreamingController @Inject()( zarrHeader = ZarrHeader.fromLayer(dataLayer, magParsed) } yield Ok(Json.toJson(zarrHeader)) + + def requestZarrJsonForMag(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + zarrJsonForMag(organizationName, datasetName, dataLayerName, mag) + } + } + + private def zarrJsonForMag(organizationName: String, datasetName: String, dataLayerName: String, mag: String)( + implicit m: MessagesProvider): Fox[Result] = + for { + (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + additionalAxes = dataLayer.additionalAxes.getOrElse(Seq.empty) + zarrHeader = Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, + data_type = Left(dataLayer.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + ) + } yield Ok(Json.toJson(zarrHeader)) + def zArrayPrivateLink(token: Option[String], accessToken: String, dataLayerName: String, @@ -282,6 +392,26 @@ class ZarrStreamingController @Inject()( } yield result } + def zArray3PrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = Action.async { implicit request => + for { + annotationSource <- remoteWebknossosClient + .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) + layer = annotationSource.getAnnotationLayer(dataLayerName) + result <- layer match { + case Some(annotationLayer) => + remoteTracingstoreClient + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))) + case None => + zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + } + } yield result + } + def requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index 7fc5ff43099..ce75e0159ab 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -43,14 +43,14 @@ GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, cxyz: String) # Zarr3 compatible routes -GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) -GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.requestRawZarr3Cube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.Zarr3StreamingController.rawZarr3CubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Segmentation mappings From 886652f01697ce46b655ff0d0831083cca5bc75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 26 Jul 2024 17:14:01 +0200 Subject: [PATCH 06/22] ensure full match when parsing zarr coordinates to avoid parsing any non-numerical characters --- .../datastore/dataformats/zarr/ZarrCoordinatesParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 43b9b77ec26..bb4585d5025 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -4,7 +4,7 @@ object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, ): Option[(Int, Int, Int, Int)] = { - val singleRx = "\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*".r + val singleRx = "^\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*$".r cxyz match { case singleRx(c, x, y, z) => @@ -16,7 +16,7 @@ object ZarrCoordinatesParser { def parseNDimensionalDotCoordinates( coordinates: String, ): Option[Array[Int]] = { - val ndCoordinatesRx = "\\s*([0-9]+).([0-9]+).([0-9]+)(.([0-9]+))+\\s*".r + val ndCoordinatesRx = "^\\s*([0-9]+).([0-9]+).([0-9]+)(.([0-9]+))+\\s*$".r ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) } } From 5f3f54c72d64c5e41f3144ea03ef9b3274cb368c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 30 Jul 2024 10:28:03 +0200 Subject: [PATCH 07/22] make NgffMetadataV2 nd compatible --- .../datastore/datareaders/zarr/NgffMetadata.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala index b1b9d66d761..bd078371f53 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala @@ -2,6 +2,7 @@ package com.scalableminds.webknossos.datastore.datareaders.zarr; import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.models +import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} import net.liftweb.common.{Box, Failure, Full} import play.api.libs.json.{Json, OFormat} @@ -108,6 +109,7 @@ object NgffMetadataV2 { def fromNameVoxelSizeAndMags(dataLayerName: String, dataSourceVoxelSize: VoxelSize, mags: List[Vec3Int], + additionalAxes: Option[Seq[AdditionalAxis]], version: String = "0.5"): NgffMetadataV2 = { val datasets = mags.map( mag => @@ -117,8 +119,10 @@ object NgffMetadataV2 { scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) )) val lengthUnitStr = dataSourceVoxelSize.unit.toString - val axes = List( - NgffAxis(name = "c", `type` = "channel"), + val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes + .getOrElse(List.empty) + .zipWithIndex + .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), From cde2845f4880f2a292ae1f56067c220d1d2ca439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 30 Jul 2024 13:00:58 +0200 Subject: [PATCH 08/22] refactor code --- .../Zarr3StreamingController.scala | 74 ++-- .../controllers/ZarrStreamingController.scala | 378 +++++++++--------- .../zarr/ZarrCoordinatesParser.scala | 4 +- .../datareaders/zarr3/Zarr3ArrayHeader.scala | 45 ++- ....scalableminds.webknossos.datastore.routes | 2 +- ...VolumeTracingZarrStreamingController.scala | 78 ++-- .../volume/Zarr3BucketStreamSink.scala | 34 +- ...alableminds.webknossos.tracingstore.routes | 4 +- 8 files changed, 293 insertions(+), 326 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala index ea79fa15ce1..b6aa7c40fa1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala @@ -7,11 +7,31 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, BloscCompressor, StringCompressionSetting} -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BloscCodecConfiguration, BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV2, + ZarrHeader +} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BloscCodecConfiguration, + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader, + Zarr3GroupHeader +} import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} +import com.scalableminds.webknossos.datastore.models.requests.{ + Cuboid, + DataServiceDataRequest, + DataServiceRequestSettings +} import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo @@ -53,15 +73,17 @@ class Zarr3StreamingController @Inject()( datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) + omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) } yield Ok(Json.toJson(zarr3GroupHeader)) } } def zarrJsonWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String = ""): Action[AnyContent] = + accessToken: String, + dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -70,7 +92,9 @@ class Zarr3StreamingController @Inject()( annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) zarr3GroupHeader <- annotationLayer match { case Some(layer) => - remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, + annotationSource.tracingStoreUrl, + relevantToken) case None => for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( @@ -78,8 +102,8 @@ class Zarr3StreamingController @Inject()( annotationSource.datasetName, dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions) + dataSource.scale, + dataLayer.resolutions) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) } yield zarr3GroupHeader } @@ -277,31 +301,7 @@ class Zarr3StreamingController @Inject()( magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND additionalAxes = dataLayer.additionalAxes.getOrElse(Seq.empty) - zarrHeader = Zarr3ArrayHeader( - zarr_format = 3, - node_type = "array", - // channel, additional axes, XYZ - shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, - data_type = Left(dataLayer.elementClass.toString), - chunk_grid = Left( - ChunkGridSpecification( - "regular", - ChunkGridConfiguration( - chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, - DataLayer.bucketLength, - DataLayer.bucketLength)) - )), - chunk_key_encoding = - ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), - fill_value = Right(0), - attributes = None, - codecs = Seq( - TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), - BytesCodecConfiguration(Some("little")), - ), - storage_transformers = None, - dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) - ) + zarrHeader = Zarr3ArrayHeader.fromDataLayerToVersion3(dataLayer) } yield Ok(Json.toJson(zarrHeader)) def zArray3PrivateLink(token: Option[String], @@ -480,9 +480,9 @@ class Zarr3StreamingController @Inject()( // TODOM def requestZarrJsonbla(token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => + organizationName: String, + datasetName: String, + dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessForSyncBlock( UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index faf25c19c0a..1a5a4ef7b55 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -7,12 +7,31 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.AxisOrder -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV2, + ZarrHeader +} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader, + Zarr3GroupHeader +} import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType, AnnotationSource} import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} +import com.scalableminds.webknossos.datastore.models.requests.{ + Cuboid, + DataServiceDataRequest, + DataServiceRequestSettings +} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo import play.api.i18n.{Messages, MessagesProvider} @@ -58,7 +77,7 @@ class ZarrStreamingController @Inject()( } } - def requestZarrJson( + def requestZarrJson( token: Option[String], organizationName: String, datasetName: String, @@ -71,7 +90,10 @@ class ZarrStreamingController @Inject()( datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions) + omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) } yield Ok(Json.toJson(zarr3GroupHeader)) } @@ -81,54 +103,54 @@ class ZarrStreamingController @Inject()( accessToken: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) - omeNgffHeader <- annotationLayer match { - case Some(layer) => - remoteTracingstoreClient.getOmeNgffHeader(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - case None => - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( - annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions) - } yield dataSourceOmeNgffHeader - } - } yield Ok(Json.toJson(omeNgffHeader)) + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + remoteTracingstoreClient + .getOmeNgffHeader(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(ngffMetadata => Ok(Json.toJson(ngffMetadata))) + }, + orElse = annotationSource => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions) + } yield Ok(Json.toJson(dataSourceOmeNgffHeader)) + ) } - def zarrJsonWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String = ""): Action[AnyContent] = + accessToken: String, + dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) - zarr3GroupHeader <- annotationLayer match { - case Some(layer) => - remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - case None => - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( - annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + remoteTracingstoreClient + .getZarrJsonGroupHeaderWithNgff(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(header => Ok(Json.toJson(header))) + }, + orElse = annotationSource => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, - dataLayer.resolutions) - zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) - } yield zarr3GroupHeader - } - } yield Ok(Json.toJson(zarr3GroupHeader)) + dataLayer.resolutions, + dataLayer.additionalAxes) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + ) } /** @@ -227,38 +249,29 @@ class ZarrStreamingController @Inject()( } def rawZarrCubePrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String, - coordinates: String): Action[AnyContent] = + accessToken: String, + dataLayerName: String, + mag: String, + coordinates: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - - // ensures access to volume layers if fallback layer with equal name exists - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getRawZarrCube(annotationLayer.tracingId, - mag, - coordinates, - annotationSource.tracingStoreUrl, - relevantToken) - .map(Ok(_)) - case None => - rawZarrCube(annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName, - mag, - coordinates) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getRawZarrCube(annotationLayer.tracingId, + mag, + coordinates, + annotationSource.tracingStoreUrl, + relevantToken) + .map(Ok(_)), + orElse = (annotationSource) => + rawZarrCube(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag, coordinates) + ) } - private def rawZarrCube( + private def rawZarrCube( organizationName: String, datasetName: String, dataLayerName: String, @@ -324,7 +337,6 @@ class ZarrStreamingController @Inject()( zarrHeader = ZarrHeader.fromLayer(dataLayer, magParsed) } yield Ok(Json.toJson(zarrHeader)) - def requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, @@ -344,73 +356,59 @@ class ZarrStreamingController @Inject()( "dataSource.notFound") ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - additionalAxes = dataLayer.additionalAxes.getOrElse(Seq.empty) - zarrHeader = Zarr3ArrayHeader( - zarr_format = 3, - node_type = "array", - // channel, additional axes, XYZ - shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, - data_type = Left(dataLayer.elementClass.toString), - chunk_grid = Left( - ChunkGridSpecification( - "regular", - ChunkGridConfiguration( - chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, - DataLayer.bucketLength, - DataLayer.bucketLength)) - )), - chunk_key_encoding = - ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), - fill_value = Right(0), - attributes = None, - codecs = Seq( - TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), - BytesCodecConfiguration(Some("little")), - ), - storage_transformers = None, - dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) - ) + zarrHeader = Zarr3ArrayHeader.fromDataLayerToVersion3(dataLayer) } yield Ok(Json.toJson(zarrHeader)) def zArrayPrivateLink(token: Option[String], accessToken: String, dataLayerName: String, mag: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient - .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) - .map(z => Ok(Json.toJson(z))) - case None => - zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))), + orElse = (annotationSource) => + zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + ) + } + + def zarrJsonPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = Action.async { implicit request => + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))), + orElse = annotationSource => + zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + ) } - def zArray3PrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = Action.async { implicit request => + def ifIsAnnotationLayerOrElse( + token: Option[String], + accessToken: String, + dataLayerName: String, + ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], + orElse: (AnnotationSource) => Fox[Result])(implicit request: Request[Any]): Fox[Result] = for { - annotationSource <- remoteWebknossosClient - .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) layer = annotationSource.getAnnotationLayer(dataLayerName) result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) - .map(z => Ok(Json.toJson(z))) - case None => - zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + case Some(annotationLayer) => ifIsAnnotationLayer(annotationLayer, annotationSource, relevantToken) + case None => orElse(annotationSource) } } yield result - } def requestDataLayerMagFolderContents(token: Option[String], organizationName: String, @@ -445,33 +443,30 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerMagFolderContents(annotationLayer.tracingId, - mag, - annotationSource.tracingStoreUrl, - relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Combined Annotation Route", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerMagFolderContents(annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName, - mag) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getDataLayerMagFolderContents(annotationLayer.tracingId, + mag, + annotationSource.tracingStoreUrl, + relevantToken) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Combined Annotation Route", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()), + orElse = annotationSource => + dataLayerMagFolderContents(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + mag) + ) } def requestDataLayerFolderContents(token: Option[String], @@ -502,27 +497,24 @@ class ZarrStreamingController @Inject()( accessToken: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Tracingstore", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Tracingstore", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()), + orElse = annotationSource => + dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) + ) } def requestDataSourceFolderContents(token: Option[String], @@ -581,19 +573,15 @@ class ZarrStreamingController @Inject()( def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map(Ok(_)) - case None => - Fox.successful(Ok(zGroupJson)) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(Ok(_)), + orElse = _ => Fox.successful(Ok(zGroupJson)) + ) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index bb4585d5025..f37ac9776ee 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -4,7 +4,7 @@ object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, ): Option[(Int, Int, Int, Int)] = { - val singleRx = "^\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*$".r + val singleRx = "^\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r cxyz match { case singleRx(c, x, y, z) => @@ -16,7 +16,7 @@ object ZarrCoordinatesParser { def parseNDimensionalDotCoordinates( coordinates: String, ): Option[Array[Int]] = { - val ndCoordinatesRx = "^\\s*([0-9]+).([0-9]+).([0-9]+)(.([0-9]+))+\\s*$".r + val ndCoordinatesRx = "^\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 3dd32b32c43..26cb196dbc6 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -5,14 +5,9 @@ import com.scalableminds.webknossos.datastore.datareaders.ArrayDataType.ArrayDat import com.scalableminds.webknossos.datastore.datareaders.ArrayOrder.ArrayOrder import com.scalableminds.webknossos.datastore.datareaders.DimensionSeparator.DimensionSeparator import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3DataType.{Zarr3DataType, raw} -import com.scalableminds.webknossos.datastore.datareaders.{ - ArrayOrder, - Compressor, - DatasetHeader, - DimensionSeparator, - NullCompressor -} +import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, Compressor, DatasetHeader, DimensionSeparator, NullCompressor} import com.scalableminds.webknossos.datastore.helpers.JsonImplicits +import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer} import net.liftweb.common.Box.tryo import net.liftweb.common.{Box, Full} import play.api.libs.json.{Format, JsArray, JsResult, JsString, JsSuccess, JsValue, Json, OFormat} @@ -248,5 +243,41 @@ object Zarr3ArrayHeader extends JsonImplicits { "storage_transformers" -> zarrArrayHeader.storage_transformers, "dimension_names" -> zarrArrayHeader.dimension_names ) + + } + def fromDataLayerToVersion3(dataLayer: DataLayer): Zarr3ArrayHeader = { + val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) + Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, + data_type = Left(dataLayer.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + ) + } + private def reorderAdditionalAxes(additionalAxes: Seq[AdditionalAxis]): Seq[AdditionalAxis] = { + val additionalAxesStartIndex = 1 // channel comes first + val sorted = additionalAxes.sortBy(_.index) + sorted.zipWithIndex.map { + case (axis, index) => axis.copy(index = index + additionalAxesStartIndex) + } } } diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index ce75e0159ab..ddf229896b3 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -49,7 +49,7 @@ GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/ GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArray3PrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 2cbaf19e620..2126d134ae5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -8,8 +8,23 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV2, ZarrHeader} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{BytesCodecConfiguration, ChunkGridConfiguration, ChunkGridSpecification, ChunkKeyEncoding, ChunkKeyEncodingConfiguration, TransposeCodecConfiguration, TransposeSetting, Zarr3ArrayHeader, Zarr3GroupHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV2, + ZarrHeader +} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader, + Zarr3GroupHeader +} import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} @@ -17,7 +32,11 @@ import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService -import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService} +import com.scalableminds.webknossos.tracingstore.{ + TSRemoteDatastoreClient, + TSRemoteWebknossosClient, + TracingStoreAccessTokenService +} import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent} @@ -206,9 +225,9 @@ class VolumeTracingZarrStreamingController @Inject()( } def zarrJson( - token: Option[String], - tracingId: String, - ): Action[AnyContent] = Action.async { implicit request => + token: Option[String], + tracingId: String, + ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -216,8 +235,9 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(tracingId, - dataSourceVoxelSize = dataSource.scale, - mags = existingMags.toList) + dataSourceVoxelSize = dataSource.scale, + mags = existingMags.toList, + additionalAxes = dataSource.additionalAxesUnion) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeader)) } yield Ok(Json.toJson(zarr3GroupHeader)) } @@ -242,47 +262,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String): Action[AnyContent] = - Action.async { implicit request => - { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - - existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - - (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND - _ <- bool2Fox(c == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND - cubeSize = DataLayer.bucketLength - wkRequest = WebknossosDataRequest( - position = Vec3Int(x, y, z) * cubeSize * magParsed, - mag = magParsed, - cubeSize = cubeSize, - fourBit = Some(false), - applyAgglomerate = None, - version = None, - additionalCoordinates = None - ) - (data, missingBucketIndices) <- if (tracing.getMappingIsEditable) - editableMappingService.volumeData(tracing, tracingId, List(wkRequest), urlOrHeaderToken(token, request)) - else tracingService.data(tracingId, tracing, List(wkRequest)) - dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, - tracingId, - data, - missingBucketIndices, - magParsed, - Vec3Int(x, y, z), - cubeSize, - None, - urlOrHeaderToken(token, request)) ~> NOT_FOUND - } yield Ok(dataWithFallback) - } - } - } - - def rawZarr3Cube(token: Option[String], tracingId: String, mag: String, coordinates: String): Action[AnyContent] = + def rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 2d1d024ace4..0498ddd0ec2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -39,39 +39,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])( implicit ec: ExecutionContext): Iterator[NamedStream] = { - val header = Zarr3ArrayHeader( - zarr_format = 3, - node_type = "array", - // channel, additional axes, XYZ - shape = Array(1) ++ additionalAxesSorted.map(_.highestValue).toArray ++ layer.boundingBox.bottomRight.toArray, - data_type = Left(layer.elementClass.toString), - chunk_grid = Left( - ChunkGridSpecification( - "regular", - ChunkGridConfiguration( - chunk_shape = Array.fill(1 + additionalAxesSorted.length)(1) ++ Array(DataLayer.bucketLength, - DataLayer.bucketLength, - DataLayer.bucketLength)) - )), - chunk_key_encoding = - ChunkKeyEncoding("default", - configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(dimensionSeparator)))), - fill_value = Right(0), - attributes = None, - codecs = Seq( - TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(rank)), - BytesCodecConfiguration(Some("little")), - BloscCodecConfiguration( - BloscCompressor.defaultCname, - BloscCompressor.defaultCLevel, - StringCompressionSetting(BloscCodecConfiguration.shuffleSettingFromInt(BloscCompressor.defaultShuffle)), - Some(BloscCompressor.defaultTypesize), - BloscCompressor.defaultBlocksize - ) - ), - storage_transformers = None, - dimension_names = Some(Array("c") ++ additionalAxesSorted.map(_.name).toArray ++ Seq("x", "y", "z")) - ) + val header = Zarr3ArrayHeader.fromDataLayerToVersion3(layer) bucketStream.flatMap { case (bucket, data) => val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 8967fbcfd7e..8b6c935fa0d 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -50,11 +50,11 @@ GET /volume/zarr/:tracingId/zarrSource @com.scalablemin GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) +GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarr3Cube(token: Option[String], tracingId: String, mag: String, coordinates: String) +GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String]) From e0628abc8c1b497c99abc62e58d5fe1ace118f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 30 Jul 2024 13:46:38 +0200 Subject: [PATCH 09/22] remove Zarr3StreamingController.scala --- .../Zarr3StreamingController.scala | 512 ------------------ 1 file changed, 512 deletions(-) delete mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala deleted file mode 100644 index b6aa7c40fa1..00000000000 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Zarr3StreamingController.scala +++ /dev/null @@ -1,512 +0,0 @@ -package com.scalableminds.webknossos.datastore.controllers - -import com.google.inject.Inject -import com.scalableminds.util.geometry.Vec3Int -import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.dataformats.MagLocator -import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} -import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser -import com.scalableminds.webknossos.datastore.datareaders.{AxisOrder, BloscCompressor, StringCompressionSetting} -import com.scalableminds.webknossos.datastore.datareaders.zarr.{ - NgffGroupHeader, - NgffMetadata, - NgffMetadataV2, - ZarrHeader -} -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ - BloscCodecConfiguration, - BytesCodecConfiguration, - ChunkGridConfiguration, - ChunkGridSpecification, - ChunkKeyEncoding, - ChunkKeyEncodingConfiguration, - TransposeCodecConfiguration, - TransposeSetting, - Zarr3ArrayHeader, - Zarr3GroupHeader -} -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType -import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{ - Cuboid, - DataServiceDataRequest, - DataServiceRequestSettings -} -import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} -import com.scalableminds.webknossos.datastore.services._ -import net.liftweb.common.Box.tryo -import play.api.i18n.{Messages, MessagesProvider} -import play.api.libs.json.{JsObject, JsValue, Json} -import play.api.mvc._ - -import scala.concurrent.ExecutionContext - -class Zarr3StreamingController @Inject()( - dataSourceRepository: DataSourceRepository, - accessTokenService: DataStoreAccessTokenService, - binaryDataServiceHolder: BinaryDataServiceHolder, - remoteWebknossosClient: DSRemoteWebknossosClient, - remoteTracingstoreClient: DSRemoteTracingstoreClient, -)(implicit ec: ExecutionContext) - extends Controller { - - override def defaultErrorCode: Int = NOT_FOUND - - val binaryDataService: BinaryDataService = binaryDataServiceHolder.binaryDataService - - override def allowRemoteOrigin: Boolean = true - - /** - * Serve .zattrs file for a dataset - * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) - */ - def requestZarrJson( - token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String = "", - ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, - datasetName, - dataLayerName) ?~> Messages( - "dataSource.notFound") ~> NOT_FOUND - omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions) - zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) - } yield Ok(Json.toJson(zarr3GroupHeader)) - } - } - - def zarrJsonWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String = ""): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) - zarr3GroupHeader <- annotationLayer match { - case Some(layer) => - remoteTracingstoreClient.getZarrJsonGroupHeaderWithNgff(layer.tracingId, - annotationSource.tracingStoreUrl, - relevantToken) - case None => - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( - annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions) - zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) - } yield zarr3GroupHeader - } - } yield Ok(Json.toJson(zarr3GroupHeader)) - } - - /** - * Zarr-specific datasource-properties.json file for a datasource. - * Note that the result here is not necessarily equal to the file used in the underlying storage. - */ - def requestDataSource( - token: Option[String], - organizationName: String, - datasetName: String, - ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - for { - dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ~> NOT_FOUND - dataLayers = dataSource.dataLayers - zarrLayers = dataLayers.map(convertLayerToZarrLayer) - zarrSource = GenericDataSource[DataLayer](dataSource.id, zarrLayers, dataSource.scale) - zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) - } yield Ok(Json.toJson(zarrSourceJson)) - } - } - - private def replaceVoxelSizeByLegacyFormat(jsValue: JsValue): Fox[JsValue] = { - val jsObject = jsValue.as[JsObject] - val voxelSizeOpt = (jsObject \ "scale").asOpt[VoxelSize] - voxelSizeOpt match { - case None => Fox.successful(jsObject) - case Some(voxelSize) => - val inNanometer = voxelSize.toNanometer - for { - newDataSource <- tryo(jsObject - "scale" + ("scale" -> Json.toJson(inNanometer))) - } yield newDataSource - } - } - - private def convertLayerToZarrLayer(layer: DataLayer): ZarrLayer = - layer match { - case s: SegmentationLayer => - ZarrSegmentationLayer( - s.name, - s.boundingBox, - s.elementClass, - s.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), - mappings = s.mappings, - largestSegmentId = s.largestSegmentId, - numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1) - ) - case d: DataLayer => - ZarrDataLayer( - d.name, - d.category, - d.boundingBox, - d.elementClass, - d.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), - numChannels = Some(if (d.elementClass == ElementClass.uint24) 3 else 1), - additionalAxes = None - ) - } - - def dataSourceWithAnnotationPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - volumeAnnotationLayers = annotationSource.annotationLayers.filter(_.typ == AnnotationLayerType.Volume) - dataSource <- dataSourceRepository - .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationName)) - .toFox ~> NOT_FOUND - dataSourceLayers = dataSource.dataLayers - .filter(dL => !volumeAnnotationLayers.exists(_.name == dL.name)) - .map(convertLayerToZarrLayer) - annotationLayers <- Fox.serialCombined(volumeAnnotationLayers)( - l => - remoteTracingstoreClient - .getVolumeLayerAsZarrLayer(l.tracingId, Some(l.name), annotationSource.tracingStoreUrl, relevantToken)) - allLayer = dataSourceLayers ++ annotationLayers - zarrSource = GenericDataSource[DataLayer](dataSource.id, allLayer, dataSource.scale) - zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) - } yield Ok(Json.toJson(zarrSourceJson)) - } - - def requestRawZarr3Cube( - token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String, - mag: String, - coordinates: String, - ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - rawZarr3Cube(organizationName, datasetName, dataLayerName, mag, coordinates) - } - } - - def rawZarr3CubePrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String, - coordinates: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - - // ensures access to volume layers if fallback layer with equal name exists - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getRawZarr3Cube(annotationLayer.tracingId, - mag, - coordinates, - annotationSource.tracingStoreUrl, - relevantToken) - .map(Ok(_)) - case None => - rawZarr3Cube(annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName, - mag, - coordinates) - } - } yield result - } - - private def rawZarr3Cube( - organizationName: String, - datasetName: String, - dataLayerName: String, - mag: String, - coordinates: String, - )(implicit m: MessagesProvider): Fox[Result] = - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, - datasetName, - dataLayerName) ~> NOT_FOUND - // (c, x, y, z) - parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND // TODO: change error message - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - _ <- bool2Fox(parsedCoordinates.head == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND - (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), - parsedCoordinates(parsedCoordinates.length - 2), - parsedCoordinates(parsedCoordinates.length - 1)) - additionalCoordinates = Some( - parsedCoordinates - .slice(1, parsedCoordinates.length - 3) - .zipWithIndex - .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) - .toList) - cubeSize = DataLayer.bucketLength - request = DataServiceDataRequest( - dataSource, - dataLayer, - Cuboid( - topLeft = VoxelPosition(x * cubeSize * magParsed.x, - y * cubeSize * magParsed.y, - z * cubeSize * magParsed.z, - magParsed), - width = cubeSize, - height = cubeSize, - depth = cubeSize - ), - DataServiceRequestSettings(halfByte = false, additionalCoordinates = additionalCoordinates) - ) - (data, notFoundIndices) <- binaryDataService.handleDataRequests(List(request)) - _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND - } yield Ok(data) - - def requestZarrJsonForMag(token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String, - mag: String, - ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - zarrJsonForMag(organizationName, datasetName, dataLayerName, mag) - } - } - - private def zarrJsonForMag(organizationName: String, datasetName: String, dataLayerName: String, mag: String)( - implicit m: MessagesProvider): Fox[Result] = - for { - (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( - "dataSource.notFound") ~> NOT_FOUND - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - additionalAxes = dataLayer.additionalAxes.getOrElse(Seq.empty) - zarrHeader = Zarr3ArrayHeader.fromDataLayerToVersion3(dataLayer) - } yield Ok(Json.toJson(zarrHeader)) - - def zArray3PrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient - .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) - .map(z => Ok(Json.toJson(z))) - case None => - zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) - } - } yield result - } - - def requestDataLayerMagFolderContents(token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String, - mag: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag) - } - } - - private def dataLayerMagFolderContents(organizationName: String, - datasetName: String, - dataLayerName: String, - mag: String)(implicit m: MessagesProvider): Fox[Result] = - for { - (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND - magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND - _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Datastore", - "%s/%s/%s/%s".format(organizationName, datasetName, dataLayerName, mag), - List(".zarray") - )).withHeaders() - - def dataLayerMagFolderContentsPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerMagFolderContents(annotationLayer.tracingId, - mag, - annotationSource.tracingStoreUrl, - relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Combined Annotation Route", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerMagFolderContents(annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName, - mag) - } - } yield result - } - - def requestDataLayerFolderContents(token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - dataLayerFolderContents(organizationName, datasetName, dataLayerName) - } - } - - private def dataLayerFolderContents(organizationName: String, datasetName: String, dataLayerName: String)( - implicit m: MessagesProvider): Fox[Result] = - for { - (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( - "dataSource.notFound") ~> NOT_FOUND - mags = dataLayer.resolutions - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Datastore", - "%s/%s/%s".format(organizationName, datasetName, dataLayerName), - List(".zattrs", ".zgroup") ++ mags.map(_.toMagLiteral(allowScalar = true)) - )).withHeaders() - - def dataLayerFolderContentsPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Tracingstore", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) - } - } yield result - } - - def requestDataSourceFolderContents(token: Option[String], - organizationName: String, - datasetName: String): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - for { - dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ?~> Messages( - "dataSource.notFound") ~> NOT_FOUND - layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name) - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Datastore", - s"$organizationName/$datasetName", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames - )) - } - } - - def dataSourceFolderContentsPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) - dataSource <- dataSourceRepository - .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationName)) - .toFox ?~> Messages("dataSource.notFound") ~> NOT_FOUND - annotationLayerNames = annotationSource.annotationLayers.filter(_.typ == AnnotationLayerType.Volume).map(_.name) - dataSourceLayerNames = dataSource.dataLayers - .map((dataLayer: DataLayer) => dataLayer.name) - .filter(!annotationLayerNames.contains(_)) - layerNames = annotationLayerNames ++ dataSourceLayerNames - } yield - Ok( - views.html.datastoreZarrDatasourceDir( - "Combined datastore and tracingstore directory", - s"$accessToken", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames - )) - } - - // TODOM - def requestZarrJsonbla(token: Option[String], - organizationName: String, - datasetName: String, - dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessForSyncBlock( - UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), - urlOrHeaderToken(token, request)) { - Ok(zGroupJson) - } - } - - private def zGroupJson: JsValue = Json.toJson(NgffGroupHeader(zarr_format = 2)) - - def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = - Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map(Ok(_)) - case None => - Fox.successful(Ok(zGroupJson)) - } - } yield result - } -} From 15a3ff0125178b674905c24ec8e7bcaaf9b37ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 30 Jul 2024 14:34:01 +0200 Subject: [PATCH 10/22] minor code fixes; mostly remove unused imports --- .../controllers/ZarrStreamingController.scala | 12 +----------- .../datareaders/zarr3/Zarr3ArrayHeader.scala | 8 +++++++- .../datareaders/zarr3/Zarr3GroupHeader.scala | 12 +----------- .../services/DSRemoteTracingstoreClient.scala | 11 +---------- .../com.scalableminds.webknossos.tracingstore.routes | 2 +- 5 files changed, 11 insertions(+), 34 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 1a5a4ef7b55..e2979098de8 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -13,17 +13,7 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr.{ NgffMetadataV2, ZarrHeader } -import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ - BytesCodecConfiguration, - ChunkGridConfiguration, - ChunkGridSpecification, - ChunkKeyEncoding, - ChunkKeyEncodingConfiguration, - TransposeCodecConfiguration, - TransposeSetting, - Zarr3ArrayHeader, - Zarr3GroupHeader -} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType, AnnotationSource} import com.scalableminds.webknossos.datastore.models.datasource._ diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 26cb196dbc6..061af45a56e 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -5,7 +5,13 @@ import com.scalableminds.webknossos.datastore.datareaders.ArrayDataType.ArrayDat import com.scalableminds.webknossos.datastore.datareaders.ArrayOrder.ArrayOrder import com.scalableminds.webknossos.datastore.datareaders.DimensionSeparator.DimensionSeparator import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3DataType.{Zarr3DataType, raw} -import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, Compressor, DatasetHeader, DimensionSeparator, NullCompressor} +import com.scalableminds.webknossos.datastore.datareaders.{ + ArrayOrder, + Compressor, + DatasetHeader, + DimensionSeparator, + NullCompressor +} import com.scalableminds.webknossos.datastore.helpers.JsonImplicits import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer} import net.liftweb.common.Box.tryo diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala index 0dfed9d62c1..4c8f7c75131 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -1,20 +1,10 @@ package com.scalableminds.webknossos.datastore.datareaders.zarr3 -import com.scalableminds.util.tools.BoxImplicits -import com.scalableminds.webknossos.datastore.datareaders.ArrayDataType.ArrayDataType -import com.scalableminds.webknossos.datastore.datareaders.ArrayOrder.ArrayOrder -import com.scalableminds.webknossos.datastore.datareaders.DimensionSeparator.DimensionSeparator -import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3DataType.{Zarr3DataType, raw} -import com.scalableminds.webknossos.datastore.datareaders._ import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV2.jsonFormat -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, NgffMetadataV2} +import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV2 import com.scalableminds.webknossos.datastore.helpers.JsonImplicits -import net.liftweb.common.Box.tryo -import net.liftweb.common.{Box, Full} import play.api.libs.json._ -import java.nio.ByteOrder - case class Zarr3GroupHeader( zarr_format: Int, // must be 3 node_type: String, // must be "group" diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 29c97f117e0..643c7674e5d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -4,7 +4,7 @@ import com.google.inject.Inject import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, NgffMetadataV2, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, ZarrHeader} import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.rpc.RPC import com.typesafe.scalalogging.LazyLogging @@ -68,15 +68,6 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithBytesResponse - def getRawZarr3Cube(tracingId: String, - mag: String, - coordinates: String, - tracingStoreUri: String, - token: Option[String]): Fox[Array[Byte]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/$coordinates").silent - .addQueryStringOptional("token", token) - .getWithBytesResponse - def getDataLayerMagFolderContents(tracingId: String, mag: String, tracingStoreUri: String, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 8b6c935fa0d..c3e57374715 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -50,7 +50,7 @@ GET /volume/zarr/:tracingId/zarrSource @com.scalablemin GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) +GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) From 6370b42e84c03dd7c3cde5f55d0b070b32654948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 31 Jul 2024 11:44:22 +0200 Subject: [PATCH 11/22] remove fixed full length match of zarr coordinate regex parsing; - wk seems to send c.0.x.y.z requests and the c at the start does not match with the regex resulting in invalid coordinate parsing --- .../datastore/dataformats/zarr/ZarrCoordinatesParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index f37ac9776ee..7199d12d90b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -4,7 +4,7 @@ object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, ): Option[(Int, Int, Int, Int)] = { - val singleRx = "^\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r + val singleRx = "\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*".r cxyz match { case singleRx(c, x, y, z) => @@ -16,7 +16,7 @@ object ZarrCoordinatesParser { def parseNDimensionalDotCoordinates( coordinates: String, ): Option[Array[Int]] = { - val ndCoordinatesRx = "^\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r + val ndCoordinatesRx = "\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*".r ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) } } From 32b58bfb560fa5aeaa94659e772442d80bfdc6b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 31 Jul 2024 11:58:56 +0200 Subject: [PATCH 12/22] add c. as a prefix to the coordinate parsing regex as defined by the zarr spec --- .../datastore/dataformats/zarr/ZarrCoordinatesParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 7199d12d90b..7cc00e8bc1a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -4,7 +4,7 @@ object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, ): Option[(Int, Int, Int, Int)] = { - val singleRx = "\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*".r + val singleRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r cxyz match { case singleRx(c, x, y, z) => @@ -16,7 +16,7 @@ object ZarrCoordinatesParser { def parseNDimensionalDotCoordinates( coordinates: String, ): Option[Array[Int]] = { - val ndCoordinatesRx = "\\s*([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*".r + val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) } } From ebcde2a1efb97d121e55eaf507c15d3aa3827416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 31 Jul 2024 16:04:46 +0200 Subject: [PATCH 13/22] exclude leading c from the zarr3 coordinate parsing --- .../datastore/dataformats/zarr/ZarrCoordinatesParser.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 7cc00e8bc1a..7baf810f47b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -17,6 +17,7 @@ object ZarrCoordinatesParser { coordinates: String, ): Option[Array[Int]] = { val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r - ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').map(coord => Integer.parseInt(coord))) + // The tail cuts of the leading "c" form the "c." at the beginning of coordinates. + ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) } } From 9bb611fa1e4536fec63c36517577db4c1f6fa6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 5 Aug 2024 17:05:38 +0200 Subject: [PATCH 14/22] apply pr feedback --- conf/messages | 3 +- .../controllers/ZarrStreamingController.scala | 34 ++----- .../zarr/ZarrCoordinatesParser.scala | 35 ++++++- .../datareaders/zarr/NgffMetadata.scala | 97 +------------------ .../datareaders/zarr/NgffMetadataV0_5.scala | 58 +++++++++++ .../zarr/SharedNgffMetadataAttributes.scala | 55 +++++++++++ .../datareaders/zarr3/Zarr3ArrayHeader.scala | 2 +- .../datareaders/zarr3/Zarr3GroupHeader.scala | 36 +------ ....scalableminds.webknossos.datastore.routes | 2 - ...VolumeTracingZarrStreamingController.scala | 17 +--- .../volume/Zarr3BucketStreamSink.scala | 2 +- 11 files changed, 164 insertions(+), 177 deletions(-) create mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala create mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala diff --git a/conf/messages b/conf/messages index 88099046e7d..8d0125a9b45 100644 --- a/conf/messages +++ b/conf/messages @@ -138,9 +138,10 @@ dataLayer.mustBeSegmentation=DataLayer “{0}” is not a segmentation layer dataLayer.wrongMag=DataLayer “{0}” does not have mag “{1}” dataLayer.invalidMag=Supplied “{0}” is not a valid mag format. Please use “x-y-z” -zarr.invalidChunkCoordinates=The requested chunk coordinates are in an invalid format. Expected c.x.y.z +zarr.invalidChunkCoordinates=Invalid chunk coordinates. Expected dot separated coordinates with a prefix of “c.”: c.x.y.z zarr.invalidFirstChunkCoord="First Channel must be 0" zarr.chunkNotFound=Could not find the requested chunk +zarr.notEnoughCoordinates=Invalid number of chunk coordinates. Expected to get at least 3 dimensions and channel 0. nml.file.uploadSuccess=Successfully uploaded file nml.file.notFound=Could not extract NML file diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index e2979098de8..0b79561f2e1 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -7,21 +7,12 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.AxisOrder -import com.scalableminds.webknossos.datastore.datareaders.zarr.{ - NgffGroupHeader, - NgffMetadata, - NgffMetadataV2, - ZarrHeader -} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV0_5, ZarrHeader} import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} -import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType, AnnotationSource} import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{ - Cuboid, - DataServiceDataRequest, - DataServiceRequestSettings -} +import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} +import com.scalableminds.webknossos.datastore.models.{VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo import play.api.i18n.{Messages, MessagesProvider} @@ -80,7 +71,7 @@ class ZarrStreamingController @Inject()( datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - omeNgffHeaderV2 = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + omeNgffHeaderV2 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions, dataLayer.additionalAxes) @@ -134,7 +125,7 @@ class ZarrStreamingController @Inject()( annotationSource.datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(dataLayerName, + dataSourceOmeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions, dataLayer.additionalAxes) @@ -272,20 +263,9 @@ class ZarrStreamingController @Inject()( (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND - // (c, x, y, z) - parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND // TODO: change error message + (x, y, z, additionalCoordinates) <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - _ <- bool2Fox(parsedCoordinates.head == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND - (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), - parsedCoordinates(parsedCoordinates.length - 2), - parsedCoordinates(parsedCoordinates.length - 1)) - additionalCoordinates = Some( - parsedCoordinates - .slice(1, parsedCoordinates.length - 3) - .zipWithIndex - .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) - .toList) cubeSize = DataLayer.bucketLength request = DataServiceDataRequest( dataSource, @@ -346,7 +326,7 @@ class ZarrStreamingController @Inject()( "dataSource.notFound") ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - zarrHeader = Zarr3ArrayHeader.fromDataLayerToVersion3(dataLayer) + zarrHeader = Zarr3ArrayHeader.fromDataLayer(dataLayer) } yield Ok(Json.toJson(zarrHeader)) def zArrayPrivateLink(token: Option[String], diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 7baf810f47b..1059afbb34d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -1,5 +1,14 @@ package com.scalableminds.webknossos.datastore.dataformats.zarr +import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} +import com.scalableminds.util.tools.JsonHelper.bool2Box +import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate +import play.api.http.Status.NOT_FOUND +import play.api.i18n.{Messages, MessagesProvider} + +import scala.concurrent.ExecutionContext + object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, @@ -15,9 +24,29 @@ object ZarrCoordinatesParser { def parseNDimensionalDotCoordinates( coordinates: String, - ): Option[Array[Int]] = { + )(implicit ec: ExecutionContext, m: MessagesProvider): Fox[(Int, Int, Int, Option[List[AdditionalCoordinate]])] = { val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r - // The tail cuts of the leading "c" form the "c." at the beginning of coordinates. - ndCoordinatesRx.findFirstIn(coordinates).map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) + // The tail cuts off the leading "c" form the "c." at the beginning of coordinates. + + for { + parsedCoordinates <- ndCoordinatesRx + .findFirstIn(coordinates) + .map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) ?~> + Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND + channelCoordinate <- parsedCoordinates.headOption ~> NOT_FOUND + _ <- bool2Fox(channelCoordinate == 0) ?~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND + _ <- bool2Fox(parsedCoordinates.length >= 4) ?~> "zarr.notEnoughCoordinates" ~> NOT_FOUND + (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), + parsedCoordinates(parsedCoordinates.length - 2), + parsedCoordinates(parsedCoordinates.length - 1)) + additionalCoordinates = if (parsedCoordinates.length > 4) + Some( + parsedCoordinates + .slice(1, parsedCoordinates.length - 3) + .zipWithIndex + .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) + .toList) + else None + } yield (x, y, z, additionalCoordinates) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala index bd078371f53..defc321c901 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala @@ -1,23 +1,12 @@ package com.scalableminds.webknossos.datastore.datareaders.zarr; import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} -import com.scalableminds.webknossos.datastore.models -import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis -import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} -import net.liftweb.common.{Box, Failure, Full} +import com.scalableminds.webknossos.datastore.models.VoxelSize import play.api.libs.json.{Json, OFormat} -case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) -object NgffCoordinateTransformation { - implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] -} -case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) -object NgffDataset { - implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] -} case class NgffGroupHeader(zarr_format: Int) object NgffGroupHeader { @@ -25,23 +14,6 @@ object NgffGroupHeader { val FILENAME_DOT_ZGROUP = ".zgroup" } -case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { - - def lengthUnit: Box[models.LengthUnit.Value] = - if (`type` != "space") - Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") - else { - unit match { - case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) - case Some(someUnit) => LengthUnit.fromString(someUnit) - } - } -} - -object NgffAxis { - implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] -} - case class NgffMultiscalesItem( version: String = "0.4", // format version number name: Option[String], @@ -58,22 +30,6 @@ object NgffMultiscalesItem { implicit val jsonFormat: OFormat[NgffMultiscalesItem] = Json.format[NgffMultiscalesItem] } -case class NgffMultiscalesItemV2( - // Ngff V2 no longer has the version inside the multiscale field. - name: Option[String], - axes: List[NgffAxis] = List( - NgffAxis(name = "c", `type` = "channel"), - NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), - NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), - NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), - ), - datasets: List[NgffDataset] -) - -object NgffMultiscalesItemV2 { - implicit val jsonFormat: OFormat[NgffMultiscalesItemV2] = Json.format[NgffMultiscalesItemV2] -} - case class NgffMetadata(multiscales: List[NgffMultiscalesItem], omero: Option[NgffOmeroMetadata]) object NgffMetadata { @@ -103,39 +59,6 @@ object NgffMetadata { val FILENAME_DOT_ZATTRS = ".zattrs" } -case class NgffMetadataV2(version: String, multiscales: List[NgffMultiscalesItemV2], omero: Option[NgffOmeroMetadata]) - -object NgffMetadataV2 { - def fromNameVoxelSizeAndMags(dataLayerName: String, - dataSourceVoxelSize: VoxelSize, - mags: List[Vec3Int], - additionalAxes: Option[Seq[AdditionalAxis]], - version: String = "0.5"): NgffMetadataV2 = { - val datasets = mags.map( - mag => - NgffDataset( - path = mag.toMagLiteral(allowScalar = true), - List(NgffCoordinateTransformation( - scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) - )) - val lengthUnitStr = dataSourceVoxelSize.unit.toString - val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes - .getOrElse(List.empty) - .zipWithIndex - .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( - NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), - NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), - NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), - ) - NgffMetadataV2(version, - multiscales = - List(NgffMultiscalesItemV2(name = Some(dataLayerName), datasets = datasets, axes = axes)), - None) - } - - implicit val jsonFormat: OFormat[NgffMetadataV2] = Json.format[NgffMetadataV2] -} - case class NgffLabelsGroup(labels: List[String]) object NgffLabelsGroup { @@ -143,21 +66,3 @@ object NgffLabelsGroup { val LABEL_PATH = "labels/.zattrs" } -case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) -object NgffOmeroMetadata { - implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] -} - -case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) -object NgffChannelWindow { - implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] -} - -case class NgffChannelAttributes(color: Option[String], - label: Option[String], - window: Option[NgffChannelWindow], - inverted: Option[Boolean], - active: Option[Boolean]) -object NgffChannelAttributes { - implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] -} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala new file mode 100644 index 00000000000..1d4841c281e --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala @@ -0,0 +1,58 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr + +import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} +import com.scalableminds.webknossos.datastore.models.VoxelSize +import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis +import play.api.libs.json.{Json, OFormat} + +// See suggested changes to version v0.5 here together with an example: https://ngff.openmicroscopy.org/rfc/2/index.html#examples +case class NgffMultiscalesItemV0_5( + // Ngff V0.5 no longer has the version inside the multiscale field. + name: Option[String], + axes: List[NgffAxis] = List( + NgffAxis(name = "c", `type` = "channel"), + NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), + ), + datasets: List[NgffDataset] +) + +object NgffMultiscalesItemV0_5 { + implicit val jsonFormat: OFormat[NgffMultiscalesItemV0_5] = Json.format[NgffMultiscalesItemV0_5] +} + +case class NgffMetadataV0_5(version: String, + multiscales: List[NgffMultiscalesItemV0_5], + omero: Option[NgffOmeroMetadata]) + +object NgffMetadataV0_5 { + def fromNameVoxelSizeAndMags(dataLayerName: String, + dataSourceVoxelSize: VoxelSize, + mags: List[Vec3Int], + additionalAxes: Option[Seq[AdditionalAxis]], + version: String = "0.5"): NgffMetadataV0_5 = { + val datasets = mags.map( + mag => + NgffDataset( + path = mag.toMagLiteral(allowScalar = true), + List(NgffCoordinateTransformation( + scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) + )) + val lengthUnitStr = dataSourceVoxelSize.unit.toString + val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes + .getOrElse(List.empty) + .zipWithIndex + .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( + NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), + ) + NgffMetadataV0_5(version, + multiscales = + List(NgffMultiscalesItemV0_5(name = Some(dataLayerName), datasets = datasets, axes = axes)), + None) + } + + implicit val jsonFormat: OFormat[NgffMetadataV0_5] = Json.format[NgffMetadataV0_5] +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala new file mode 100644 index 00000000000..56e2a3a9fb8 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala @@ -0,0 +1,55 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr + +import com.scalableminds.webknossos.datastore.models +import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} +import net.liftweb.common.{Box, Failure, Full} +import play.api.libs.json.{Json, OFormat} + +case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) + +object NgffCoordinateTransformation { + implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] +} + +case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) + +object NgffDataset { + implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] +} + +case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { + + def lengthUnit: Box[models.LengthUnit.Value] = + if (`type` != "space") + Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") + else { + unit match { + case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) + case Some(someUnit) => LengthUnit.fromString(someUnit) + } + } +} + +object NgffAxis { + implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] +} + +case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) +object NgffOmeroMetadata { + implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] +} + +case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) +object NgffChannelWindow { + implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] +} + +case class NgffChannelAttributes(color: Option[String], + label: Option[String], + window: Option[NgffChannelWindow], + inverted: Option[Boolean], + active: Option[Boolean]) +object NgffChannelAttributes { + implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] +} + diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 061af45a56e..005c3a1d99f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -251,7 +251,7 @@ object Zarr3ArrayHeader extends JsonImplicits { ) } - def fromDataLayerToVersion3(dataLayer: DataLayer): Zarr3ArrayHeader = { + def fromDataLayer (dataLayer: DataLayer): Zarr3ArrayHeader = { val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) Zarr3ArrayHeader( zarr_format = 3, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala index 4c8f7c75131..affa4837535 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -1,42 +1,14 @@ package com.scalableminds.webknossos.datastore.datareaders.zarr3 -import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV2.jsonFormat -import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV2 -import com.scalableminds.webknossos.datastore.helpers.JsonImplicits +import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV0_5 import play.api.libs.json._ case class Zarr3GroupHeader( zarr_format: Int, // must be 3 node_type: String, // must be "group" - ngffMetadata: Option[NgffMetadataV2], + ngffMetadata: Option[NgffMetadataV0_5], ) -object Zarr3GroupHeader extends JsonImplicits { - - def FILENAME_ZARR_JSON = "zarr.json" - implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { - override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = - for { - zarr_format <- (json \ "zarr_format").validate[Int] - node_type <- (json \ "node_type").validate[String] - ngffMetadata = (json \ "attributes" \ "ome").validate[NgffMetadataV2].asOpt - } yield - Zarr3GroupHeader( - zarr_format, - node_type, - ngffMetadata, - ) - - override def writes(zarr3GroupHeader: Zarr3GroupHeader): JsValue = { - val groupHeaderBuilder = Json.newBuilder - groupHeaderBuilder ++= Seq( - "zarr_format" -> zarr3GroupHeader.zarr_format, - "node_type" -> zarr3GroupHeader.node_type, - ) - if (zarr3GroupHeader.ngffMetadata.isDefined) { - groupHeaderBuilder += ("attributes" -> Json.obj("ome" -> zarr3GroupHeader.ngffMetadata.get)) - } - groupHeaderBuilder.result() - } - } +object Zarr3GroupHeader { + implicit val jsonFormat: OFormat[Zarr3GroupHeader] = Json.format[Zarr3GroupHeader] } diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index ddf229896b3..db3339d4abb 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -47,12 +47,10 @@ GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/ GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) - GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) - # Segmentation mappings GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mappingName: String) GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 2126d134ae5..5e6f41beb67 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -11,7 +11,7 @@ import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesPa import com.scalableminds.webknossos.datastore.datareaders.zarr.{ NgffGroupHeader, NgffMetadata, - NgffMetadataV2, + NgffMetadataV0_5, ZarrHeader } import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ @@ -234,7 +234,7 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND - omeNgffHeader = NgffMetadataV2.fromNameVoxelSizeAndMags(tracingId, + omeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(tracingId, dataSourceVoxelSize = dataSource.scale, mags = existingMags.toList, additionalAxes = dataSource.additionalAxesUnion) @@ -273,19 +273,8 @@ class VolumeTracingZarrStreamingController @Inject()( magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - parsedCoordinates <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> Messages( + (x, y, z, additionalCoordinates) <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> Messages( "zarr.invalidChunkCoordinates") ~> NOT_FOUND - (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), - parsedCoordinates(parsedCoordinates.length - 2), - parsedCoordinates(parsedCoordinates.length - 1)) - additionalCoordinates = Some( - parsedCoordinates - .slice(1, parsedCoordinates.length - 3) - .zipWithIndex - .map(coordWithIndex => - new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) - .toList) - _ <- bool2Fox(parsedCoordinates.head == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND cubeSize = DataLayer.bucketLength wkRequest = WebknossosDataRequest( position = Vec3Int(x, y, z) * cubeSize * magParsed, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 0498ddd0ec2..a7b676b1839 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -39,7 +39,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])( implicit ec: ExecutionContext): Iterator[NamedStream] = { - val header = Zarr3ArrayHeader.fromDataLayerToVersion3(layer) + val header = Zarr3ArrayHeader.fromDataLayer(layer) bucketStream.flatMap { case (bucket, data) => val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) From b48ef862d6bea533dd1123875bd6113bcd91a0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 5 Aug 2024 18:21:40 +0200 Subject: [PATCH 15/22] remove unused imports and format code --- .../controllers/ZarrStreamingController.scala | 31 ++++++++++++------- .../zarr/ZarrCoordinatesParser.scala | 1 - .../datareaders/zarr/NgffMetadata.scala | 7 +---- .../zarr/SharedNgffMetadataAttributes.scala | 1 - .../datareaders/zarr3/Zarr3ArrayHeader.scala | 2 +- ...VolumeTracingZarrStreamingController.scala | 6 ++-- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 0b79561f2e1..c35a614f478 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -7,11 +7,20 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.AxisOrder -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, NgffMetadataV0_5, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV0_5, + ZarrHeader +} import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType, AnnotationSource} import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.models.requests.{Cuboid, DataServiceDataRequest, DataServiceRequestSettings} +import com.scalableminds.webknossos.datastore.models.requests.{ + Cuboid, + DataServiceDataRequest, + DataServiceRequestSettings +} import com.scalableminds.webknossos.datastore.models.{VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo @@ -72,9 +81,9 @@ class ZarrStreamingController @Inject()( dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND omeNgffHeaderV2 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions, - dataLayer.additionalAxes) + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) } yield Ok(Json.toJson(zarr3GroupHeader)) } @@ -126,9 +135,9 @@ class ZarrStreamingController @Inject()( dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND dataSourceOmeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions, - dataLayer.additionalAxes) + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) } yield Ok(Json.toJson(zarr3GroupHeader)) ) @@ -247,7 +256,7 @@ class ZarrStreamingController @Inject()( annotationSource.tracingStoreUrl, relevantToken) .map(Ok(_)), - orElse = (annotationSource) => + orElse = annotationSource => rawZarrCube(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag, coordinates) ) } @@ -341,7 +350,7 @@ class ZarrStreamingController @Inject()( remoteTracingstoreClient .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) .map(z => Ok(Json.toJson(z))), - orElse = (annotationSource) => + orElse = annotationSource => zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) ) } @@ -368,7 +377,7 @@ class ZarrStreamingController @Inject()( accessToken: String, dataLayerName: String, ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], - orElse: (AnnotationSource) => Fox[Result])(implicit request: Request[Any]): Fox[Result] = + orElse: AnnotationSource => Fox[Result])(implicit request: Request[Any]): Fox[Result] = for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index 1059afbb34d..6d7aa99cc22 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -2,7 +2,6 @@ package com.scalableminds.webknossos.datastore.dataformats.zarr import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} -import com.scalableminds.util.tools.JsonHelper.bool2Box import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate import play.api.http.Status.NOT_FOUND import play.api.i18n.{Messages, MessagesProvider} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala index defc321c901..2828e6a84a4 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala @@ -1,13 +1,9 @@ -package com.scalableminds.webknossos.datastore.datareaders.zarr; +package com.scalableminds.webknossos.datastore.datareaders.zarr import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.models.VoxelSize import play.api.libs.json.{Json, OFormat} - - - - case class NgffGroupHeader(zarr_format: Int) object NgffGroupHeader { implicit val jsonFormat: OFormat[NgffGroupHeader] = Json.format[NgffGroupHeader] @@ -65,4 +61,3 @@ object NgffLabelsGroup { implicit val jsonFormat: OFormat[NgffLabelsGroup] = Json.format[NgffLabelsGroup] val LABEL_PATH = "labels/.zattrs" } - diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala index 56e2a3a9fb8..7b72f80e077 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala @@ -52,4 +52,3 @@ case class NgffChannelAttributes(color: Option[String], object NgffChannelAttributes { implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] } - diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 005c3a1d99f..1f03b831649 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -251,7 +251,7 @@ object Zarr3ArrayHeader extends JsonImplicits { ) } - def fromDataLayer (dataLayer: DataLayer): Zarr3ArrayHeader = { + def fromDataLayer(dataLayer: DataLayer): Zarr3ArrayHeader = { val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) Zarr3ArrayHeader( zarr_format = 3, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index ec994fdb54b..d0cac02c240 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -235,9 +235,9 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(tracingId, - dataSourceVoxelSize = dataSource.scale, - mags = existingMags.toList, - additionalAxes = dataSource.additionalAxesUnion) + dataSourceVoxelSize = dataSource.scale, + mags = existingMags.toList, + additionalAxes = dataSource.additionalAxesUnion) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeader)) } yield Ok(Json.toJson(zarr3GroupHeader)) } From 01abdf9e039684b6151f6db3be91c7515ef6d166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 6 Aug 2024 18:18:21 +0200 Subject: [PATCH 16/22] add datasource-properties.json and dir listing routes to zarr 3 streaming --- .../controllers/ZarrStreamingController.scala | 99 ++++++++++++------- .../dataformats/layers/ZarrDataLayers.scala | 10 +- .../datastore/explore/NgffExplorer.scala | 7 +- .../datastore/explore/ZarrArrayExplorer.scala | 17 +++- .../services/DSRemoteTracingstoreClient.scala | 12 ++- ....scalableminds.webknossos.datastore.routes | 47 ++++++--- ...VolumeTracingZarrStreamingController.scala | 40 +++++--- ...alableminds.webknossos.tracingstore.routes | 23 +++-- 8 files changed, 174 insertions(+), 81 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index c35a614f478..983428f6999 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -151,13 +151,14 @@ class ZarrStreamingController @Inject()( token: Option[String], organizationName: String, datasetName: String, + zarrVersion: Int, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ~> NOT_FOUND dataLayers = dataSource.dataLayers - zarrLayers = dataLayers.map(convertLayerToZarrLayer) + zarrLayers = dataLayers.map(convertLayerToZarrLayer(_, zarrVersion)) zarrSource = GenericDataSource[DataLayer](dataSource.id, zarrLayers, dataSource.scale) zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) } yield Ok(Json.toJson(zarrSourceJson)) @@ -177,7 +178,8 @@ class ZarrStreamingController @Inject()( } } - private def convertLayerToZarrLayer(layer: DataLayer): ZarrLayer = + private def convertLayerToZarrLayer(layer: DataLayer, zarrVersion: Int): ZarrLayer = { + val dataFormat = if (zarrVersion == 2) DataFormat.zarr else DataFormat.zarr3 layer match { case s: SegmentationLayer => ZarrSegmentationLayer( @@ -187,7 +189,8 @@ class ZarrStreamingController @Inject()( s.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), mappings = s.mappings, largestSegmentId = s.largestSegmentId, - numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1) + numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1), + dataFormat = dataFormat ) case d: DataLayer => ZarrDataLayer( @@ -197,11 +200,15 @@ class ZarrStreamingController @Inject()( d.elementClass, d.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), numChannels = Some(if (d.elementClass == ElementClass.uint24) 3 else 1), - additionalAxes = None + additionalAxes = None, + dataFormat = dataFormat ) } + } - def dataSourceWithAnnotationPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + def dataSourceWithAnnotationPrivateLink(token: Option[String], + accessToken: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -213,7 +220,7 @@ class ZarrStreamingController @Inject()( .toFox ~> NOT_FOUND dataSourceLayers = dataSource.dataLayers .filter(dL => !volumeAnnotationLayers.exists(_.name == dL.name)) - .map(convertLayerToZarrLayer) + .map(convertLayerToZarrLayer(_, zarrVersion)) annotationLayers <- Fox.serialCombined(volumeAnnotationLayers)( l => remoteTracingstoreClient @@ -350,8 +357,8 @@ class ZarrStreamingController @Inject()( remoteTracingstoreClient .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) .map(z => Ok(Json.toJson(z))), - orElse = annotationSource => - zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + orElse = + annotationSource => zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) ) } @@ -372,12 +379,11 @@ class ZarrStreamingController @Inject()( ) } - def ifIsAnnotationLayerOrElse( - token: Option[String], - accessToken: String, - dataLayerName: String, - ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], - orElse: AnnotationSource => Fox[Result])(implicit request: Request[Any]): Fox[Result] = + def ifIsAnnotationLayerOrElse(token: Option[String], + accessToken: String, + dataLayerName: String, + ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], + orElse: AnnotationSource => Fox[Result])(implicit request: Request[Any]): Fox[Result] = for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) @@ -393,34 +399,39 @@ class ZarrStreamingController @Inject()( organizationName: String, datasetName: String, dataLayerName: String, - mag: String): Action[AnyContent] = + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag) + dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag, zarrVersion) } } private def dataLayerMagFolderContents(organizationName: String, datasetName: String, dataLayerName: String, - mag: String)(implicit m: MessagesProvider): Fox[Result] = + mag: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + additionalEntries = if (zarrVersion == 2) List(ZarrHeader.FILENAME_DOT_ZARRAY) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", "%s/%s/%s/%s".format(organizationName, datasetName, dataLayerName, mag), - List(".zarray") + additionalEntries )).withHeaders() def dataLayerMagFolderContentsPrivateLink(token: Option[String], accessToken: String, dataLayerName: String, - mag: String): Action[AnyContent] = + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( token, @@ -431,7 +442,8 @@ class ZarrStreamingController @Inject()( .getDataLayerMagFolderContents(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, - relevantToken) + relevantToken, + zarrVersion) .map( layers => Ok( @@ -444,37 +456,45 @@ class ZarrStreamingController @Inject()( dataLayerMagFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, - mag) + mag, + zarrVersion) ) } def requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, - dataLayerName: String): Action[AnyContent] = Action.async { implicit request => + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - dataLayerFolderContents(organizationName, datasetName, dataLayerName) + dataLayerFolderContents(organizationName, datasetName, dataLayerName, zarrVersion) } } - private def dataLayerFolderContents(organizationName: String, datasetName: String, dataLayerName: String)( - implicit m: MessagesProvider): Fox[Result] = + private def dataLayerFolderContents(organizationName: String, + datasetName: String, + dataLayerName: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND mags = dataLayer.resolutions + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", "%s/%s/%s".format(organizationName, datasetName, dataLayerName), - List(".zattrs", ".zgroup") ++ mags.map(_.toMagLiteral(allowScalar = true)) + additionalFiles ++ mags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() def dataLayerFolderContentsPrivateLink(token: Option[String], accessToken: String, - dataLayerName: String): Action[AnyContent] = + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( token, @@ -482,7 +502,10 @@ class ZarrStreamingController @Inject()( dataLayerName, ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .getDataLayerFolderContents(annotationLayer.tracingId, + annotationSource.tracingStoreUrl, + relevantToken, + zarrVersion) .map( layers => Ok( @@ -492,13 +515,17 @@ class ZarrStreamingController @Inject()( layers )).withHeaders()), orElse = annotationSource => - dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) + dataLayerFolderContents(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + zarrVersion) ) } def requestDataSourceFolderContents(token: Option[String], organizationName: String, - datasetName: String): Action[AnyContent] = + datasetName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { @@ -506,17 +533,20 @@ class ZarrStreamingController @Inject()( dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name) + additionalVersionDependantFiles = if (zarrVersion == 2) List(NgffGroupHeader.FILENAME_DOT_ZGROUP) else List.empty } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", s"$organizationName/$datasetName", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) ++ additionalVersionDependantFiles ++ layerNames )) } } - def dataSourceFolderContentsPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + def dataSourceFolderContentsPrivateLink(token: Option[String], + accessToken: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) @@ -528,12 +558,15 @@ class ZarrStreamingController @Inject()( .map((dataLayer: DataLayer) => dataLayer.name) .filter(!annotationLayerNames.contains(_)) layerNames = annotationLayerNames ++ dataSourceLayerNames + additionalEntries = if (zarrVersion == 2) List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Combined datastore and tracingstore directory", s"$accessToken", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + additionalEntries ++ layerNames )) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala index 1ebc67f223b..17810f5efb3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala @@ -4,15 +4,13 @@ import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.webknossos.datastore.dataformats.{DatasetArrayBucketProvider, MagLocator} import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration -import com.scalableminds.webknossos.datastore.models.datasource._ +import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, _} import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService import play.api.libs.json.{Json, OFormat} import ucar.ma2.{Array => MultiArray} trait ZarrLayer extends DataLayerWithMagLocators { - val dataFormat: DataFormat.Value = DataFormat.zarr - def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]) = @@ -36,7 +34,8 @@ case class ZarrDataLayer( adminViewConfiguration: Option[LayerViewConfiguration] = None, coordinateTransformations: Option[List[CoordinateTransformation]] = None, override val numChannels: Option[Int] = Some(1), - override val additionalAxes: Option[Seq[AdditionalAxis]] + override val additionalAxes: Option[Seq[AdditionalAxis]], + override val dataFormat: DataFormat.Value, ) extends ZarrLayer object ZarrDataLayer { @@ -54,7 +53,8 @@ case class ZarrSegmentationLayer( adminViewConfiguration: Option[LayerViewConfiguration] = None, coordinateTransformations: Option[List[CoordinateTransformation]] = None, override val numChannels: Option[Int] = Some(1), - additionalAxes: Option[Seq[AdditionalAxis]] = None + additionalAxes: Option[Seq[AdditionalAxis]] = None, + override val dataFormat: DataFormat.Value, ) extends SegmentationLayer with ZarrLayer diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala index 78f4c850f90..96ad60754b9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala @@ -14,6 +14,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfigu import com.scalableminds.webknossos.datastore.models.datasource.{ AdditionalAxis, Category, + DataFormat, ElementClass, LayerViewConfiguration } @@ -123,7 +124,8 @@ class NgffExplorer(implicit val ec: ExecutionContext) extends RemoteLayerExplore magsWithAttributes.map(_.mag), largestSegmentId = None, additionalAxes = Some(additionalAxes), - defaultViewConfiguration = Some(viewConfig) + defaultViewConfiguration = Some(viewConfig), + dataFormat = DataFormat.zarr ) } else ZarrDataLayer( @@ -133,7 +135,8 @@ class NgffExplorer(implicit val ec: ExecutionContext) extends RemoteLayerExplore elementClass, magsWithAttributes.map(_.mag), additionalAxes = Some(additionalAxes), - defaultViewConfiguration = Some(viewConfig) + defaultViewConfiguration = Some(viewConfig), + dataFormat = DataFormat.zarr ) } yield (layer, VoxelSize(voxelSizeFactor, unifiedAxisUnit)) }) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala index 416915ed398..11f83904b4b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala @@ -8,7 +8,7 @@ import com.scalableminds.webknossos.datastore.datareaders.AxisOrder import com.scalableminds.webknossos.datastore.datareaders.zarr.ZarrHeader import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.VoxelSize -import com.scalableminds.webknossos.datastore.models.datasource.Category +import com.scalableminds.webknossos.datastore.models.datasource.{Category, DataFormat} import scala.concurrent.ExecutionContext @@ -26,9 +26,20 @@ class ZarrArrayExplorer(mag: Vec3Int = Vec3Int.ones)(implicit val ec: ExecutionC boundingBox <- zarrHeader.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format" magLocator = MagLocator(mag, Some(remotePath.toUri.toString), None, Some(guessedAxisOrder), None, credentialId) layer: ZarrLayer = if (looksLikeSegmentationLayer(name, elementClass)) { - ZarrSegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None) + ZarrSegmentationLayer(name, + boundingBox, + elementClass, + List(magLocator), + largestSegmentId = None, + dataFormat = DataFormat.zarr) } else - ZarrDataLayer(name, Category.color, boundingBox, elementClass, List(magLocator), additionalAxes = None) + ZarrDataLayer(name, + Category.color, + boundingBox, + elementClass, + List(magLocator), + additionalAxes = None, + dataFormat = DataFormat.zarr) } yield List((layer, VoxelSize.fromFactorWithDefaultUnit(Vec3Double.ones))) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 643c7674e5d..2bdf9dc90cd 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -25,6 +25,8 @@ class DSRemoteTracingstoreClient @Inject()( val lifecycle: ApplicationLifecycle, ) extends LazyLogging with FoxImplicits { + private def getZarrVersionDependantSubPath = (zarrVersion: Int) => if(zarrVersion == 2) "zarr" else "zarr3_experimental" + def getZArray(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[ZarrHeader] = rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/.zarray") .addQueryStringOptional("token", token) @@ -71,13 +73,15 @@ class DSRemoteTracingstoreClient @Inject()( def getDataLayerMagFolderContents(tracingId: String, mag: String, tracingStoreUri: String, - token: Option[String]): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/json/$tracingId/$mag") + token: Option[String], + zarrVersion: Int): Fox[List[String]] = { + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] + } - def getDataLayerFolderContents(tracingId: String, tracingStoreUri: String, token: Option[String]): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/json/$tracingId") + def getDataLayerFolderContents(tracingId: String, tracingStoreUri: String, token: Option[String], zarrVersion: Int): Fox[List[String]] = + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index d8f11d8f5c3..3181a8fc4e5 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -16,38 +16,53 @@ GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/his GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) # Zarr2 compatible routes -GET /zarr/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) -GET /zarr/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) +GET /zarr/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationName: String, datasetName: String, dataLayerName="") -GET /zarr/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) +GET /zarr/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) GET /zarr/:organizationName/:datasetName/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, cxyz: String) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) -GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String) -GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String) +GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName="") -GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, cxyz: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Zarr3 compatible routes +GET /zarr3_experimental/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +# GET /zarr3_experimental/:organizationName/:datasetName/ TODO M ask whether there is a zarr.json on this level +GET /zarr3_experimental/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) +GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index d0cac02c240..5271c743dfa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -28,7 +28,7 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} -import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer, ElementClass} +import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataFormat, DataLayer, ElementClass} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService @@ -55,33 +55,42 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(token: Option[String], tracingId: String): Action[AnyContent] = + def volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Tracingstore", "%s".format(tracingId), - List(".zattrs", ".zgroup") ++ existingMags.map(_.toMagLiteral(allowScalar = true)) + additionalFiles ++ existingMags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() } } - def volumeTracingFolderContentJson(token: Option[String], tracingId: String): Action[AnyContent] = + def volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) - } yield Ok(Json.toJson(List(".zattrs", ".zgroup") ++ existingMags)) + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) + } yield Ok(Json.toJson(additionalFiles ++ existingMags)) } } - def volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + def volumeTracingMagFolderContent(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -90,17 +99,21 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + files = if (zarrVersion == 2) List(".zarray") else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Tracingstore", "%s".format(tracingId), - List(".zarray") + files )).withHeaders() } } - def volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + def volumeTracingMagFolderContentJson(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -109,7 +122,8 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - } yield Ok(Json.toJson(List(".zarray"))) + files = if (zarrVersion == 2) List(".zarray") else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) + } yield Ok(Json.toJson(files)) } } @@ -243,7 +257,10 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]): Action[AnyContent] = + def zarrSource(token: Option[String], + tracingId: String, + tracingName: Option[String], + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -256,7 +273,8 @@ class VolumeTracingZarrStreamingController @Inject()( elementClass = tracing.elementClass, mags = tracing.resolutions.toList.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), mappings = None, - numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1) + numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1), + dataFormat = if (zarrVersion == 2) DataFormat.zarr else DataFormat.zarr3 ) } yield Ok(Json.toJson(zarrLayer)) } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index e79014fcba4..fae27dfb475 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -40,19 +40,28 @@ GET /mapping/:tracingId/segmentsForAgglomerate @com.scalablemin POST /mapping/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdsForSegments(token: Option[String], tracingId: String) # Zarr endpoints for volume annotations -GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String) -GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) +# Zarr version 2 +GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], tracingId: String) GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]) -GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) +# Zarr version 3 +GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) +GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) From f7e5426973e27ad31811282aecd7b1390ed8a4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 6 Aug 2024 18:30:05 +0200 Subject: [PATCH 17/22] format code --- .../controllers/ZarrStreamingController.scala | 6 ++++-- .../services/DSRemoteTracingstoreClient.scala | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 983428f6999..9bab5b70a64 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -533,7 +533,8 @@ class ZarrStreamingController @Inject()( dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name) - additionalVersionDependantFiles = if (zarrVersion == 2) List(NgffGroupHeader.FILENAME_DOT_ZGROUP) else List.empty + additionalVersionDependantFiles = if (zarrVersion == 2) List(NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List.empty } yield Ok( views.html.datastoreZarrDatasourceDir( @@ -558,7 +559,8 @@ class ZarrStreamingController @Inject()( .map((dataLayer: DataLayer) => dataLayer.name) .filter(!annotationLayerNames.contains(_)) layerNames = annotationLayerNames ++ dataSourceLayerNames - additionalEntries = if (zarrVersion == 2) List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, NgffGroupHeader.FILENAME_DOT_ZGROUP) + additionalEntries = if (zarrVersion == 2) + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, NgffGroupHeader.FILENAME_DOT_ZGROUP) else List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) } yield diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 2bdf9dc90cd..4d044540f3c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -25,7 +25,8 @@ class DSRemoteTracingstoreClient @Inject()( val lifecycle: ApplicationLifecycle, ) extends LazyLogging with FoxImplicits { - private def getZarrVersionDependantSubPath = (zarrVersion: Int) => if(zarrVersion == 2) "zarr" else "zarr3_experimental" + private def getZarrVersionDependantSubPath = + (zarrVersion: Int) => if (zarrVersion == 2) "zarr" else "zarr3_experimental" def getZArray(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[ZarrHeader] = rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/.zarray") @@ -74,13 +75,15 @@ class DSRemoteTracingstoreClient @Inject()( mag: String, tracingStoreUri: String, token: Option[String], - zarrVersion: Int): Fox[List[String]] = { + zarrVersion: Int): Fox[List[String]] = rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] - } - def getDataLayerFolderContents(tracingId: String, tracingStoreUri: String, token: Option[String], zarrVersion: Int): Fox[List[String]] = + def getDataLayerFolderContents(tracingId: String, + tracingStoreUri: String, + token: Option[String], + zarrVersion: Int): Fox[List[String]] = rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] From 56aa0f087207f8184f6d532427f1d26f75f156ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 13 Aug 2024 09:53:21 +0200 Subject: [PATCH 18/22] add changelog entry --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 619b3a34d95..b07bf97fb4f 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -19,6 +19,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added option to expand or collapse all subgroups of a segment group in the segments tab. [#7911](https://github.com/scalableminds/webknossos/pull/7911) - The context menu that is opened upon right-clicking a segment in the dataview port now contains the segment's name. [#7920](https://github.com/scalableminds/webknossos/pull/7920) - Upgraded backend dependencies for improved performance and stability. [#7922](https://github.com/scalableminds/webknossos/pull/7922) +- Added Support for streaming datasets via Zarr version 3. [#7941](https://github.com/scalableminds/webknossos/pull/7941) - It is now saved whether segment groups are collapsed or expanded, so this information doesn't get lost e.g. upon page reload. [#7928](https://github.com/scalableminds/webknossos/pull/7928/) - It is now saved whether skeleton groups are collapsed or expanded. This information is also persisted to NML output. [#7939](https://github.com/scalableminds/webknossos/pull/7939) - The context menu entry "Focus in Segment List" expands all necessary segment groups in the segments tab to show the highlighted segment. [#7950](https://github.com/scalableminds/webknossos/pull/7950) From 9fc06f32eac28341c36f2f9992c466f0d6717096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 13 Aug 2024 11:13:40 +0200 Subject: [PATCH 19/22] fix Zarr3GroupHeader json serialization format --- .../controllers/ZarrStreamingController.scala | 4 ++-- .../datareaders/zarr/NgffMetadataV0_5.scala | 2 +- .../datareaders/zarr3/Zarr3ArrayHeader.scala | 2 +- .../datareaders/zarr3/Zarr3GroupHeader.scala | 21 ++++++++++++++++++- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 9bab5b70a64..2b4279a3a85 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -80,11 +80,11 @@ class ZarrStreamingController @Inject()( datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND - omeNgffHeaderV2 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, + omeNgffHeaderV0_5 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, dataSource.scale, dataLayer.resolutions, dataLayer.additionalAxes) - zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV2)) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV0_5)) } yield Ok(Json.toJson(zarr3GroupHeader)) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala index 1d4841c281e..4469e2c21bc 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala @@ -31,7 +31,7 @@ object NgffMetadataV0_5 { dataSourceVoxelSize: VoxelSize, mags: List[Vec3Int], additionalAxes: Option[Seq[AdditionalAxis]], - version: String = "0.5"): NgffMetadataV0_5 = { + version: String = "0.5-dev2"): NgffMetadataV0_5 = { val datasets = mags.map( mag => NgffDataset( diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 1f03b831649..39f75300b27 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -241,7 +241,7 @@ object Zarr3ArrayHeader extends JsonImplicits { ChunkGridConfiguration(Array(1, 1, 1))))), // Extension not supported for now "chunk_key_encoding" -> zarrArrayHeader.chunk_key_encoding, "fill_value" -> zarrArrayHeader.fill_value, - "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map("" -> ""))), + "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map.empty)), "codecs" -> zarrArrayHeader.codecs.map { codec: CodecConfiguration => val configurationJson = if (codec.includeConfiguration) Json.obj("configuration" -> codec) else Json.obj() Json.obj("name" -> codec.name) ++ configurationJson diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala index affa4837535..a2a0403120a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -10,5 +10,24 @@ case class Zarr3GroupHeader( ) object Zarr3GroupHeader { - implicit val jsonFormat: OFormat[Zarr3GroupHeader] = Json.format[Zarr3GroupHeader] + implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { + override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = + for { + zarr_format <- (json \ "zarr_format").validate[Int] + node_type <- (json \ "node_type").validate[String] + ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5] + } yield + Zarr3GroupHeader( + zarr_format, + node_type, + ngffMetadata, + ) + + override def writes(zarrArrayGroup: Zarr3GroupHeader): JsValue = + Json.obj( + "zarr_format" -> zarrArrayGroup.zarr_format, + "node_type" -> zarrArrayGroup.node_type, + "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata), + ) + } } From e785efcda58aa4261e80ce2eed36eb17ef0a2f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 13 Aug 2024 11:34:46 +0200 Subject: [PATCH 20/22] fix backend format --- .../datastore/controllers/ZarrStreamingController.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 2b4279a3a85..46782237eec 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -81,9 +81,9 @@ class ZarrStreamingController @Inject()( dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND omeNgffHeaderV0_5 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions, - dataLayer.additionalAxes) + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV0_5)) } yield Ok(Json.toJson(zarr3GroupHeader)) } From 8023bf8821c91af1eb873a390ed3b51e48d6ef43 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Tue, 13 Aug 2024 16:07:34 +0200 Subject: [PATCH 21/22] Update webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala --- .../datastore/datareaders/zarr/NgffMetadataV0_5.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala index 4469e2c21bc..1d4841c281e 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala @@ -31,7 +31,7 @@ object NgffMetadataV0_5 { dataSourceVoxelSize: VoxelSize, mags: List[Vec3Int], additionalAxes: Option[Seq[AdditionalAxis]], - version: String = "0.5-dev2"): NgffMetadataV0_5 = { + version: String = "0.5"): NgffMetadataV0_5 = { val datasets = mags.map( mag => NgffDataset( From b04195afdd1a96546e3255497d74113ee73ac112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 14 Aug 2024 14:48:55 +0200 Subject: [PATCH 22/22] add comments about why manual json serializer is needed --- .../datastore/datareaders/zarr3/Zarr3GroupHeader.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala index a2a0403120a..396f830288b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -15,6 +15,7 @@ object Zarr3GroupHeader { for { zarr_format <- (json \ "zarr_format").validate[Int] node_type <- (json \ "node_type").validate[String] + // Read the metadata from the correct json path. ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5] } yield Zarr3GroupHeader( @@ -27,6 +28,7 @@ object Zarr3GroupHeader { Json.obj( "zarr_format" -> zarrArrayGroup.zarr_format, "node_type" -> zarrArrayGroup.node_type, + // Enforce correct path for ngffMetadata in the json. "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata), ) }