diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index adae2ceae1d..913438ac8d0 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -19,6 +19,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added option to expand or collapse all subgroups of a segment group in the segments tab. [#7911](https://github.com/scalableminds/webknossos/pull/7911) - The context menu that is opened upon right-clicking a segment in the dataview port now contains the segment's name. [#7920](https://github.com/scalableminds/webknossos/pull/7920) - Upgraded backend dependencies for improved performance and stability. [#7922](https://github.com/scalableminds/webknossos/pull/7922) +- Added Support for streaming datasets via Zarr version 3. [#7941](https://github.com/scalableminds/webknossos/pull/7941) - It is now saved whether segment groups are collapsed or expanded, so this information doesn't get lost e.g. upon page reload. [#7928](https://github.com/scalableminds/webknossos/pull/7928/) - It is now saved whether skeleton groups are collapsed or expanded. This information is also persisted to NML output. [#7939](https://github.com/scalableminds/webknossos/pull/7939) - The context menu entry "Focus in Segment List" expands all necessary segment groups in the segments tab to show the highlighted segment. [#7950](https://github.com/scalableminds/webknossos/pull/7950) diff --git a/conf/messages b/conf/messages index 4be8d656088..f78b1b5c99e 100644 --- a/conf/messages +++ b/conf/messages @@ -138,9 +138,10 @@ dataLayer.mustBeSegmentation=DataLayer “{0}” is not a segmentation layer dataLayer.wrongMag=DataLayer “{0}” does not have mag “{1}” dataLayer.invalidMag=Supplied “{0}” is not a valid mag format. Please use “x-y-z” -zarr.invalidChunkCoordinates=The requested chunk coordinates are in an invalid format. Expected c.x.y.z +zarr.invalidChunkCoordinates=Invalid chunk coordinates. Expected dot separated coordinates with a prefix of “c.”: c.x.y.z zarr.invalidFirstChunkCoord="First Channel must be 0" zarr.chunkNotFound=Could not find the requested chunk +zarr.notEnoughCoordinates=Invalid number of chunk coordinates. Expected to get at least 3 dimensions and channel 0. nml.file.uploadSuccess=Successfully uploaded file nml.file.notFound=Could not extract NML file diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index b6ef82ef9fc..46782237eec 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -7,15 +7,21 @@ import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.{ZarrDataLayer, ZarrLayer, ZarrSegmentationLayer} import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser import com.scalableminds.webknossos.datastore.datareaders.AxisOrder -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} -import com.scalableminds.webknossos.datastore.models.{VoxelPosition, VoxelSize} -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV0_5, + ZarrHeader +} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} +import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType, AnnotationSource} import com.scalableminds.webknossos.datastore.models.datasource._ import com.scalableminds.webknossos.datastore.models.requests.{ Cuboid, DataServiceDataRequest, DataServiceRequestSettings } +import com.scalableminds.webknossos.datastore.models.{VoxelPosition, VoxelSize} import com.scalableminds.webknossos.datastore.services._ import net.liftweb.common.Box.tryo import play.api.i18n.{Messages, MessagesProvider} @@ -61,30 +67,80 @@ class ZarrStreamingController @Inject()( } } + def requestZarrJson( + token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String = "", + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, + datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + omeNgffHeaderV0_5 = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeaderV0_5)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + } + } + def zAttrsWithAnnotationPrivateLink(token: Option[String], accessToken: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - annotationLayer = annotationSource.getAnnotationLayer(dataLayerName) - omeNgffHeader <- annotationLayer match { - case Some(layer) => - remoteTracingstoreClient.getOmeNgffHeader(layer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - case None => - for { - (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer( - annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName) ?~> Messages("dataSource.notFound") ~> NOT_FOUND - dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, - dataSource.scale, - dataLayer.resolutions) - } yield dataSourceOmeNgffHeader - } - } yield Ok(Json.toJson(omeNgffHeader)) + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + remoteTracingstoreClient + .getOmeNgffHeader(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(ngffMetadata => Ok(Json.toJson(ngffMetadata))) + }, + orElse = annotationSource => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions) + } yield Ok(Json.toJson(dataSourceOmeNgffHeader)) + ) + } + + def zarrJsonWithAnnotationPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String = ""): Action[AnyContent] = + Action.async { implicit request => + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + remoteTracingstoreClient + .getZarrJsonGroupHeaderWithNgff(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(header => Ok(Json.toJson(header))) + }, + orElse = annotationSource => + for { + (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + dataSourceOmeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(dataLayerName, + dataSource.scale, + dataLayer.resolutions, + dataLayer.additionalAxes) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(dataSourceOmeNgffHeader)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + ) } /** @@ -95,13 +151,14 @@ class ZarrStreamingController @Inject()( token: Option[String], organizationName: String, datasetName: String, + zarrVersion: Int, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ~> NOT_FOUND dataLayers = dataSource.dataLayers - zarrLayers = dataLayers.map(convertLayerToZarrLayer) + zarrLayers = dataLayers.map(convertLayerToZarrLayer(_, zarrVersion)) zarrSource = GenericDataSource[DataLayer](dataSource.id, zarrLayers, dataSource.scale) zarrSourceJson <- replaceVoxelSizeByLegacyFormat(Json.toJson(zarrSource)) } yield Ok(Json.toJson(zarrSourceJson)) @@ -121,7 +178,8 @@ class ZarrStreamingController @Inject()( } } - private def convertLayerToZarrLayer(layer: DataLayer): ZarrLayer = + private def convertLayerToZarrLayer(layer: DataLayer, zarrVersion: Int): ZarrLayer = { + val dataFormat = if (zarrVersion == 2) DataFormat.zarr else DataFormat.zarr3 layer match { case s: SegmentationLayer => ZarrSegmentationLayer( @@ -131,7 +189,8 @@ class ZarrStreamingController @Inject()( s.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), mappings = s.mappings, largestSegmentId = s.largestSegmentId, - numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1) + numChannels = Some(if (s.elementClass == ElementClass.uint24) 3 else 1), + dataFormat = dataFormat ) case d: DataLayer => ZarrDataLayer( @@ -141,11 +200,15 @@ class ZarrStreamingController @Inject()( d.elementClass, d.resolutions.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), numChannels = Some(if (d.elementClass == ElementClass.uint24) 3 else 1), - additionalAxes = None + additionalAxes = None, + dataFormat = dataFormat ) } + } - def dataSourceWithAnnotationPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + def dataSourceWithAnnotationPrivateLink(token: Option[String], + accessToken: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND @@ -157,7 +220,7 @@ class ZarrStreamingController @Inject()( .toFox ~> NOT_FOUND dataSourceLayers = dataSource.dataLayers .filter(dL => !volumeAnnotationLayers.exists(_.name == dL.name)) - .map(convertLayerToZarrLayer) + .map(convertLayerToZarrLayer(_, zarrVersion)) annotationLayers <- Fox.serialCombined(volumeAnnotationLayers)( l => remoteTracingstoreClient @@ -174,11 +237,11 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String, mag: String, - cxyz: String, + coordinates: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - rawZarrCube(organizationName, datasetName, dataLayerName, mag, cxyz) + rawZarrCube(organizationName, datasetName, dataLayerName, mag, coordinates) } } @@ -186,24 +249,23 @@ class ZarrStreamingController @Inject()( accessToken: String, dataLayerName: String, mag: String, - cxyz: String): Action[AnyContent] = + coordinates: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - - // ensures access to volume layers if fallback layer with equal name exists - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getRawZarrCube(annotationLayer.tracingId, mag, cxyz, annotationSource.tracingStoreUrl, relevantToken) - .map(Ok(_)) - case None => - rawZarrCube(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag, cxyz) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getRawZarrCube(annotationLayer.tracingId, + mag, + coordinates, + annotationSource.tracingStoreUrl, + relevantToken) + .map(Ok(_)), + orElse = annotationSource => + rawZarrCube(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag, coordinates) + ) } private def rawZarrCube( @@ -211,16 +273,15 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String, mag: String, - cxyz: String, + coordinates: String, )(implicit m: MessagesProvider): Fox[Result] = for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND - (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND + (x, y, z, additionalCoordinates) <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> "zarr.invalidChunkCoordinates" ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND - _ <- bool2Fox(c == 0) ~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND cubeSize = DataLayer.bucketLength request = DataServiceDataRequest( dataSource, @@ -234,7 +295,7 @@ class ZarrStreamingController @Inject()( height = cubeSize, depth = cubeSize ), - DataServiceRequestSettings(halfByte = false) + DataServiceRequestSettings(halfByte = false, additionalCoordinates = additionalCoordinates) ) (data, notFoundIndices) <- binaryDataService.handleDataRequests(List(request)) _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND @@ -262,142 +323,209 @@ class ZarrStreamingController @Inject()( zarrHeader = ZarrHeader.fromLayer(dataLayer, magParsed) } yield Ok(Json.toJson(zarrHeader)) + def requestZarrJsonForMag(token: Option[String], + organizationName: String, + datasetName: String, + dataLayerName: String, + mag: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), + urlOrHeaderToken(token, request)) { + zarrJsonForMag(organizationName, datasetName, dataLayerName, mag) + } + } + + private def zarrJsonForMag(organizationName: String, datasetName: String, dataLayerName: String, mag: String)( + implicit m: MessagesProvider): Fox[Result] = + for { + (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( + "dataSource.notFound") ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + zarrHeader = Zarr3ArrayHeader.fromDataLayer(dataLayer) + } yield Ok(Json.toJson(zarrHeader)) + def zArrayPrivateLink(token: Option[String], accessToken: String, dataLayerName: String, mag: String): Action[AnyContent] = Action.async { implicit request => + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))), + orElse = + annotationSource => zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + ) + } + + def zarrJsonPrivateLink(token: Option[String], + accessToken: String, + dataLayerName: String, + mag: String): Action[AnyContent] = Action.async { implicit request => + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .map(z => Ok(Json.toJson(z))), + orElse = annotationSource => + zarrJsonForMag(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + ) + } + + def ifIsAnnotationLayerOrElse(token: Option[String], + accessToken: String, + dataLayerName: String, + ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], + orElse: AnnotationSource => Fox[Result])(implicit request: Request[Any]): Fox[Result] = for { - annotationSource <- remoteWebknossosClient - .getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) else urlOrHeaderToken(token, request) + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND + relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) + else urlOrHeaderToken(token, request) layer = annotationSource.getAnnotationLayer(dataLayerName) result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) - .map(z => Ok(Json.toJson(z))) - case None => - zArray(annotationSource.organizationName, annotationSource.datasetName, dataLayerName, mag) + case Some(annotationLayer) => ifIsAnnotationLayer(annotationLayer, annotationSource, relevantToken) + case None => orElse(annotationSource) } } yield result - } def requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, - mag: String): Action[AnyContent] = + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag) + dataLayerMagFolderContents(organizationName, datasetName, dataLayerName, mag, zarrVersion) } } private def dataLayerMagFolderContents(organizationName: String, datasetName: String, dataLayerName: String, - mag: String)(implicit m: MessagesProvider): Fox[Result] = + mag: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ~> NOT_FOUND magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(dataLayer.containsResolution(magParsed)) ?~> Messages("dataLayer.wrongMag", dataLayerName, mag) ~> NOT_FOUND + additionalEntries = if (zarrVersion == 2) List(ZarrHeader.FILENAME_DOT_ZARRAY) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", "%s/%s/%s/%s".format(organizationName, datasetName, dataLayerName, mag), - List(".zarray") + additionalEntries )).withHeaders() def dataLayerMagFolderContentsPrivateLink(token: Option[String], accessToken: String, dataLayerName: String, - mag: String): Action[AnyContent] = + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - layer = annotationSource.getAnnotationLayer(dataLayerName) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerMagFolderContents(annotationLayer.tracingId, - mag, - annotationSource.tracingStoreUrl, - relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Combined Annotation Route", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerMagFolderContents(annotationSource.organizationName, - annotationSource.datasetName, - dataLayerName, - mag) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getDataLayerMagFolderContents(annotationLayer.tracingId, + mag, + annotationSource.tracingStoreUrl, + relevantToken, + zarrVersion) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Combined Annotation Route", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()), + orElse = annotationSource => + dataLayerMagFolderContents(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + mag, + zarrVersion) + ) } def requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, - dataLayerName: String): Action[AnyContent] = Action.async { implicit request => + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { - dataLayerFolderContents(organizationName, datasetName, dataLayerName) + dataLayerFolderContents(organizationName, datasetName, dataLayerName, zarrVersion) } } - private def dataLayerFolderContents(organizationName: String, datasetName: String, dataLayerName: String)( - implicit m: MessagesProvider): Fox[Result] = + private def dataLayerFolderContents(organizationName: String, + datasetName: String, + dataLayerName: String, + zarrVersion: Int)(implicit m: MessagesProvider): Fox[Result] = for { (_, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationName, datasetName, dataLayerName) ?~> Messages( "dataSource.notFound") ~> NOT_FOUND mags = dataLayer.resolutions + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", "%s/%s/%s".format(organizationName, datasetName, dataLayerName), - List(".zattrs", ".zgroup") ++ mags.map(_.toMagLiteral(allowScalar = true)) + additionalFiles ++ mags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() def dataLayerFolderContentsPrivateLink(token: Option[String], accessToken: String, - dataLayerName: String): Action[AnyContent] = + dataLayerName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map( - layers => - Ok( - views.html.datastoreZarrDatasourceDir( - "Tracingstore", - s"${annotationLayer.tracingId}", - layers - )).withHeaders()) - case None => - dataLayerFolderContents(annotationSource.organizationName, annotationSource.datasetName, dataLayerName) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getDataLayerFolderContents(annotationLayer.tracingId, + annotationSource.tracingStoreUrl, + relevantToken, + zarrVersion) + .map( + layers => + Ok( + views.html.datastoreZarrDatasourceDir( + "Tracingstore", + s"${annotationLayer.tracingId}", + layers + )).withHeaders()), + orElse = annotationSource => + dataLayerFolderContents(annotationSource.organizationName, + annotationSource.datasetName, + dataLayerName, + zarrVersion) + ) } def requestDataSourceFolderContents(token: Option[String], organizationName: String, - datasetName: String): Action[AnyContent] = + datasetName: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationName)), urlOrHeaderToken(token, request)) { @@ -405,17 +533,21 @@ class ZarrStreamingController @Inject()( dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationName)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND layerNames = dataSource.dataLayers.map((dataLayer: DataLayer) => dataLayer.name) + additionalVersionDependantFiles = if (zarrVersion == 2) List(NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List.empty } yield Ok( views.html.datastoreZarrDatasourceDir( "Datastore", s"$organizationName/$datasetName", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) ++ additionalVersionDependantFiles ++ layerNames )) } } - def dataSourceFolderContentsPrivateLink(token: Option[String], accessToken: String): Action[AnyContent] = + def dataSourceFolderContentsPrivateLink(token: Option[String], + accessToken: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) @@ -427,12 +559,16 @@ class ZarrStreamingController @Inject()( .map((dataLayer: DataLayer) => dataLayer.name) .filter(!annotationLayerNames.contains(_)) layerNames = annotationLayerNames ++ dataSourceLayerNames + additionalEntries = if (zarrVersion == 2) + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else + List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Combined datastore and tracingstore directory", s"$accessToken", - List(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON, ".zgroup") ++ layerNames + additionalEntries ++ layerNames )) } @@ -451,19 +587,15 @@ class ZarrStreamingController @Inject()( def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - layer = annotationSource.getAnnotationLayer(dataLayerName) - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) - result <- layer match { - case Some(annotationLayer) => - remoteTracingstoreClient - .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) - .map(Ok(_)) - case None => - Fox.successful(Ok(zGroupJson)) - } - } yield result + ifIsAnnotationLayerOrElse( + token, + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + remoteTracingstoreClient + .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .map(Ok(_)), + orElse = _ => Fox.successful(Ok(zGroupJson)) + ) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala index 1ebc67f223b..17810f5efb3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala @@ -4,15 +4,13 @@ import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.webknossos.datastore.dataformats.{DatasetArrayBucketProvider, MagLocator} import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration -import com.scalableminds.webknossos.datastore.models.datasource._ +import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, _} import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService import play.api.libs.json.{Json, OFormat} import ucar.ma2.{Array => MultiArray} trait ZarrLayer extends DataLayerWithMagLocators { - val dataFormat: DataFormat.Value = DataFormat.zarr - def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]) = @@ -36,7 +34,8 @@ case class ZarrDataLayer( adminViewConfiguration: Option[LayerViewConfiguration] = None, coordinateTransformations: Option[List[CoordinateTransformation]] = None, override val numChannels: Option[Int] = Some(1), - override val additionalAxes: Option[Seq[AdditionalAxis]] + override val additionalAxes: Option[Seq[AdditionalAxis]], + override val dataFormat: DataFormat.Value, ) extends ZarrLayer object ZarrDataLayer { @@ -54,7 +53,8 @@ case class ZarrSegmentationLayer( adminViewConfiguration: Option[LayerViewConfiguration] = None, coordinateTransformations: Option[List[CoordinateTransformation]] = None, override val numChannels: Option[Int] = Some(1), - additionalAxes: Option[Seq[AdditionalAxis]] = None + additionalAxes: Option[Seq[AdditionalAxis]] = None, + override val dataFormat: DataFormat.Value, ) extends SegmentationLayer with ZarrLayer diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala index b17d5ed1ad0..6d7aa99cc22 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala @@ -1,10 +1,18 @@ package com.scalableminds.webknossos.datastore.dataformats.zarr +import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox} +import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate +import play.api.http.Status.NOT_FOUND +import play.api.i18n.{Messages, MessagesProvider} + +import scala.concurrent.ExecutionContext + object ZarrCoordinatesParser { def parseDotCoordinates( cxyz: String, ): Option[(Int, Int, Int, Int)] = { - val singleRx = "\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*".r + val singleRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r cxyz match { case singleRx(c, x, y, z) => @@ -12,4 +20,32 @@ object ZarrCoordinatesParser { case _ => None } } + + def parseNDimensionalDotCoordinates( + coordinates: String, + )(implicit ec: ExecutionContext, m: MessagesProvider): Fox[(Int, Int, Int, Option[List[AdditionalCoordinate]])] = { + val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r + // The tail cuts off the leading "c" form the "c." at the beginning of coordinates. + + for { + parsedCoordinates <- ndCoordinatesRx + .findFirstIn(coordinates) + .map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) ?~> + Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND + channelCoordinate <- parsedCoordinates.headOption ~> NOT_FOUND + _ <- bool2Fox(channelCoordinate == 0) ?~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND + _ <- bool2Fox(parsedCoordinates.length >= 4) ?~> "zarr.notEnoughCoordinates" ~> NOT_FOUND + (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3), + parsedCoordinates(parsedCoordinates.length - 2), + parsedCoordinates(parsedCoordinates.length - 1)) + additionalCoordinates = if (parsedCoordinates.length > 4) + Some( + parsedCoordinates + .slice(1, parsedCoordinates.length - 3) + .zipWithIndex + .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1)) + .toList) + else None + } yield (x, y, z, additionalCoordinates) + } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala index cb55fbaf3ec..2828e6a84a4 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala @@ -1,46 +1,15 @@ -package com.scalableminds.webknossos.datastore.datareaders.zarr; +package com.scalableminds.webknossos.datastore.datareaders.zarr import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} -import com.scalableminds.webknossos.datastore.models -import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} -import net.liftweb.common.{Box, Failure, Full} +import com.scalableminds.webknossos.datastore.models.VoxelSize import play.api.libs.json.{Json, OFormat} -case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) - -object NgffCoordinateTransformation { - implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] -} - -case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) - -object NgffDataset { - implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] -} - case class NgffGroupHeader(zarr_format: Int) object NgffGroupHeader { implicit val jsonFormat: OFormat[NgffGroupHeader] = Json.format[NgffGroupHeader] val FILENAME_DOT_ZGROUP = ".zgroup" } -case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { - - def lengthUnit: Box[models.LengthUnit.Value] = - if (`type` != "space") - Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") - else { - unit match { - case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) - case Some(someUnit) => LengthUnit.fromString(someUnit) - } - } -} - -object NgffAxis { - implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] -} - case class NgffMultiscalesItem( version: String = "0.4", // format version number name: Option[String], @@ -92,22 +61,3 @@ object NgffLabelsGroup { implicit val jsonFormat: OFormat[NgffLabelsGroup] = Json.format[NgffLabelsGroup] val LABEL_PATH = "labels/.zattrs" } - -case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) -object NgffOmeroMetadata { - implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] -} - -case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) -object NgffChannelWindow { - implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] -} - -case class NgffChannelAttributes(color: Option[String], - label: Option[String], - window: Option[NgffChannelWindow], - inverted: Option[Boolean], - active: Option[Boolean]) -object NgffChannelAttributes { - implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] -} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala new file mode 100644 index 00000000000..1d4841c281e --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala @@ -0,0 +1,58 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr + +import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} +import com.scalableminds.webknossos.datastore.models.VoxelSize +import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis +import play.api.libs.json.{Json, OFormat} + +// See suggested changes to version v0.5 here together with an example: https://ngff.openmicroscopy.org/rfc/2/index.html#examples +case class NgffMultiscalesItemV0_5( + // Ngff V0.5 no longer has the version inside the multiscale field. + name: Option[String], + axes: List[NgffAxis] = List( + NgffAxis(name = "c", `type` = "channel"), + NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")), + NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")), + ), + datasets: List[NgffDataset] +) + +object NgffMultiscalesItemV0_5 { + implicit val jsonFormat: OFormat[NgffMultiscalesItemV0_5] = Json.format[NgffMultiscalesItemV0_5] +} + +case class NgffMetadataV0_5(version: String, + multiscales: List[NgffMultiscalesItemV0_5], + omero: Option[NgffOmeroMetadata]) + +object NgffMetadataV0_5 { + def fromNameVoxelSizeAndMags(dataLayerName: String, + dataSourceVoxelSize: VoxelSize, + mags: List[Vec3Int], + additionalAxes: Option[Seq[AdditionalAxis]], + version: String = "0.5"): NgffMetadataV0_5 = { + val datasets = mags.map( + mag => + NgffDataset( + path = mag.toMagLiteral(allowScalar = true), + List(NgffCoordinateTransformation( + scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList))) + )) + val lengthUnitStr = dataSourceVoxelSize.unit.toString + val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes + .getOrElse(List.empty) + .zipWithIndex + .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List( + NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)), + NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)), + ) + NgffMetadataV0_5(version, + multiscales = + List(NgffMultiscalesItemV0_5(name = Some(dataLayerName), datasets = datasets, axes = axes)), + None) + } + + implicit val jsonFormat: OFormat[NgffMetadataV0_5] = Json.format[NgffMetadataV0_5] +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala new file mode 100644 index 00000000000..7b72f80e077 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala @@ -0,0 +1,54 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr + +import com.scalableminds.webknossos.datastore.models +import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize} +import net.liftweb.common.{Box, Failure, Full} +import play.api.libs.json.{Json, OFormat} + +case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]]) + +object NgffCoordinateTransformation { + implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation] +} + +case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation]) + +object NgffDataset { + implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset] +} + +case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) { + + def lengthUnit: Box[models.LengthUnit.Value] = + if (`type` != "space") + Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit") + else { + unit match { + case None | Some("") => Full(VoxelSize.DEFAULT_UNIT) + case Some(someUnit) => LengthUnit.fromString(someUnit) + } + } +} + +object NgffAxis { + implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis] +} + +case class NgffOmeroMetadata(channels: List[NgffChannelAttributes]) +object NgffOmeroMetadata { + implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata] +} + +case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double) +object NgffChannelWindow { + implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow] +} + +case class NgffChannelAttributes(color: Option[String], + label: Option[String], + window: Option[NgffChannelWindow], + inverted: Option[Boolean], + active: Option[Boolean]) +object NgffChannelAttributes { + implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes] +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala index 3dd32b32c43..39f75300b27 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala @@ -13,6 +13,7 @@ import com.scalableminds.webknossos.datastore.datareaders.{ NullCompressor } import com.scalableminds.webknossos.datastore.helpers.JsonImplicits +import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer} import net.liftweb.common.Box.tryo import net.liftweb.common.{Box, Full} import play.api.libs.json.{Format, JsArray, JsResult, JsString, JsSuccess, JsValue, Json, OFormat} @@ -240,7 +241,7 @@ object Zarr3ArrayHeader extends JsonImplicits { ChunkGridConfiguration(Array(1, 1, 1))))), // Extension not supported for now "chunk_key_encoding" -> zarrArrayHeader.chunk_key_encoding, "fill_value" -> zarrArrayHeader.fill_value, - "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map("" -> ""))), + "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map.empty)), "codecs" -> zarrArrayHeader.codecs.map { codec: CodecConfiguration => val configurationJson = if (codec.includeConfiguration) Json.obj("configuration" -> codec) else Json.obj() Json.obj("name" -> codec.name) ++ configurationJson @@ -248,5 +249,41 @@ object Zarr3ArrayHeader extends JsonImplicits { "storage_transformers" -> zarrArrayHeader.storage_transformers, "dimension_names" -> zarrArrayHeader.dimension_names ) + + } + def fromDataLayer(dataLayer: DataLayer): Zarr3ArrayHeader = { + val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty)) + Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray, + data_type = Left(dataLayer.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + ) + } + private def reorderAdditionalAxes(additionalAxes: Seq[AdditionalAxis]): Seq[AdditionalAxis] = { + val additionalAxesStartIndex = 1 // channel comes first + val sorted = additionalAxes.sortBy(_.index) + sorted.zipWithIndex.map { + case (axis, index) => axis.copy(index = index + additionalAxesStartIndex) + } } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala new file mode 100644 index 00000000000..396f830288b --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala @@ -0,0 +1,35 @@ +package com.scalableminds.webknossos.datastore.datareaders.zarr3 + +import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV0_5 +import play.api.libs.json._ + +case class Zarr3GroupHeader( + zarr_format: Int, // must be 3 + node_type: String, // must be "group" + ngffMetadata: Option[NgffMetadataV0_5], +) + +object Zarr3GroupHeader { + implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] { + override def reads(json: JsValue): JsResult[Zarr3GroupHeader] = + for { + zarr_format <- (json \ "zarr_format").validate[Int] + node_type <- (json \ "node_type").validate[String] + // Read the metadata from the correct json path. + ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5] + } yield + Zarr3GroupHeader( + zarr_format, + node_type, + ngffMetadata, + ) + + override def writes(zarrArrayGroup: Zarr3GroupHeader): JsValue = + Json.obj( + "zarr_format" -> zarrArrayGroup.zarr_format, + "node_type" -> zarrArrayGroup.node_type, + // Enforce correct path for ngffMetadata in the json. + "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata), + ) + } +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala index 78f4c850f90..96ad60754b9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NgffExplorer.scala @@ -14,6 +14,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfigu import com.scalableminds.webknossos.datastore.models.datasource.{ AdditionalAxis, Category, + DataFormat, ElementClass, LayerViewConfiguration } @@ -123,7 +124,8 @@ class NgffExplorer(implicit val ec: ExecutionContext) extends RemoteLayerExplore magsWithAttributes.map(_.mag), largestSegmentId = None, additionalAxes = Some(additionalAxes), - defaultViewConfiguration = Some(viewConfig) + defaultViewConfiguration = Some(viewConfig), + dataFormat = DataFormat.zarr ) } else ZarrDataLayer( @@ -133,7 +135,8 @@ class NgffExplorer(implicit val ec: ExecutionContext) extends RemoteLayerExplore elementClass, magsWithAttributes.map(_.mag), additionalAxes = Some(additionalAxes), - defaultViewConfiguration = Some(viewConfig) + defaultViewConfiguration = Some(viewConfig), + dataFormat = DataFormat.zarr ) } yield (layer, VoxelSize(voxelSizeFactor, unifiedAxisUnit)) }) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala index 416915ed398..11f83904b4b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ZarrArrayExplorer.scala @@ -8,7 +8,7 @@ import com.scalableminds.webknossos.datastore.datareaders.AxisOrder import com.scalableminds.webknossos.datastore.datareaders.zarr.ZarrHeader import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.VoxelSize -import com.scalableminds.webknossos.datastore.models.datasource.Category +import com.scalableminds.webknossos.datastore.models.datasource.{Category, DataFormat} import scala.concurrent.ExecutionContext @@ -26,9 +26,20 @@ class ZarrArrayExplorer(mag: Vec3Int = Vec3Int.ones)(implicit val ec: ExecutionC boundingBox <- zarrHeader.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format" magLocator = MagLocator(mag, Some(remotePath.toUri.toString), None, Some(guessedAxisOrder), None, credentialId) layer: ZarrLayer = if (looksLikeSegmentationLayer(name, elementClass)) { - ZarrSegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None) + ZarrSegmentationLayer(name, + boundingBox, + elementClass, + List(magLocator), + largestSegmentId = None, + dataFormat = DataFormat.zarr) } else - ZarrDataLayer(name, Category.color, boundingBox, elementClass, List(magLocator), additionalAxes = None) + ZarrDataLayer(name, + Category.color, + boundingBox, + elementClass, + List(magLocator), + additionalAxes = None, + dataFormat = DataFormat.zarr) } yield List((layer, VoxelSize.fromFactorWithDefaultUnit(Vec3Double.ones))) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 038812c2bf1..4d044540f3c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -5,6 +5,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} import com.scalableminds.webknossos.datastore.rpc.RPC import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle @@ -24,11 +25,22 @@ class DSRemoteTracingstoreClient @Inject()( val lifecycle: ApplicationLifecycle, ) extends LazyLogging with FoxImplicits { + private def getZarrVersionDependantSubPath = + (zarrVersion: Int) => if (zarrVersion == 2) "zarr" else "zarr3_experimental" + def getZArray(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[ZarrHeader] = rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/.zarray") .addQueryStringOptional("token", token) .getWithJsonResponse[ZarrHeader] + def getZarrJson(tracingId: String, + mag: String, + tracingStoreUri: String, + token: Option[String]): Fox[Zarr3ArrayHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/zarr.json") + .addQueryStringOptional("token", token) + .getWithJsonResponse[Zarr3ArrayHeader] + def getVolumeLayerAsZarrLayer(tracingId: String, tracingName: Option[String], tracingStoreUri: String, @@ -43,6 +55,13 @@ class DSRemoteTracingstoreClient @Inject()( .addQueryStringOptional("token", token) .getWithJsonResponse[NgffMetadata] + def getZarrJsonGroupHeaderWithNgff(tracingId: String, + tracingStoreUri: String, + token: Option[String]): Fox[Zarr3GroupHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/zarr.json") + .addQueryStringOptional("token", token) + .getWithJsonResponse[Zarr3GroupHeader] + def getRawZarrCube(tracingId: String, mag: String, cxyz: String, @@ -55,13 +74,17 @@ class DSRemoteTracingstoreClient @Inject()( def getDataLayerMagFolderContents(tracingId: String, mag: String, tracingStoreUri: String, - token: Option[String]): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/json/$tracingId/$mag") + token: Option[String], + zarrVersion: Int): Fox[List[String]] = + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] - def getDataLayerFolderContents(tracingId: String, tracingStoreUri: String, token: Option[String]): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/json/$tracingId") + def getDataLayerFolderContents(tracingId: String, + tracingStoreUri: String, + token: Option[String], + zarrVersion: Int): Fox[List[String]] = + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId") .addQueryStringOptional("token", token) .getWithJsonResponse[List[String]] diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index 333787f6b64..3181a8fc4e5 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -15,32 +15,56 @@ GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/his # Knossos compatible routes GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) -# Zarr compatible routes -GET /zarr/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) -GET /zarr/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String) +# Zarr2 compatible routes +GET /zarr/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationName: String, datasetName: String, dataLayerName="") -GET /zarr/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) +GET /zarr/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) GET /zarr/:organizationName/:datasetName/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, cxyz: String) +GET /zarr/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) -GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String) -GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String) +GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName="") -GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:cxyz @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, cxyz: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) + +# Zarr3 compatible routes +GET /zarr3_experimental/:organizationName/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +# GET /zarr3_experimental/:organizationName/:datasetName/ TODO M ask whether there is a zarr.json on this level +GET /zarr3_experimental/:organizationName/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationName: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr3_experimental/:organizationName/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) + +GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Segmentation mappings GET /datasets/:organizationName/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(token: Option[String], organizationName: String, datasetName: String, dataLayerName: String, mappingName: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 02781cfc9fe..5271c743dfa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -8,11 +8,27 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.dataformats.zarr.ZarrCoordinatesParser -import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffGroupHeader, NgffMetadata, ZarrHeader} +import com.scalableminds.webknossos.datastore.datareaders.zarr.{ + NgffGroupHeader, + NgffMetadata, + NgffMetadataV0_5, + ZarrHeader +} +import com.scalableminds.webknossos.datastore.datareaders.zarr3.{ + BytesCodecConfiguration, + ChunkGridConfiguration, + ChunkGridSpecification, + ChunkKeyEncoding, + ChunkKeyEncodingConfiguration, + TransposeCodecConfiguration, + TransposeSetting, + Zarr3ArrayHeader, + Zarr3GroupHeader +} import com.scalableminds.webknossos.datastore.datareaders.{ArrayOrder, AxisOrder} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import com.scalableminds.webknossos.datastore.models.WebknossosDataRequest -import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass} +import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} +import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataFormat, DataLayer, ElementClass} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService @@ -39,33 +55,42 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(token: Option[String], tracingId: String): Action[AnyContent] = + def volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Tracingstore", "%s".format(tracingId), - List(".zattrs", ".zgroup") ++ existingMags.map(_.toMagLiteral(allowScalar = true)) + additionalFiles ++ existingMags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() } } - def volumeTracingFolderContentJson(token: Option[String], tracingId: String): Action[AnyContent] = + def volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) - } yield Ok(Json.toJson(List(".zattrs", ".zgroup") ++ existingMags)) + additionalFiles = if (zarrVersion == 2) + List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) + else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) + } yield Ok(Json.toJson(additionalFiles ++ existingMags)) } } - def volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + def volumeTracingMagFolderContent(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -74,17 +99,21 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + files = if (zarrVersion == 2) List(".zarray") else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) } yield Ok( views.html.datastoreZarrDatasourceDir( "Tracingstore", "%s".format(tracingId), - List(".zarray") + files )).withHeaders() } } - def volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = + def volumeTracingMagFolderContentJson(token: Option[String], + tracingId: String, + mag: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -93,7 +122,8 @@ class VolumeTracingZarrStreamingController @Inject()( existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - } yield Ok(Json.toJson(List(".zarray"))) + files = if (zarrVersion == 2) List(".zarray") else List(Zarr3ArrayHeader.FILENAME_ZARR_JSON) + } yield Ok(Json.toJson(files)) } } @@ -133,6 +163,53 @@ class VolumeTracingZarrStreamingController @Inject()( } } + def zarrJsonForMag(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { + implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + magParsed <- Vec3Int + .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND + + additionalAxes = AdditionalAxis.fromProtos(tracing.additionalAxes) + dimNames = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z")) + + zarrHeader = Zarr3ArrayHeader( + zarr_format = 3, + node_type = "array", + // channel, additional axes, XYZ + shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ Array( + (tracing.boundingBox.width + tracing.boundingBox.topLeft.x) / magParsed.x, + (tracing.boundingBox.height + tracing.boundingBox.topLeft.y) / magParsed.y, + (tracing.boundingBox.depth + tracing.boundingBox.topLeft.z) / magParsed.z + ), + data_type = Left(tracing.elementClass.toString), + chunk_grid = Left( + ChunkGridSpecification( + "regular", + ChunkGridConfiguration( + chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength, + DataLayer.bucketLength, + DataLayer.bucketLength)) + )), + chunk_key_encoding = + ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))), + fill_value = Right(0), + attributes = None, + codecs = Seq( + TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)), + BytesCodecConfiguration(Some("little")), + ), + storage_transformers = None, + dimension_names = dimNames + ) + } yield Ok(Json.toJson(zarrHeader)) + } + } + def zGroup(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) @@ -161,7 +238,29 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]): Action[AnyContent] = + def zarrJson( + token: Option[String], + tracingId: String, + ): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + + existingMags = tracing.resolutions.map(vec3IntFromProto) + dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND + omeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(tracingId, + dataSourceVoxelSize = dataSource.scale, + mags = existingMags.toList, + additionalAxes = dataSource.additionalAxesUnion) + zarr3GroupHeader = Zarr3GroupHeader(3, "group", Some(omeNgffHeader)) + } yield Ok(Json.toJson(zarr3GroupHeader)) + } + } + + def zarrSource(token: Option[String], + tracingId: String, + tracingName: Option[String], + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { @@ -174,13 +273,14 @@ class VolumeTracingZarrStreamingController @Inject()( elementClass = tracing.elementClass, mags = tracing.resolutions.toList.map(x => MagLocator(x, None, None, Some(AxisOrder.cxyz), None, None)), mappings = None, - numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1) + numChannels = Some(if (tracing.elementClass.isuint24) 3 else 1), + dataFormat = if (zarrVersion == 2) DataFormat.zarr else DataFormat.zarr3 ) } yield Ok(Json.toJson(zarrLayer)) } } - def rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String): Action[AnyContent] = + def rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { @@ -191,8 +291,8 @@ class VolumeTracingZarrStreamingController @Inject()( magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND - (c, x, y, z) <- ZarrCoordinatesParser.parseDotCoordinates(cxyz) ?~> Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND - _ <- bool2Fox(c == 0) ~> Messages("zarr.invalidFirstChunkCoord") ~> NOT_FOUND + (x, y, z, additionalCoordinates) <- ZarrCoordinatesParser.parseNDimensionalDotCoordinates(coordinates) ?~> Messages( + "zarr.invalidChunkCoordinates") ~> NOT_FOUND cubeSize = DataLayer.bucketLength wkRequest = WebknossosDataRequest( position = Vec3Int(x, y, z) * cubeSize * magParsed, @@ -201,7 +301,7 @@ class VolumeTracingZarrStreamingController @Inject()( fourBit = Some(false), applyAgglomerate = None, version = None, - additionalCoordinates = None + additionalCoordinates = additionalCoordinates ) (data, missingBucketIndices) <- if (tracing.getHasEditableMapping) editableMappingService.volumeData(tracing, tracingId, List(wkRequest), urlOrHeaderToken(token, request)) @@ -213,6 +313,7 @@ class VolumeTracingZarrStreamingController @Inject()( magParsed, Vec3Int(x, y, z), cubeSize, + additionalCoordinates, urlOrHeaderToken(token, request)) ~> NOT_FOUND } yield Ok(dataWithFallback) } @@ -226,6 +327,7 @@ class VolumeTracingZarrStreamingController @Inject()( mag: Vec3Int, position: Vec3Int, cubeSize: Int, + additionalCoordinates: Option[Seq[AdditionalCoordinate]], urlToken: Option[String]): Fox[Array[Byte]] = if (missingBucketIndices.nonEmpty) { for { @@ -237,7 +339,7 @@ class VolumeTracingZarrStreamingController @Inject()( fourBit = Some(false), applyAgglomerate = tracing.mappingName, version = None, - additionalCoordinates = None + additionalCoordinates = additionalCoordinates ) (fallbackData, fallbackMissingBucketIndices) <- remoteDataStoreClient.getData(remoteFallbackLayer, List(request), diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 2d1d024ace4..a7b676b1839 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -39,39 +39,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int], voxelSize: Option[VoxelSize])( implicit ec: ExecutionContext): Iterator[NamedStream] = { - val header = Zarr3ArrayHeader( - zarr_format = 3, - node_type = "array", - // channel, additional axes, XYZ - shape = Array(1) ++ additionalAxesSorted.map(_.highestValue).toArray ++ layer.boundingBox.bottomRight.toArray, - data_type = Left(layer.elementClass.toString), - chunk_grid = Left( - ChunkGridSpecification( - "regular", - ChunkGridConfiguration( - chunk_shape = Array.fill(1 + additionalAxesSorted.length)(1) ++ Array(DataLayer.bucketLength, - DataLayer.bucketLength, - DataLayer.bucketLength)) - )), - chunk_key_encoding = - ChunkKeyEncoding("default", - configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(dimensionSeparator)))), - fill_value = Right(0), - attributes = None, - codecs = Seq( - TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(rank)), - BytesCodecConfiguration(Some("little")), - BloscCodecConfiguration( - BloscCompressor.defaultCname, - BloscCompressor.defaultCLevel, - StringCompressionSetting(BloscCodecConfiguration.shuffleSettingFromInt(BloscCompressor.defaultShuffle)), - Some(BloscCompressor.defaultTypesize), - BloscCompressor.defaultBlocksize - ) - ), - storage_transformers = None, - dimension_names = Some(Array("c") ++ additionalAxesSorted.map(_.name).toArray ++ Seq("x", "y", "z")) - ) + val header = Zarr3ArrayHeader.fromDataLayer(layer) bucketStream.flatMap { case (bucket, data) => val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index cf4668e3a06..fae27dfb475 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -40,17 +40,30 @@ GET /mapping/:tracingId/segmentsForAgglomerate @com.scalablemin POST /mapping/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdsForSegments(token: Option[String], tracingId: String) # Zarr endpoints for volume annotations -GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String) -GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String) +# Zarr version 2 +GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], tracingId: String) GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String]) -GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/:cxyz @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, cxyz: String) +GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) + +# Zarr version 3 +GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) +GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) +GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String])