Add zarr3 streaming v0

CHANGELOG.unreleased.md

@@ -19,6 +19,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added option to expand or collapse all subgroups of a segment group in the segments tab. [#7911](https://github.com/scalableminds/webknossos/pull/7911)
 - The context menu that is opened upon right-clicking a segment in the dataview port now contains the segment's name. [#7920](https://github.com/scalableminds/webknossos/pull/7920) 
 - Upgraded backend dependencies for improved performance and stability. [#7922](https://github.com/scalableminds/webknossos/pull/7922)
+- Added Support for streaming datasets via Zarr version 3. [#7941](https://github.com/scalableminds/webknossos/pull/7941)
 - It is now saved whether segment groups are collapsed or expanded, so this information doesn't get lost e.g. upon page reload. [#7928](https://github.com/scalableminds/webknossos/pull/7928/)
 - It is now saved whether skeleton groups are collapsed or expanded. This information is also persisted to NML output. [#7939](https://github.com/scalableminds/webknossos/pull/7939)
 - The context menu entry "Focus in Segment List" expands all necessary segment groups in the segments tab to show the highlighted segment. [#7950](https://github.com/scalableminds/webknossos/pull/7950)

conf/messages

@@ -138,9 +138,10 @@ dataLayer.mustBeSegmentation=DataLayer “{0}” is not a segmentation layer
 dataLayer.wrongMag=DataLayer “{0}” does not have mag “{1}”
 dataLayer.invalidMag=Supplied “{0}” is not a valid mag format. Please use “x-y-z”
 
-zarr.invalidChunkCoordinates=The requested chunk coordinates are in an invalid format. Expected c.x.y.z
+zarr.invalidChunkCoordinates=Invalid chunk coordinates. Expected dot separated coordinates with a prefix of “c.”: c.<additional_axes.>x.y.z
 zarr.invalidFirstChunkCoord="First Channel must be 0"
 zarr.chunkNotFound=Could not find the requested chunk
+zarr.notEnoughCoordinates=Invalid number of chunk coordinates. Expected to get at least 3 dimensions and channel 0.
 
 nml.file.uploadSuccess=Successfully uploaded file
 nml.file.notFound=Could not extract NML file

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/layers/ZarrDataLayers.scala

@@ -4,15 +4,13 @@ import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.webknossos.datastore.dataformats.{DatasetArrayBucketProvider, MagLocator}
 import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration
-import com.scalableminds.webknossos.datastore.models.datasource._
+import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, _}
 import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
 import play.api.libs.json.{Json, OFormat}
 import ucar.ma2.{Array => MultiArray}
 
 trait ZarrLayer extends DataLayerWithMagLocators {
 
-  val dataFormat: DataFormat.Value = DataFormat.zarr
-
   def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService],
                      dataSourceId: DataSourceId,
                      sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]) =
@@ -36,7 +34,8 @@ case class ZarrDataLayer(
     adminViewConfiguration: Option[LayerViewConfiguration] = None,
     coordinateTransformations: Option[List[CoordinateTransformation]] = None,
     override val numChannels: Option[Int] = Some(1),
-    override val additionalAxes: Option[Seq[AdditionalAxis]]
+    override val additionalAxes: Option[Seq[AdditionalAxis]],
+    override val dataFormat: DataFormat.Value,
 ) extends ZarrLayer
 
 object ZarrDataLayer {
@@ -54,7 +53,8 @@ case class ZarrSegmentationLayer(
     adminViewConfiguration: Option[LayerViewConfiguration] = None,
     coordinateTransformations: Option[List[CoordinateTransformation]] = None,
     override val numChannels: Option[Int] = Some(1),
-    additionalAxes: Option[Seq[AdditionalAxis]] = None
+    additionalAxes: Option[Seq[AdditionalAxis]] = None,
+    override val dataFormat: DataFormat.Value,
 ) extends SegmentationLayer
     with ZarrLayer
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrCoordinatesParser.scala

@@ -1,15 +1,51 @@
 package com.scalableminds.webknossos.datastore.dataformats.zarr
 
+import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox}
+import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
+import play.api.http.Status.NOT_FOUND
+import play.api.i18n.{Messages, MessagesProvider}
+
+import scala.concurrent.ExecutionContext
+
 object ZarrCoordinatesParser {
   def parseDotCoordinates(
       cxyz: String,
   ): Option[(Int, Int, Int, Int)] = {
-    val singleRx = "\\s*([0-9]+).([0-9]+).([0-9]+).([0-9]+)\\s*".r
+    val singleRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\s*$".r
 
     cxyz match {
       case singleRx(c, x, y, z) =>
         Some(Integer.parseInt(c), Integer.parseInt(x), Integer.parseInt(y), Integer.parseInt(z))
       case _ => None
     }
   }
+
+  def parseNDimensionalDotCoordinates(
+      coordinates: String,
+  )(implicit ec: ExecutionContext, m: MessagesProvider): Fox[(Int, Int, Int, Option[List[AdditionalCoordinate]])] = {
+    val ndCoordinatesRx = "^\\s*c\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(\\.([0-9]+))+\\s*$".r
+    // The tail cuts off the leading "c" form the "c." at the beginning of coordinates.
+
+    for {
+      parsedCoordinates <- ndCoordinatesRx
+        .findFirstIn(coordinates)
+        .map(m => m.split('.').tail.map(coord => Integer.parseInt(coord))) ?~>
+        Messages("zarr.invalidChunkCoordinates") ~> NOT_FOUND
+      channelCoordinate <- parsedCoordinates.headOption ~> NOT_FOUND
+      _ <- bool2Fox(channelCoordinate == 0) ?~> "zarr.invalidFirstChunkCoord" ~> NOT_FOUND
+      _ <- bool2Fox(parsedCoordinates.length >= 4) ?~> "zarr.notEnoughCoordinates" ~> NOT_FOUND
+      (x, y, z) = (parsedCoordinates(parsedCoordinates.length - 3),
+                   parsedCoordinates(parsedCoordinates.length - 2),
+                   parsedCoordinates(parsedCoordinates.length - 1))
+      additionalCoordinates = if (parsedCoordinates.length > 4)
+        Some(
+          parsedCoordinates
+            .slice(1, parsedCoordinates.length - 3)
+            .zipWithIndex
+            .map(coordWithIndex => new AdditionalCoordinate(name = s"t${coordWithIndex._2}", value = coordWithIndex._1))
+            .toList)
+      else None
+    } yield (x, y, z, additionalCoordinates)
+  }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadata.scala

@@ -1,46 +1,15 @@
-package com.scalableminds.webknossos.datastore.datareaders.zarr;
+package com.scalableminds.webknossos.datastore.datareaders.zarr
 
 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
-import com.scalableminds.webknossos.datastore.models
-import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
-import net.liftweb.common.{Box, Failure, Full}
+import com.scalableminds.webknossos.datastore.models.VoxelSize
 import play.api.libs.json.{Json, OFormat}
 
-case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]])
-
-object NgffCoordinateTransformation {
-  implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation]
-}
-
-case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation])
-
-object NgffDataset {
-  implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset]
-}
-
 case class NgffGroupHeader(zarr_format: Int)
 object NgffGroupHeader {
   implicit val jsonFormat: OFormat[NgffGroupHeader] = Json.format[NgffGroupHeader]
   val FILENAME_DOT_ZGROUP = ".zgroup"
 }
 
-case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) {
-
-  def lengthUnit: Box[models.LengthUnit.Value] =
-    if (`type` != "space")
-      Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit")
-    else {
-      unit match {
-        case None | Some("") => Full(VoxelSize.DEFAULT_UNIT)
-        case Some(someUnit)  => LengthUnit.fromString(someUnit)
-      }
-    }
-}
-
-object NgffAxis {
-  implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis]
-}
-
 case class NgffMultiscalesItem(
     version: String = "0.4", // format version number
     name: Option[String],
@@ -92,22 +61,3 @@ object NgffLabelsGroup {
   implicit val jsonFormat: OFormat[NgffLabelsGroup] = Json.format[NgffLabelsGroup]
   val LABEL_PATH = "labels/.zattrs"
 }
-
-case class NgffOmeroMetadata(channels: List[NgffChannelAttributes])
-object NgffOmeroMetadata {
-  implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata]
-}
-
-case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double)
-object NgffChannelWindow {
-  implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow]
-}
-
-case class NgffChannelAttributes(color: Option[String],
-                                 label: Option[String],
-                                 window: Option[NgffChannelWindow],
-                                 inverted: Option[Boolean],
-                                 active: Option[Boolean])
-object NgffChannelAttributes {
-  implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes]
-}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/NgffMetadataV0_5.scala

@@ -0,0 +1,58 @@
+package com.scalableminds.webknossos.datastore.datareaders.zarr
+
+import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
+import com.scalableminds.webknossos.datastore.models.VoxelSize
+import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis
+import play.api.libs.json.{Json, OFormat}
+
+// See suggested changes to version v0.5 here together with an example: https://ngff.openmicroscopy.org/rfc/2/index.html#examples
+case class NgffMultiscalesItemV0_5(
+    // Ngff V0.5 no longer has the version inside the multiscale field.
+    name: Option[String],
+    axes: List[NgffAxis] = List(
+      NgffAxis(name = "c", `type` = "channel"),
+      NgffAxis(name = "x", `type` = "space", unit = Some("nanometer")),
+      NgffAxis(name = "y", `type` = "space", unit = Some("nanometer")),
+      NgffAxis(name = "z", `type` = "space", unit = Some("nanometer")),
+    ),
+    datasets: List[NgffDataset]
+)
+
+object NgffMultiscalesItemV0_5 {
+  implicit val jsonFormat: OFormat[NgffMultiscalesItemV0_5] = Json.format[NgffMultiscalesItemV0_5]
+}
+
+case class NgffMetadataV0_5(version: String,
+                            multiscales: List[NgffMultiscalesItemV0_5],
+                            omero: Option[NgffOmeroMetadata])
+
+object NgffMetadataV0_5 {
+  def fromNameVoxelSizeAndMags(dataLayerName: String,
+                               dataSourceVoxelSize: VoxelSize,
+                               mags: List[Vec3Int],
+                               additionalAxes: Option[Seq[AdditionalAxis]],
+                               version: String = "0.5-dev2"): NgffMetadataV0_5 = {
+    val datasets = mags.map(
+      mag =>
+        NgffDataset(
+          path = mag.toMagLiteral(allowScalar = true),
+          List(NgffCoordinateTransformation(
+            scale = Some(List[Double](1.0) ++ (dataSourceVoxelSize.factor * Vec3Double(mag)).toList)))
+      ))
+    val lengthUnitStr = dataSourceVoxelSize.unit.toString
+    val axes = List(NgffAxis(name = "c", `type` = "channel")) ++ additionalAxes
+      .getOrElse(List.empty)
+      .zipWithIndex
+      .map(axisAndIndex => NgffAxis(name = s"t${axisAndIndex._2}", `type` = "space", unit = Some(lengthUnitStr))) ++ List(
+      NgffAxis(name = "x", `type` = "space", unit = Some(lengthUnitStr)),
+      NgffAxis(name = "y", `type` = "space", unit = Some(lengthUnitStr)),
+      NgffAxis(name = "z", `type` = "space", unit = Some(lengthUnitStr)),
+    )
+    NgffMetadataV0_5(version,
+                     multiscales =
+                       List(NgffMultiscalesItemV0_5(name = Some(dataLayerName), datasets = datasets, axes = axes)),
+                     None)
+  }
+
+  implicit val jsonFormat: OFormat[NgffMetadataV0_5] = Json.format[NgffMetadataV0_5]
+}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr/SharedNgffMetadataAttributes.scala

@@ -0,0 +1,54 @@
+package com.scalableminds.webknossos.datastore.datareaders.zarr
+
+import com.scalableminds.webknossos.datastore.models
+import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
+import net.liftweb.common.{Box, Failure, Full}
+import play.api.libs.json.{Json, OFormat}
+
+case class NgffCoordinateTransformation(`type`: String = "scale", scale: Option[List[Double]])
+
+object NgffCoordinateTransformation {
+  implicit val jsonFormat: OFormat[NgffCoordinateTransformation] = Json.format[NgffCoordinateTransformation]
+}
+
+case class NgffDataset(path: String, coordinateTransformations: List[NgffCoordinateTransformation])
+
+object NgffDataset {
+  implicit val jsonFormat: OFormat[NgffDataset] = Json.format[NgffDataset]
+}
+
+case class NgffAxis(name: String, `type`: String, unit: Option[String] = None) {
+
+  def lengthUnit: Box[models.LengthUnit.Value] =
+    if (`type` != "space")
+      Failure(f"Could not convert NGFF unit $name of type ${`type`} to LengthUnit")
+    else {
+      unit match {
+        case None | Some("") => Full(VoxelSize.DEFAULT_UNIT)
+        case Some(someUnit)  => LengthUnit.fromString(someUnit)
+      }
+    }
+}
+
+object NgffAxis {
+  implicit val jsonFormat: OFormat[NgffAxis] = Json.format[NgffAxis]
+}
+
+case class NgffOmeroMetadata(channels: List[NgffChannelAttributes])
+object NgffOmeroMetadata {
+  implicit val jsonFormat: OFormat[NgffOmeroMetadata] = Json.format[NgffOmeroMetadata]
+}
+
+case class NgffChannelWindow(min: Double, max: Double, start: Double, end: Double)
+object NgffChannelWindow {
+  implicit val jsonFormat: OFormat[NgffChannelWindow] = Json.format[NgffChannelWindow]
+}
+
+case class NgffChannelAttributes(color: Option[String],
+                                 label: Option[String],
+                                 window: Option[NgffChannelWindow],
+                                 inverted: Option[Boolean],
+                                 active: Option[Boolean])
+object NgffChannelAttributes {
+  implicit val jsonFormat: OFormat[NgffChannelAttributes] = Json.format[NgffChannelAttributes]
+}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3ArrayHeader.scala

@@ -13,6 +13,7 @@ import com.scalableminds.webknossos.datastore.datareaders.{
   NullCompressor
 }
 import com.scalableminds.webknossos.datastore.helpers.JsonImplicits
+import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer}
 import net.liftweb.common.Box.tryo
 import net.liftweb.common.{Box, Full}
 import play.api.libs.json.{Format, JsArray, JsResult, JsString, JsSuccess, JsValue, Json, OFormat}
@@ -240,13 +241,49 @@ object Zarr3ArrayHeader extends JsonImplicits {
             ChunkGridConfiguration(Array(1, 1, 1))))), // Extension not supported for now
         "chunk_key_encoding" -> zarrArrayHeader.chunk_key_encoding,
         "fill_value" -> zarrArrayHeader.fill_value,
-        "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map("" -> ""))),
+        "attributes" -> Json.toJsFieldJsValueWrapper(zarrArrayHeader.attributes.getOrElse(Map.empty)),
         "codecs" -> zarrArrayHeader.codecs.map { codec: CodecConfiguration =>
           val configurationJson = if (codec.includeConfiguration) Json.obj("configuration" -> codec) else Json.obj()
           Json.obj("name" -> codec.name) ++ configurationJson
         },
         "storage_transformers" -> zarrArrayHeader.storage_transformers,
         "dimension_names" -> zarrArrayHeader.dimension_names
       )
+
+  }
+  def fromDataLayer(dataLayer: DataLayer): Zarr3ArrayHeader = {
+    val additionalAxes = reorderAdditionalAxes(dataLayer.additionalAxes.getOrElse(Seq.empty))
+    Zarr3ArrayHeader(
+      zarr_format = 3,
+      node_type = "array",
+      // channel, additional axes, XYZ
+      shape = Array(1) ++ additionalAxes.map(_.highestValue).toArray ++ dataLayer.boundingBox.bottomRight.toArray,
+      data_type = Left(dataLayer.elementClass.toString),
+      chunk_grid = Left(
+        ChunkGridSpecification(
+          "regular",
+          ChunkGridConfiguration(
+            chunk_shape = Array.fill(1 + additionalAxes.length)(1) ++ Array(DataLayer.bucketLength,
+                                                                            DataLayer.bucketLength,
+                                                                            DataLayer.bucketLength))
+        )),
+      chunk_key_encoding =
+        ChunkKeyEncoding("default", configuration = Some(ChunkKeyEncodingConfiguration(separator = Some(".")))),
+      fill_value = Right(0),
+      attributes = None,
+      codecs = Seq(
+        TransposeCodecConfiguration(TransposeSetting.fOrderFromRank(additionalAxes.length + 4)),
+        BytesCodecConfiguration(Some("little")),
+      ),
+      storage_transformers = None,
+      dimension_names = Some(Array("c") ++ additionalAxes.map(_.name).toArray ++ Seq("x", "y", "z"))
+    )
+  }
+  private def reorderAdditionalAxes(additionalAxes: Seq[AdditionalAxis]): Seq[AdditionalAxis] = {
+    val additionalAxesStartIndex = 1 // channel comes first
+    val sorted = additionalAxes.sortBy(_.index)
+    sorted.zipWithIndex.map {
+      case (axis, index) => axis.copy(index = index + additionalAxesStartIndex)
+    }
   }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3GroupHeader.scala

@@ -0,0 +1,33 @@
+package com.scalableminds.webknossos.datastore.datareaders.zarr3
+
+import com.scalableminds.webknossos.datastore.datareaders.zarr.NgffMetadataV0_5
+import play.api.libs.json._
+
+case class Zarr3GroupHeader(
+    zarr_format: Int, // must be 3
+    node_type: String, // must be "group"
+    ngffMetadata: Option[NgffMetadataV0_5],
+)
+
+object Zarr3GroupHeader {
+  implicit object Zarr3GroupHeaderFormat extends Format[Zarr3GroupHeader] {
+    override def reads(json: JsValue): JsResult[Zarr3GroupHeader] =
+      for {
+        zarr_format <- (json \ "zarr_format").validate[Int]
+        node_type <- (json \ "node_type").validate[String]
+        ngffMetadata <- (json \ "attributes" \ "ome").validateOpt[NgffMetadataV0_5]
+      } yield
+        Zarr3GroupHeader(
+          zarr_format,
+          node_type,
+          ngffMetadata,
+        )
+
+    override def writes(zarrArrayGroup: Zarr3GroupHeader): JsValue =
+      Json.obj(
+        "zarr_format" -> zarrArrayGroup.zarr_format,
+        "node_type" -> zarrArrayGroup.node_type,
+        "attributes" -> Json.obj("ome" -> zarrArrayGroup.ngffMetadata),
+      )
+  }
+}