Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

histogram route #4014

Merged
merged 18 commits into from
May 16, 2019
Merged

histogram route #4014

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c3f98bd
[WIP] first version of histogram route #3975
Apr 12, 2019
84f0eeb
lint backend
Apr 12, 2019
3517269
Merge branch 'master' of github.com:scalableminds/webknossos into his…
Apr 18, 2019
5019f07
implement data conversion
Apr 18, 2019
546a9b5
show all layers in json and use dynamic histogram bucket width
Apr 18, 2019
2eb2807
Merge branch 'master' of github.com:scalableminds/webknossos into his…
Apr 25, 2019
ce7c68d
implement pr feedback
Apr 25, 2019
90c355b
merge master
Apr 25, 2019
5e2f418
implement pr feedback 2
Apr 25, 2019
11d0a83
remove unnecessary complexity by removing configurable histogram width
Apr 25, 2019
aa11a92
Merge branch 'master' into histogram-backend-route
youri-k May 2, 2019
ccaf5b3
add error messages
May 2, 2019
0f8f01a
Merge branch 'master' into histogram-backend-route
fm3 May 7, 2019
2db6edc
Merge branch 'master' of github.com:scalableminds/webknossos into his…
May 9, 2019
8802d7e
add float support
May 9, 2019
c90d4f8
fix bracket
May 9, 2019
eeabe04
Merge branch 'master' into histogram-backend-route
jstriebel May 9, 2019
12857ce
Merge branch 'master' into histogram-backend-route
youri-k May 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions ...tastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,19 @@ class BinaryDataController @Inject()(
}
}

def createHistogram(organizationName: String, dataSetName: String, dataLayerName: String) = Action.async {
implicit request =>
accessTokenService
.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationName))) {
AllowRemoteOrigin {
for {
(dataSource, dataLayer) <- getDataSourceAndDataLayer(organizationName, dataSetName, dataLayerName)
(histogram, count) <- findDataService.createHistogram(dataSource, dataLayer)
} yield Ok(Json.obj("histogram" -> histogram, "count" -> count))
}
}
}

private def getDataSourceAndDataLayer(organizationName: String, dataSetName: String, dataLayerName: String)(
implicit m: MessagesProvider): Fox[(DataSource, DataLayer)] =
for {
Expand Down
270 changes: 168 additions & 102 deletions ...ossos-datastore/app/com/scalableminds/webknossos/datastore/services/FindDataService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,140 @@ import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, Data
import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
import com.scalableminds.util.tools.Math
import net.liftweb.common.Full
import play.api.i18n.{Messages, MessagesProvider}
import play.api.i18n.MessagesProvider
import spire.math._

import scala.concurrent.ExecutionContext
import scala.reflect.ClassTag

class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(implicit ec: ExecutionContext)
extends FoxImplicits {
val binaryDataService: BinaryDataService = dataServicesHolder.binaryDataService
var i = 0

def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)(
implicit m: MessagesProvider): Fox[Option[(Point3D, Point3D)]] =
private def convertData(
data: Array[Byte],
elementClass: ElementClass.Value,
filterZeroes: Boolean = false): Array[_ >: UByte with UShort with UInt with ULong with Float] =
elementClass match {
case ElementClass.uint8 =>
convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
.map(UByte(_))
.filter(filterZeroes && _ != UByte(0))
case ElementClass.uint16 =>
convertDataImpl[Short, ShortBuffer](data,
DataTypeFunctors[Short, ShortBuffer](_.asShortBuffer, _.get(_), _.toShort))
.map(UShort(_))
.filter(filterZeroes && _ != UShort(0))
case ElementClass.uint24 =>
convertDataImpl[Byte, ByteBuffer](combineBytes(data),
DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
.map(UByte(_))
.filter(filterZeroes && _ != UByte(0))
case ElementClass.uint32 =>
convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
.map(UInt(_))
.filter(filterZeroes && _ != UInt(0))
case ElementClass.uint64 =>
convertDataImpl[Long, LongBuffer](data, DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
.map(ULong(_))
.filter(filterZeroes && _ != ULong(0))
case ElementClass.float =>
convertDataImpl[Float, FloatBuffer](
data,
DataTypeFunctors[Float, FloatBuffer](_.asFloatBuffer(), _.get(_), _.toFloat)).filter(filterZeroes && _ != 0f)
}

private def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte],
dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = {
val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
srcBuffer.rewind()
val dstArray = Array.ofDim[T](srcBuffer.remaining())
dataTypeFunctor.copyDataFn(srcBuffer, dstArray)
dstArray
}

private def combineBytes(data: Array[Byte], numBytes: Int = 3) = {
val result = Array.ofDim[Byte](data.length / numBytes)
for (i <- data.indices by numBytes) {
var sum = 0
for (j <- 0 until numBytes) {
sum += data(i + j)
}
result(i / numBytes) = (sum / numBytes).toByte
}
result
}

private def getDataFor(dataSource: DataSource,
dataLayer: DataLayer,
position: Point3D,
resolution: Point3D): Fox[Array[Byte]] = {
val request = DataRequest(
new VoxelPosition(position.x, position.y, position.z, resolution),
DataLayer.bucketLength,
DataLayer.bucketLength,
DataLayer.bucketLength
)
binaryDataService.handleDataRequest(
DataServiceDataRequest(dataSource, dataLayer, None, request.cuboid(dataLayer), request.settings))
}

private def concatenateBuckets(buckets: Seq[Array[Byte]]): Array[Byte] =
buckets.foldLeft(Array[Byte]()) { (acc, i) =>
{
acc ++ i
}
}

private def getConcatenatedDataFor(dataSource: DataSource,
dataLayer: DataLayer,
positions: List[Point3D],
resolution: Point3D) =
for {
positionAndResolutionOpt <- checkAllPositionsForData(dataSource, dataLayer)
} yield positionAndResolutionOpt
dataBucketWise: Seq[Array[Byte]] <- Fox.serialCombined(positions)(pos =>
getDataFor(dataSource, dataLayer, pos, resolution))
dataConcatenated = concatenateBuckets(dataBucketWise)
} yield dataConcatenated

private def createPositions(dataLayer: DataLayer, iterationCount: Int = 4) = {

def positionCreationIter(remainingRuns: List[Int], currentPositions: List[Point3D]): List[Point3D] = {

def createPositionsFromExponent(exponent: Int) = {
val power = math.pow(2, exponent).toInt
val spaceBetweenWidth = dataLayer.boundingBox.width / power
val spaceBetweenHeight = dataLayer.boundingBox.height / power
val spaceBetweenDepth = dataLayer.boundingBox.depth / power
val topLeft = dataLayer.boundingBox.topLeft

if (spaceBetweenWidth < DataLayer.bucketLength && spaceBetweenHeight < DataLayer.bucketLength && spaceBetweenDepth < DataLayer.bucketLength) {
None
} else {
Some(
(for {
z <- 1 until power
y <- 1 until power
x <- 1 until power
} yield
Point3D(topLeft.x + x * spaceBetweenWidth,
topLeft.y + y * spaceBetweenHeight,
topLeft.z + z * spaceBetweenDepth)).toList
)
}
}

remainingRuns match {
case List() => currentPositions
case head :: tail =>
createPositionsFromExponent(head) match {
case Some(values) => positionCreationIter(tail, currentPositions ::: values)
case None => currentPositions
}
}
}

positionCreationIter((1 to iterationCount).toList, List[Point3D]())
}

private def checkAllPositionsForData(dataSource: DataSource,
dataLayer: DataLayer): Fox[Option[(Point3D, Point3D)]] = {
Expand Down Expand Up @@ -55,19 +174,11 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
}
}

def checkIfPositionHasData(position: Point3D, resolution: Point3D) = {
val request = DataRequest(
new VoxelPosition(position.x, position.y, position.z, resolution),
DataLayer.bucketLength,
DataLayer.bucketLength,
DataLayer.bucketLength
)
def checkIfPositionHasData(position: Point3D, resolution: Point3D) =
for {
data <- binaryDataService.handleDataRequest(
DataServiceDataRequest(dataSource, dataLayer, None, request.cuboid(dataLayer), request.settings))
data <- getDataFor(dataSource, dataLayer, position, resolution)
if data.nonEmpty && data.exists(_ != 0)
} yield position.move(getExactDataOffset(data))
}

def resolutionIter(positions: List[Point3D], remainingResolutions: List[Point3D]): Fox[Option[(Point3D, Point3D)]] =
remainingResolutions match {
Expand All @@ -85,95 +196,28 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
resolutionIter(createPositions(dataLayer).distinct, dataLayer.resolutions.sortBy(_.maxDim))
}

private def createPositions(dataLayer: DataLayer, iterationCount: Int = 4) = {

def positionCreationIter(remainingRuns: List[Int], currentPositions: List[Point3D]): List[Point3D] = {

def createPositionsFromExponent(exponent: Int) = {
val power = math.pow(2, exponent).toInt
val spaceBetweenWidth = dataLayer.boundingBox.width / power
val spaceBetweenHeight = dataLayer.boundingBox.height / power
val spaceBetweenDepth = dataLayer.boundingBox.depth / power
val topLeft = dataLayer.boundingBox.topLeft

if (spaceBetweenWidth < DataLayer.bucketLength && spaceBetweenHeight < DataLayer.bucketLength && spaceBetweenDepth < DataLayer.bucketLength) {
None
} else {
Some(
(for {
z <- 1 until power
y <- 1 until power
x <- 1 until power
} yield
Point3D(topLeft.x + x * spaceBetweenWidth,
topLeft.y + y * spaceBetweenHeight,
topLeft.z + z * spaceBetweenDepth)).toList
)
}
}

remainingRuns match {
case List() => currentPositions
case head :: tail =>
createPositionsFromExponent(head) match {
case Some(values) => positionCreationIter(tail, currentPositions ::: values)
case None => currentPositions
}
}
}

positionCreationIter((1 to iterationCount).toList, List[Point3D]())
}
def findPositionWithData(dataSource: DataSource, dataLayer: DataLayer)(
implicit m: MessagesProvider): Fox[Option[(Point3D, Point3D)]] =
for {
positionAndResolutionOpt <- checkAllPositionsForData(dataSource, dataLayer)
} yield positionAndResolutionOpt

def meanAndStdDev(dataSource: DataSource, dataLayer: DataLayer)(
implicit m: MessagesProvider): Fox[(Double, Double)] = {

def getDataFor(position: Point3D, resolution: Point3D): Fox[Array[Byte]] = {
val request = DataRequest(
new VoxelPosition(position.x, position.y, position.z, resolution),
DataLayer.bucketLength,
DataLayer.bucketLength,
DataLayer.bucketLength
)
binaryDataService.handleDataRequest(
DataServiceDataRequest(dataSource, dataLayer, None, request.cuboid(dataLayer), request.settings))
}

def concatenateBuckets(buckets: Seq[Array[Byte]]): Array[Byte] =
buckets.foldLeft(Array[Byte]()) { (acc, i) =>
{
acc ++ i
}
}

def convertNonZeroDataToDouble(data: Array[Byte], elementClass: ElementClass.Value): Array[Double] =
elementClass match {
case ElementClass.uint8 =>
convertDataImpl[Byte, ByteBuffer](data, DataTypeFunctors[Byte, ByteBuffer](identity, _.get(_), _.toByte))
.filter(_ != 0)
.map(spire.math.UByte(_).toDouble)
case ElementClass.uint16 =>
convertDataImpl[Short, ShortBuffer](data,
DataTypeFunctors[Short, ShortBuffer](
_.asShortBuffer,
_.get(_),
_.toShort)).filter(_ != 0).map(spire.math.UShort(_).toDouble)
case ElementClass.uint32 =>
convertDataImpl[Int, IntBuffer](data, DataTypeFunctors[Int, IntBuffer](_.asIntBuffer, _.get(_), _.toInt))
.filter(_ != 0)
.map(spire.math.UInt(_).toDouble)
case ElementClass.uint64 =>
convertDataImpl[Long, LongBuffer](data,
DataTypeFunctors[Long, LongBuffer](_.asLongBuffer, _.get(_), identity))
.filter(_ != 0)
.map(spire.math.ULong(_).toDouble)
convertData(data, elementClass, filterZeroes = true) match {
case d: Array[UByte] => d.map(_.toDouble)
case d: Array[UShort] => d.map(_.toDouble)
case d: Array[UInt] => d.map(_.toDouble)
case d: Array[ULong] => d.map(_.toDouble)
case d: Array[Float] => d.map(_.toDouble)
}

def meanAndStdDevForPositions(positions: List[Point3D], resolution: Point3D)(
implicit m: MessagesProvider): Fox[(Double, Double)] =
for {
dataBucketWise: Seq[Array[Byte]] <- Fox.serialCombined(positions)(pos => getDataFor(pos, resolution))
dataConcatenated = concatenateBuckets(dataBucketWise)
dataConcatenated <- getConcatenatedDataFor(dataSource, dataLayer, positions, resolution)
dataAsDoubles = convertNonZeroDataToDouble(dataConcatenated, dataLayer.elementClass)
_ <- Fox.bool2Fox(dataAsDoubles.nonEmpty) ?~> "dataSet.sampledOnlyBlack"
} yield (Math.mean(dataAsDoubles), Math.stdDev(dataAsDoubles))
Expand All @@ -185,12 +229,34 @@ class FindDataService @Inject()(dataServicesHolder: BinaryDataServiceHolder)(imp
} yield meanAndStdDev
}

private def convertDataImpl[T: ClassTag, B <: Buffer](data: Array[Byte],
dataTypeFunctor: DataTypeFunctors[T, B]): Array[T] = {
val srcBuffer = dataTypeFunctor.getTypedBufferFn(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN))
srcBuffer.rewind()
val dstArray = Array.ofDim[T](srcBuffer.remaining())
dataTypeFunctor.copyDataFn(srcBuffer, dstArray)
dstArray
def createHistogram(dataSource: DataSource, dataLayer: DataLayer) = {

def calculateHistogramValues(data: Array[_ >: UByte with UShort with UInt with ULong with Float]) = {
val counts = Array.ofDim[Long](256)
data match {
case byteData: Array[UByte] =>
byteData.foreach(el => counts(el.toInt) += 1)
case shortData: Array[UShort] =>
shortData.foreach(el => counts((el / UShort(256)).toInt) += 1)
case intData: Array[UInt] =>
intData.foreach(el => counts((el / UInt(16777216)).toInt) += 1)
case longData: Array[ULong] =>
longData.foreach(el => counts((el / ULong(math.pow(2, 56).toLong)).toInt) += 1)
case floatData: Array[Float] =>
floatData.foreach(el => counts(Math.clamp(Math.roundDown(el * 255), 0, 255)) += 1)
}
counts
}

def histogramForPositions(positions: List[Point3D], resolution: Point3D) =
for {
dataConcatenated <- getConcatenatedDataFor(dataSource, dataLayer, positions, resolution) ?~> "getting data failed"
convertedData = convertData(dataConcatenated, dataLayer.elementClass, filterZeroes = true)
} yield (calculateHistogramValues(convertedData), convertedData.length)

if (dataLayer.resolutions.nonEmpty)
histogramForPositions(createPositions(dataLayer, 2).distinct, dataLayer.resolutions.minBy(_.maxDim))
else
Fox.empty ?~> "empty resolutions"
}
}
1 change: 1 addition & 0 deletions webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumb
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestImageThumbnailJpeg(organizationName: String, dataSetName: String, dataLayerName: String, width: Int, height: Int, centerX: Option[Int], centerY: Option[Int], centerZ: Option[Int], zoom: Option[Double])
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(organizationName: String, dataSetName: String, dataLayerName: String)
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/colorStatistics @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.colorStatistics(organizationName: String, dataSetName: String, dataLayerName: String)
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.createHistogram(organizationName: String, dataSetName: String, dataLayerName: String)

# Knossos compatibale routes
GET /datasets/:organizationName/:dataSetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(organizationName: String, dataSetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int)
Expand Down