Skip to content

Commit

Permalink
Merge pull request #43 from thehyve/protobuf-endpoint
Browse files Browse the repository at this point in the history 
Protobuf enabled endpoint for hypercube.
  • Loading branch information
@ewelinagr
ewelinagr authored Nov 16, 2016
2 parents ab10f29 + 358ef5c commit 6e40217
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 64 deletions.
3 changes: 2 additions & 1 deletion transmart-rest-api/grails-app/conf/application.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ grails.mime.types = [
rss : 'application/rss+xml',
text : 'text/plain',
hal : ['application/hal+json', 'application/hal+xml'],
xml : ['text/xml', 'application/xml']
xml : ['text/xml', 'application/xml'],
protobuf : 'application/x-protobuf',
]

grails.mime.use.accept.header = true
Expand Down
28 changes: 19 additions & 9 deletions transmart-rest-api/grails-app/controllers/org/transmartproject/rest/QueryController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,17 @@ import grails.converters.JSON
import groovy.util.logging.Slf4j
import org.grails.web.converters.exceptions.ConverterException
import org.springframework.beans.factory.annotation.Autowired
import org.transmartproject.core.dataquery.highdim.HighDimensionDataTypeResource
import org.transmartproject.core.dataquery.highdim.HighDimensionResource
import org.transmartproject.core.dataquery.highdim.dataconstraints.DataConstraint
import org.transmartproject.core.dataquery.highdim.projections.Projection
import org.transmartproject.core.exceptions.InvalidArgumentsException
import org.transmartproject.core.users.UsersResource
import org.transmartproject.db.clinical.MultidimensionalDataResourceService
import org.transmartproject.db.dataquery2.HypercubeImpl
import org.transmartproject.db.dataquery.highdim.DefaultHighDimensionTabularResult
import org.transmartproject.db.dataquery.highdim.HighDimensionResourceService
import org.transmartproject.db.dataquery.highdim.mrna.MrnaModule
import org.transmartproject.db.dataquery2.QueryService
import org.transmartproject.db.dataquery2.query.*
import org.transmartproject.db.user.User
import org.transmartproject.rest.misc.CurrentUser
import org.transmartproject.rest.misc.LazyOutputStreamDecorator
import org.transmartproject.rest.protobuf.HighDimBuilder
import org.transmartproject.rest.protobuf.ObservationsSerializer

@Slf4j
class QueryController {
Expand Down Expand Up @@ -108,22 +102,38 @@ class QueryController {
* @return a hypercube representing the observations that satisfy the constraint.
*/
def hypercube() {
ObservationsSerializer.Format format = ObservationsSerializer.Format.NONE
withFormat {
json {
format = ObservationsSerializer.Format.JSON
}
protobuf {
format = ObservationsSerializer.Format.PROTOBUF
}
}
if (format == ObservationsSerializer.Format.NONE) {
throw new InvalidArgumentsException("Format not supported.")
}

Constraint constraint = bindConstraint()
if (constraint == null) {
return
}
User user = (User) usersResource.getUserFromUsername(currentUser.username)
HypercubeImpl result = queryService.retrieveClinicalData(constraint, user)

log.info "Writing to format: ${format}"
OutputStream out = new LazyOutputStreamDecorator(
outputStreamProducer: { ->
response.contentType = 'application/json'
response.contentType = format.toString()
response.outputStream
})
try {
multidimensionalDataSerialisationService.writeData(result, "json", out)
multidimensionalDataSerialisationService.serialise(result, format, out)
} finally {
out.close()
}
return false
}

/**
Expand Down
13 changes: 5 additions & 8 deletions ...ls-app/services/org/transmartproject/rest/MultidimensionalDataSerialisationService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,14 @@ import org.transmartproject.rest.protobuf.ObservationsSerializer
class MultidimensionalDataSerialisationService {

/**
* Serialise hypercube data to <code>out</code>.
* Serialises hypercube data to <code>out</code>.
*
* @param hypercube the hypercube to serialise.
* @param format the output format. Currently only 'json' is supported.
* @param format the output format. Supports JSON and PROTOBUF.
* @param out the stream to serialise to.
*/
def writeData(HypercubeImpl hypercube, String format = "json", OutputStream out) {
if (!format.equalsIgnoreCase("json")) {
throw new UnsupportedEncodingException("Serialization format ${format} is not supported")
}
ObservationsSerializer builder = new ObservationsSerializer(hypercube)
builder.writeTo(out, format)
void serialise(HypercubeImpl hypercube, ObservationsSerializer.Format format, OutputStream out) {
ObservationsSerializer builder = new ObservationsSerializer(hypercube, format)
builder.write(out)
}
}
126 changes: 96 additions & 30 deletions ...rest-api/src/main/groovy/org/transmartproject/rest/protobuf/ObservationsSerializer.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package org.transmartproject.rest.protobuf

import com.google.protobuf.util.JsonFormat
import com.google.protobuf.Message
import groovy.util.logging.Slf4j
import org.transmartproject.core.exceptions.InvalidArgumentsException
import org.transmartproject.db.dataquery2.AssayDimension
import org.transmartproject.db.dataquery2.BioMarkerDimension
import org.transmartproject.db.dataquery2.DimensionImpl
Expand All @@ -10,6 +11,7 @@ import org.transmartproject.db.dataquery2.HypercubeValueImpl
import org.transmartproject.db.dataquery2.ProjectionDimension
import org.transmartproject.db.dataquery2.query.DimensionMetadata

import static com.google.protobuf.util.JsonFormat.*
import static org.transmartproject.rest.hypercubeProto.ObservationsProto.*
import static org.transmartproject.rest.hypercubeProto.ObservationsProto.FieldDefinition.*

Expand All @@ -19,14 +21,81 @@ import static org.transmartproject.rest.hypercubeProto.ObservationsProto.FieldDe
@Slf4j
public class ObservationsSerializer {

HypercubeImpl cube
JsonFormat.Printer jsonPrinter
Map<DimensionImpl, List<Object>> dimensionElements = [:]
Map<DimensionImpl, List<FieldDefinition>> dimensionFields = [:]
enum Format {
JSON('application/json'),
PROTOBUF('application/x-protobuf'),
NONE('none')

ObservationsSerializer(HypercubeImpl cube) {
jsonPrinter = JsonFormat.printer()
private String format

Format(String format) {
this.format = format
}

private static final Map<String, Format> mapping = Format.values().collectEntries {
[(it.format): it]
}

public static Format from(String format) {
if (mapping.containsKey(format)) {
return mapping[format]
} else {
throw new Exception("Unknown format: ${format}")
}
}

public String toString() {
format
}
}

protected HypercubeImpl cube
protected Printer jsonPrinter
protected Writer writer
protected Format format

protected Map<DimensionImpl, List<Object>> dimensionElements = [:]
protected Map<DimensionImpl, List<FieldDefinition>> dimensionFields = [:]

ObservationsSerializer(HypercubeImpl cube, Format format) {
this.cube = cube
if (format == Format.NONE) {
throw new InvalidArgumentsException("No format selected.")
} else if (format == Format.JSON) {
jsonPrinter = printer()
}
this.format = format
}

protected boolean first = true

protected void begin(OutputStream out) {
first = true
if (format == Format.JSON) {
writer = new PrintWriter(new BufferedOutputStream(out))
writer.print('[')
}
}

protected void writeMessage(OutputStream out, Message message) {
if (format == Format.JSON) {
if (!first) {
writer.print(', ')
}
jsonPrinter.appendTo(message, writer)
} else {
message.writeTo(out)
}
if (first) {
first = false
}
}

protected void end(OutputStream out) {
if (format == Format.JSON) {
writer.print(']')
writer.flush()
}
}

static ColumnType getFieldType(Class type) {
Expand All @@ -46,7 +115,7 @@ public class ObservationsSerializer {
}
}

def getDimensionsDefs() {
protected getDimensionsDefs() {
def dimensionDeclarations = cube.dimensions.collect { dim ->
def builder = DimensionDeclaration.newBuilder()
String dimensionName = dim.toString()
Expand Down Expand Up @@ -88,21 +157,21 @@ public class ObservationsSerializer {
dimensionDeclarations
}

def writeHeader(BufferedWriter out, String format = "json") {
def header = Header.newBuilder().addAllDimensionDeclarations(dimensionsDefs)
jsonPrinter. appendTo(header, out)
protected Header buildHeader() {
Header.newBuilder().addAllDimensionDeclarations(dimensionsDefs).build()
}

def writeCells(BufferedWriter out) {
protected void writeCells(OutputStream out) {
Iterator<HypercubeValueImpl> it = cube.iterator
while (it.hasNext()) {
HypercubeValueImpl value = it.next()
Observation observation = createCell(value)
jsonPrinter.appendTo(observation, out)
def cell = createCell(value)
cell.last = !it.hasNext()
writeMessage(out, cell.build())
}
}

Observation createCell(HypercubeValueImpl value) {
protected Observation.Builder createCell(HypercubeValueImpl value) {
Observation.Builder builder = Observation.newBuilder()
if (value.value instanceof Number) {
builder.numericValue = value.value
Expand All @@ -117,7 +186,7 @@ public class ObservationsSerializer {
builder.addDimensionIndexes(determineFooterIndex(dim, dimElement))
}
}
builder.build()
builder
}

static buildValue(FieldDefinition field, Object value) {
Expand Down Expand Up @@ -223,15 +292,15 @@ public class ObservationsSerializer {
builder.build()
}

private DimensionElement buildSparseCell(DimensionImpl dim, Object dimElement) {
protected DimensionElement buildSparseCell(DimensionImpl dim, Object dimElement) {
def builder = DimensionElement.newBuilder()
for (FieldDefinition field: dimensionFields[dim]) {
builder.addFields(buildValue(field, dimElement.getAt(field.name)))
}
builder.build()
}

def getFooter() {
protected getFooter() {
cube.dimensions.findAll({ it.density != DimensionImpl.Density.SPARSE }).collect { dim ->
def fields = dimensionFields[dim] ?: []
def elementsBuilder = DimensionElements.newBuilder()
Expand All @@ -248,12 +317,11 @@ public class ObservationsSerializer {
}
}

def writeFooter(BufferedWriter out) {
def footer = Footer.newBuilder().addAllDimension(footer)
jsonPrinter.appendTo(footer, out)
protected Footer buildFooter() {
Footer.newBuilder().addAllDimension(footer).build()
}

int determineFooterIndex(DimensionImpl dim, Object element) {
protected int determineFooterIndex(DimensionImpl dim, Object element) {
if (dimensionElements[dim] == null) {
dimensionElements[dim] = []
}
Expand All @@ -265,14 +333,12 @@ public class ObservationsSerializer {
index
}

void writeTo(OutputStream out, String format = "json") {
Writer writer = new OutputStreamWriter(out)
BufferedWriter bufferedWriter = new BufferedWriter(writer)
writeHeader(bufferedWriter)
writeCells(bufferedWriter)
writeFooter(bufferedWriter)
bufferedWriter.flush()
bufferedWriter.close()
void write(OutputStream out) {
begin(out)
writeMessage(out, buildHeader())
writeCells(out)
writeMessage(out, buildFooter())
end(out)
}

}
56 changes: 41 additions & 15 deletions ...st-api/src/test/groovy/org/transmartproject/rest/protobug/ObservationsBuilderTests.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@ package org.transmartproject.rest.protobug

import grails.test.mixin.integration.Integration
import grails.transaction.Rollback
import groovy.json.JsonSlurper
import org.junit.Test
import org.springframework.beans.factory.annotation.Autowired
import org.transmartproject.db.clinical.MultidimensionalDataResourceService
import org.transmartproject.db.dataquery.clinical.ClinicalTestData
import org.transmartproject.db.dataquery2.DimensionImpl
import org.transmartproject.db.dataquery2.HypercubeValueImpl
import org.transmartproject.db.dataquery2.query.Constraint
import org.transmartproject.db.dataquery2.query.StudyConstraint
import org.transmartproject.db.metadata.DimensionDescription
import org.transmartproject.db.TestData
import org.transmartproject.rest.protobuf.ObservationsSerializer

import static org.hamcrest.MatcherAssert.assertThat
import static org.hamcrest.core.IsNull.notNullValue
import static org.hamcrest.Matchers.*


/**
Expand All @@ -33,22 +33,48 @@ class ObservationsBuilderTests {
MultidimensionalDataResourceService queryResource

@Test
public void testSerialization() throws Exception {
public void testJsonSerialization() throws Exception {
setupData()
Constraint constraint = new StudyConstraint(studyId: clinicalData.longitudinalStudy.studyId)
def mockedCube = queryResource.retrieveData('clinical', [clinicalData.longitudinalStudy], constraint: constraint)
def builder = new ObservationsSerializer(mockedCube)
def blob = builder.getDimensionsDefs()
assertThat(blob, notNullValue())
Iterator<HypercubeValueImpl> it = mockedCube.iterator
def cellMsgs = []
while (it.hasNext()) {
HypercubeValueImpl value = it.next()
cellMsgs.add(builder.createCell(value))
}
assertThat(cellMsgs, notNullValue())
def footer = builder.getFooter()
assertThat(footer, notNullValue())
def builder = new ObservationsSerializer(mockedCube, ObservationsSerializer.Format.JSON)

def out = new ByteArrayOutputStream()
builder.write(out)
out.flush()
Collection result = new JsonSlurper().parse(out.toByteArray())

assertThat result.size(), is(14)
assertThat result, contains(
hasKey('dimensionDeclarations'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimensionIndexes'),
hasKey('dimension')
)
}

@Test
public void testProtobufSerialization() throws Exception {
setupData()
Constraint constraint = new StudyConstraint(studyId: clinicalData.longitudinalStudy.studyId)
def mockedCube = queryResource.retrieveData('clinical', [clinicalData.longitudinalStudy], constraint: constraint)
def builder = new ObservationsSerializer(mockedCube, ObservationsSerializer.Format.PROTOBUF)

def out = new ByteArrayOutputStream()
builder.write(out)
out.flush()

assertThat out.toByteArray(), not(empty())
}

void setupData() {
Expand Down
Loading

0 comments on commit 6e40217

Please sign in to comment.