From 358ef5c6615576cb5d8b1a3121163b7463ce904a Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Wed, 16 Nov 2016 00:12:15 +0100 Subject: [PATCH] Protobuf enabled endpoint for hypercube. --- .../grails-app/conf/application.groovy | 3 +- .../rest/QueryController.groovy | 28 ++-- ...dimensionalDataSerialisationService.groovy | 13 +- .../protobuf/ObservationsSerializer.groovy | 126 +++++++++++++----- .../protobug/ObservationsBuilderTests.groovy | 56 +++++--- .../grails-app/conf/application.groovy | 3 +- 6 files changed, 165 insertions(+), 64 deletions(-) diff --git a/transmart-rest-api/grails-app/conf/application.groovy b/transmart-rest-api/grails-app/conf/application.groovy index 1a94d9a82..8a16d9e3f 100644 --- a/transmart-rest-api/grails-app/conf/application.groovy +++ b/transmart-rest-api/grails-app/conf/application.groovy @@ -54,7 +54,8 @@ grails.mime.types = [ rss : 'application/rss+xml', text : 'text/plain', hal : ['application/hal+json', 'application/hal+xml'], - xml : ['text/xml', 'application/xml'] + xml : ['text/xml', 'application/xml'], + protobuf : 'application/x-protobuf', ] grails.mime.use.accept.header = true diff --git a/transmart-rest-api/grails-app/controllers/org/transmartproject/rest/QueryController.groovy b/transmart-rest-api/grails-app/controllers/org/transmartproject/rest/QueryController.groovy index 9894ca75b..87adfd885 100644 --- a/transmart-rest-api/grails-app/controllers/org/transmartproject/rest/QueryController.groovy +++ b/transmart-rest-api/grails-app/controllers/org/transmartproject/rest/QueryController.groovy @@ -4,23 +4,17 @@ import grails.converters.JSON import groovy.util.logging.Slf4j import org.grails.web.converters.exceptions.ConverterException import org.springframework.beans.factory.annotation.Autowired -import org.transmartproject.core.dataquery.highdim.HighDimensionDataTypeResource -import org.transmartproject.core.dataquery.highdim.HighDimensionResource -import org.transmartproject.core.dataquery.highdim.dataconstraints.DataConstraint -import org.transmartproject.core.dataquery.highdim.projections.Projection import org.transmartproject.core.exceptions.InvalidArgumentsException import org.transmartproject.core.users.UsersResource -import org.transmartproject.db.clinical.MultidimensionalDataResourceService import org.transmartproject.db.dataquery2.HypercubeImpl -import org.transmartproject.db.dataquery.highdim.DefaultHighDimensionTabularResult import org.transmartproject.db.dataquery.highdim.HighDimensionResourceService -import org.transmartproject.db.dataquery.highdim.mrna.MrnaModule import org.transmartproject.db.dataquery2.QueryService import org.transmartproject.db.dataquery2.query.* import org.transmartproject.db.user.User import org.transmartproject.rest.misc.CurrentUser import org.transmartproject.rest.misc.LazyOutputStreamDecorator import org.transmartproject.rest.protobuf.HighDimBuilder +import org.transmartproject.rest.protobuf.ObservationsSerializer @Slf4j class QueryController { @@ -108,22 +102,38 @@ class QueryController { * @return a hypercube representing the observations that satisfy the constraint. */ def hypercube() { + ObservationsSerializer.Format format = ObservationsSerializer.Format.NONE + withFormat { + json { + format = ObservationsSerializer.Format.JSON + } + protobuf { + format = ObservationsSerializer.Format.PROTOBUF + } + } + if (format == ObservationsSerializer.Format.NONE) { + throw new InvalidArgumentsException("Format not supported.") + } + Constraint constraint = bindConstraint() if (constraint == null) { return } User user = (User) usersResource.getUserFromUsername(currentUser.username) HypercubeImpl result = queryService.retrieveClinicalData(constraint, user) + + log.info "Writing to format: ${format}" OutputStream out = new LazyOutputStreamDecorator( outputStreamProducer: { -> - response.contentType = 'application/json' + response.contentType = format.toString() response.outputStream }) try { - multidimensionalDataSerialisationService.writeData(result, "json", out) + multidimensionalDataSerialisationService.serialise(result, format, out) } finally { out.close() } + return false } /** diff --git a/transmart-rest-api/grails-app/services/org/transmartproject/rest/MultidimensionalDataSerialisationService.groovy b/transmart-rest-api/grails-app/services/org/transmartproject/rest/MultidimensionalDataSerialisationService.groovy index a8cffb434..fa25049fe 100644 --- a/transmart-rest-api/grails-app/services/org/transmartproject/rest/MultidimensionalDataSerialisationService.groovy +++ b/transmart-rest-api/grails-app/services/org/transmartproject/rest/MultidimensionalDataSerialisationService.groovy @@ -8,17 +8,14 @@ import org.transmartproject.rest.protobuf.ObservationsSerializer class MultidimensionalDataSerialisationService { /** - * Serialise hypercube data to out. + * Serialises hypercube data to out. * * @param hypercube the hypercube to serialise. - * @param format the output format. Currently only 'json' is supported. + * @param format the output format. Supports JSON and PROTOBUF. * @param out the stream to serialise to. */ - def writeData(HypercubeImpl hypercube, String format = "json", OutputStream out) { - if (!format.equalsIgnoreCase("json")) { - throw new UnsupportedEncodingException("Serialization format ${format} is not supported") - } - ObservationsSerializer builder = new ObservationsSerializer(hypercube) - builder.writeTo(out, format) + void serialise(HypercubeImpl hypercube, ObservationsSerializer.Format format, OutputStream out) { + ObservationsSerializer builder = new ObservationsSerializer(hypercube, format) + builder.write(out) } } diff --git a/transmart-rest-api/src/main/groovy/org/transmartproject/rest/protobuf/ObservationsSerializer.groovy b/transmart-rest-api/src/main/groovy/org/transmartproject/rest/protobuf/ObservationsSerializer.groovy index 4a2323818..be8ef6d48 100644 --- a/transmart-rest-api/src/main/groovy/org/transmartproject/rest/protobuf/ObservationsSerializer.groovy +++ b/transmart-rest-api/src/main/groovy/org/transmartproject/rest/protobuf/ObservationsSerializer.groovy @@ -1,7 +1,8 @@ package org.transmartproject.rest.protobuf -import com.google.protobuf.util.JsonFormat +import com.google.protobuf.Message import groovy.util.logging.Slf4j +import org.transmartproject.core.exceptions.InvalidArgumentsException import org.transmartproject.db.dataquery2.AssayDimension import org.transmartproject.db.dataquery2.BioMarkerDimension import org.transmartproject.db.dataquery2.DimensionImpl @@ -10,6 +11,7 @@ import org.transmartproject.db.dataquery2.HypercubeValueImpl import org.transmartproject.db.dataquery2.ProjectionDimension import org.transmartproject.db.dataquery2.query.DimensionMetadata +import static com.google.protobuf.util.JsonFormat.* import static org.transmartproject.rest.hypercubeProto.ObservationsProto.* import static org.transmartproject.rest.hypercubeProto.ObservationsProto.FieldDefinition.* @@ -19,14 +21,81 @@ import static org.transmartproject.rest.hypercubeProto.ObservationsProto.FieldDe @Slf4j public class ObservationsSerializer { - HypercubeImpl cube - JsonFormat.Printer jsonPrinter - Map> dimensionElements = [:] - Map> dimensionFields = [:] + enum Format { + JSON('application/json'), + PROTOBUF('application/x-protobuf'), + NONE('none') - ObservationsSerializer(HypercubeImpl cube) { - jsonPrinter = JsonFormat.printer() + private String format + + Format(String format) { + this.format = format + } + + private static final Map mapping = Format.values().collectEntries { + [(it.format): it] + } + + public static Format from(String format) { + if (mapping.containsKey(format)) { + return mapping[format] + } else { + throw new Exception("Unknown format: ${format}") + } + } + + public String toString() { + format + } + } + + protected HypercubeImpl cube + protected Printer jsonPrinter + protected Writer writer + protected Format format + + protected Map> dimensionElements = [:] + protected Map> dimensionFields = [:] + + ObservationsSerializer(HypercubeImpl cube, Format format) { this.cube = cube + if (format == Format.NONE) { + throw new InvalidArgumentsException("No format selected.") + } else if (format == Format.JSON) { + jsonPrinter = printer() + } + this.format = format + } + + protected boolean first = true + + protected void begin(OutputStream out) { + first = true + if (format == Format.JSON) { + writer = new PrintWriter(new BufferedOutputStream(out)) + writer.print('[') + } + } + + protected void writeMessage(OutputStream out, Message message) { + if (format == Format.JSON) { + if (!first) { + writer.print(', ') + } + jsonPrinter.appendTo(message, writer) + } else { + message.writeTo(out) + } + if (first) { + first = false + } + } + + protected void end(OutputStream out) { + if (format == Format.JSON) { + writer.print(']') + writer.flush() + } } static ColumnType getFieldType(Class type) { @@ -46,7 +115,7 @@ public class ObservationsSerializer { } } - def getDimensionsDefs() { + protected getDimensionsDefs() { def dimensionDeclarations = cube.dimensions.collect { dim -> def builder = DimensionDeclaration.newBuilder() String dimensionName = dim.toString() @@ -88,21 +157,21 @@ public class ObservationsSerializer { dimensionDeclarations } - def writeHeader(BufferedWriter out, String format = "json") { - def header = Header.newBuilder().addAllDimensionDeclarations(dimensionsDefs) - jsonPrinter. appendTo(header, out) + protected Header buildHeader() { + Header.newBuilder().addAllDimensionDeclarations(dimensionsDefs).build() } - def writeCells(BufferedWriter out) { + protected void writeCells(OutputStream out) { Iterator it = cube.iterator while (it.hasNext()) { HypercubeValueImpl value = it.next() - Observation observation = createCell(value) - jsonPrinter.appendTo(observation, out) + def cell = createCell(value) + cell.last = !it.hasNext() + writeMessage(out, cell.build()) } } - Observation createCell(HypercubeValueImpl value) { + protected Observation.Builder createCell(HypercubeValueImpl value) { Observation.Builder builder = Observation.newBuilder() if (value.value instanceof Number) { builder.numericValue = value.value @@ -117,7 +186,7 @@ public class ObservationsSerializer { builder.addDimensionIndexes(determineFooterIndex(dim, dimElement)) } } - builder.build() + builder } static buildValue(FieldDefinition field, Object value) { @@ -223,7 +292,7 @@ public class ObservationsSerializer { builder.build() } - private DimensionElement buildSparseCell(DimensionImpl dim, Object dimElement) { + protected DimensionElement buildSparseCell(DimensionImpl dim, Object dimElement) { def builder = DimensionElement.newBuilder() for (FieldDefinition field: dimensionFields[dim]) { builder.addFields(buildValue(field, dimElement.getAt(field.name))) @@ -231,7 +300,7 @@ public class ObservationsSerializer { builder.build() } - def getFooter() { + protected getFooter() { cube.dimensions.findAll({ it.density != DimensionImpl.Density.SPARSE }).collect { dim -> def fields = dimensionFields[dim] ?: [] def elementsBuilder = DimensionElements.newBuilder() @@ -248,12 +317,11 @@ public class ObservationsSerializer { } } - def writeFooter(BufferedWriter out) { - def footer = Footer.newBuilder().addAllDimension(footer) - jsonPrinter.appendTo(footer, out) + protected Footer buildFooter() { + Footer.newBuilder().addAllDimension(footer).build() } - int determineFooterIndex(DimensionImpl dim, Object element) { + protected int determineFooterIndex(DimensionImpl dim, Object element) { if (dimensionElements[dim] == null) { dimensionElements[dim] = [] } @@ -265,14 +333,12 @@ public class ObservationsSerializer { index } - void writeTo(OutputStream out, String format = "json") { - Writer writer = new OutputStreamWriter(out) - BufferedWriter bufferedWriter = new BufferedWriter(writer) - writeHeader(bufferedWriter) - writeCells(bufferedWriter) - writeFooter(bufferedWriter) - bufferedWriter.flush() - bufferedWriter.close() + void write(OutputStream out) { + begin(out) + writeMessage(out, buildHeader()) + writeCells(out) + writeMessage(out, buildFooter()) + end(out) } } diff --git a/transmart-rest-api/src/test/groovy/org/transmartproject/rest/protobug/ObservationsBuilderTests.groovy b/transmart-rest-api/src/test/groovy/org/transmartproject/rest/protobug/ObservationsBuilderTests.groovy index f5397b6d4..f6b137690 100644 --- a/transmart-rest-api/src/test/groovy/org/transmartproject/rest/protobug/ObservationsBuilderTests.groovy +++ b/transmart-rest-api/src/test/groovy/org/transmartproject/rest/protobug/ObservationsBuilderTests.groovy @@ -2,12 +2,12 @@ package org.transmartproject.rest.protobug import grails.test.mixin.integration.Integration import grails.transaction.Rollback +import groovy.json.JsonSlurper import org.junit.Test import org.springframework.beans.factory.annotation.Autowired import org.transmartproject.db.clinical.MultidimensionalDataResourceService import org.transmartproject.db.dataquery.clinical.ClinicalTestData import org.transmartproject.db.dataquery2.DimensionImpl -import org.transmartproject.db.dataquery2.HypercubeValueImpl import org.transmartproject.db.dataquery2.query.Constraint import org.transmartproject.db.dataquery2.query.StudyConstraint import org.transmartproject.db.metadata.DimensionDescription @@ -15,7 +15,7 @@ import org.transmartproject.db.TestData import org.transmartproject.rest.protobuf.ObservationsSerializer import static org.hamcrest.MatcherAssert.assertThat -import static org.hamcrest.core.IsNull.notNullValue +import static org.hamcrest.Matchers.* /** @@ -33,22 +33,48 @@ class ObservationsBuilderTests { MultidimensionalDataResourceService queryResource @Test - public void testSerialization() throws Exception { + public void testJsonSerialization() throws Exception { setupData() Constraint constraint = new StudyConstraint(studyId: clinicalData.longitudinalStudy.studyId) def mockedCube = queryResource.retrieveData('clinical', [clinicalData.longitudinalStudy], constraint: constraint) - def builder = new ObservationsSerializer(mockedCube) - def blob = builder.getDimensionsDefs() - assertThat(blob, notNullValue()) - Iterator it = mockedCube.iterator - def cellMsgs = [] - while (it.hasNext()) { - HypercubeValueImpl value = it.next() - cellMsgs.add(builder.createCell(value)) - } - assertThat(cellMsgs, notNullValue()) - def footer = builder.getFooter() - assertThat(footer, notNullValue()) + def builder = new ObservationsSerializer(mockedCube, ObservationsSerializer.Format.JSON) + + def out = new ByteArrayOutputStream() + builder.write(out) + out.flush() + Collection result = new JsonSlurper().parse(out.toByteArray()) + + assertThat result.size(), is(14) + assertThat result, contains( + hasKey('dimensionDeclarations'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimensionIndexes'), + hasKey('dimension') + ) + } + + @Test + public void testProtobufSerialization() throws Exception { + setupData() + Constraint constraint = new StudyConstraint(studyId: clinicalData.longitudinalStudy.studyId) + def mockedCube = queryResource.retrieveData('clinical', [clinicalData.longitudinalStudy], constraint: constraint) + def builder = new ObservationsSerializer(mockedCube, ObservationsSerializer.Format.PROTOBUF) + + def out = new ByteArrayOutputStream() + builder.write(out) + out.flush() + + assertThat out.toByteArray(), not(empty()) } void setupData() { diff --git a/transmartApp/grails-app/conf/application.groovy b/transmartApp/grails-app/conf/application.groovy index 50f69ec57..9ca3d4c15 100644 --- a/transmartApp/grails-app/conf/application.groovy +++ b/transmartApp/grails-app/conf/application.groovy @@ -51,7 +51,8 @@ grails.mime.types = [html : [ ], form : 'application/x-www-form-urlencoded', multipartForm: 'multipart/form-data', - jnlp : 'application/x-java-jnlp-file' + jnlp : 'application/x-java-jnlp-file', + protobuf : 'application/x-protobuf', ] // The default codec used to encode data with ${} grails.views.javascript.library="jquery"