diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java index 2ec05fe1d4..3cfb603172 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/CellBaseCliOptionsParser.java @@ -83,9 +83,6 @@ public class QueryCommandOptions { @Parameter(names = {"--group-by"}, description = "", required = false, arity = 1) public String groupBy; - @Parameter(names = {"--rank"}, description = "", required = false, arity = 1) - public String rank; - @Parameter(names = {"--distinct"}, description = "", required = false, arity = 1) public String distinct; diff --git a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/QueryCommandExecutor.java b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/QueryCommandExecutor.java index ee6d295c0d..fe0d189d84 100644 --- a/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/QueryCommandExecutor.java +++ b/cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/executors/QueryCommandExecutor.java @@ -276,12 +276,6 @@ private void executeFeatureAggregation(FeatureDBAdaptor featureDBAdaptor, Query return; } - if (queryCommandOptions.rank != null && !queryCommandOptions.rank.isEmpty()) { - CellBaseDataResult rank = featureDBAdaptor.rank(query, queryCommandOptions.rank, queryCommandOptions.limit, true); - output.println(objectMapper.writeValueAsString(rank)); - return; - } - if (queryCommandOptions.count) { CellBaseDataResult count = featureDBAdaptor.count(query); output.println(objectMapper.writeValueAsString(count)); diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/CellBaseDBAdaptor.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/CellBaseDBAdaptor.java index a21e5daec7..c49ae727a2 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/CellBaseDBAdaptor.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/CellBaseDBAdaptor.java @@ -53,11 +53,11 @@ default CellBaseDataResult distinct(String field) { CellBaseDataResult distinct(Query query, String field); - default CellBaseDataResult stats() { - return stats(new Query()); - } - - CellBaseDataResult stats(Query query); +// default CellBaseDataResult stats() { +// return stats(new Query()); +// } +// +// CellBaseDataResult stats(Query query); /* Main methods to query. @@ -97,7 +97,7 @@ default Iterator nativeIterator() { Iterator nativeIterator(Query query, QueryOptions options); - CellBaseDataResult rank(Query query, String field, int numResults, boolean asc); +// CellBaseDataResult rank(Query query, String field, int numResults, boolean asc); CellBaseDataResult groupBy(Query query, String field, QueryOptions options); diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/TranscriptDBAdaptor.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/TranscriptDBAdaptor.java index 0353cc34df..1c7c7f3a4d 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/TranscriptDBAdaptor.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/core/TranscriptDBAdaptor.java @@ -17,10 +17,6 @@ package org.opencb.cellbase.core.api.core; import org.opencb.commons.datastore.core.QueryParam; -import org.opencb.cellbase.core.result.CellBaseDataResult; - -import java.util.ArrayList; -import java.util.List; import static org.opencb.commons.datastore.core.QueryParam.Type.TEXT_ARRAY; @@ -51,7 +47,6 @@ enum QueryParams implements QueryParam { private Type type; private String description; - @Override public String key() { return key; @@ -68,14 +63,14 @@ public Type type() { } } - CellBaseDataResult getCdna(String id); - - default List> getCdna(List idList) { - List> cellBaseDataResults = new ArrayList<>(); - for (String id : idList) { - cellBaseDataResults.add(getCdna(id)); - } - return cellBaseDataResults; - } +// CellBaseDataResult getCdna(String id); +// +// default List> getCdna(List idList) { +// List> cellBaseDataResults = new ArrayList<>(); +// for (String id : idList) { +// cellBaseDataResults.add(getCdna(id)); +// } +// return cellBaseDataResults; +// } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java index e5e61d9a05..4342d53a91 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ClinicalMongoDBAdaptor.java @@ -58,10 +58,10 @@ public CellBaseDataResult nativeNext(Query query, QueryOptions options) { return null; } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { @@ -95,10 +95,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bson)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ConservationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ConservationMongoDBAdaptor.java index 256af6e780..f95c78616b 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ConservationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ConservationMongoDBAdaptor.java @@ -64,10 +64,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bson)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { @@ -91,10 +91,10 @@ public Iterator nativeIterator(Query query, QueryOptions options) { return mongoDBCollection.nativeQuery().find(bson, options); } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java index a5232c550f..5eb028ad33 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GeneMongoDBAdaptor.java @@ -96,10 +96,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bsonDocument)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions inputOptions) { @@ -164,10 +164,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java index 778e5e4ceb..ff0daaed47 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/GenomeMongoDBAdaptor.java @@ -286,10 +286,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bson)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { @@ -314,10 +314,10 @@ public Iterator nativeIterator(Query query, QueryOptions options) { return mongoDBCollection.nativeQuery().find(bson, options); } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java index 0260efc8a7..5371151f58 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MetaMongoDBAdaptor.java @@ -56,10 +56,10 @@ public CellBaseDataResult distinct(Query query, String field) { return null; } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { @@ -82,10 +82,10 @@ public Iterator nativeIterator(Query query, QueryOptions options) { return null; } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java index 6531874109..7b608bc754 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/MongoDBAdaptor.java @@ -17,9 +17,15 @@ package org.opencb.cellbase.lib.impl.core; import com.fasterxml.jackson.databind.ObjectMapper; -import org.bson.Document; +import com.mongodb.QueryBuilder; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import org.bson.*; import org.bson.conversions.Bson; +import org.opencb.biodata.models.core.Region; +import org.opencb.cellbase.core.common.IntervalFeatureFrequency; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; @@ -28,14 +34,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; +import java.math.BigInteger; +import java.util.*; public class MongoDBAdaptor { -// enum QueryValueType {INTEGER, STRING} + enum QueryValueType {INTEGER, STRING} protected String species; protected String assembly; @@ -70,125 +74,126 @@ private void initSpeciesAssembly(String species, String assembly) { } } } -// -// protected QueryOptions addPrivateExcludeOptions(QueryOptions options) { -// return addPrivateExcludeOptions(options, "_id,_chunkIds"); -// } -// -// protected QueryOptions addPrivateExcludeOptions(QueryOptions options, String csvFields) { -// if (options != null) { -// if (options.get("exclude") == null) { -// options.put("exclude", csvFields); -// } else { -// String exclude = options.getString("exclude"); -// if (exclude.contains(csvFields)) { -// return options; -// } else { -// options.put("exclude", exclude + "," + csvFields); -// } -// } -// } else { -// options = new QueryOptions("exclude", csvFields); -// } -// return options; -// } -// -// protected void createRegionQuery(Query query, String queryParam, List andBsonList) { -// if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { -// List regions = Region.parseRegions(query.getString(queryParam)); -// if (regions != null && regions.size() > 0) { -// // if there is only one region we add the AND filter directly to the andBsonList passed -// if (regions.size() == 1) { -// Bson chromosome = Filters.eq("chromosome", regions.get(0).getChromosome()); -// Bson start = Filters.lte("start", regions.get(0).getEnd()); -// Bson end = Filters.gte("end", regions.get(0).getStart()); -// andBsonList.add(Filters.and(chromosome, start, end)); -// } else { -// // when multiple regions then we create and OR list before add it to andBsonList -// List orRegionBsonList = new ArrayList<>(regions.size()); -// for (Region region : regions) { -// Bson chromosome = Filters.eq("chromosome", region.getChromosome()); -// Bson start = Filters.lte("start", region.getEnd()); -// Bson end = Filters.gte("end", region.getStart()); -// orRegionBsonList.add(Filters.and(chromosome, start, end)); -// } -// andBsonList.add(Filters.or(orRegionBsonList)); -// } -// } else { -// logger.warn("Region query no created, region object is null or empty."); -// } -// } -// } -// -// protected void createRegionQuery(Query query, String queryParam, int chunkSize, List andBsonList) { -// if (chunkSize <= 0) { -// // if chunkSize is not valid we call to the default method -// createRegionQuery(query, queryParam, andBsonList); -// } else { -// if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { -// List regions = Region.parseRegions(query.getString(queryParam)); -// if (regions != null && regions.size() > 0) { -// if (regions.size() == 1) { -// Bson chunkQuery = createChunkQuery(regions.get(0), chunkSize); -// andBsonList.add(chunkQuery); -// } else { -// // if multiple regions we add them first to a OR list -// List orRegionBsonList = new ArrayList<>(regions.size()); -// for (Region region : regions) { -// Bson chunkQuery = createChunkQuery(region, chunkSize); -// orRegionBsonList.add(chunkQuery); -// } -// andBsonList.add(Filters.or(orRegionBsonList)); -// } -// } -// } -// } -// } -// -// private Bson createChunkQuery(Region region, int chunkSize) { -// int startChunkId = getChunkId(region.getStart(), chunkSize); -// int endChunkId = getChunkId(region.getEnd(), chunkSize); -// -// List chunkIds = new ArrayList<>(endChunkId - startChunkId + 1); -// for (int chunkId = startChunkId; chunkId <= endChunkId; chunkId++) { -// chunkIds.add(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); -// logger.debug(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); -// } -// -// Bson chunk = Filters.in("_chunkIds", chunkIds); -// Bson start = Filters.lte("start", region.getEnd()); -// Bson end = Filters.gte("end", region.getStart()); -// return Filters.and(chunk, start, end); -// } -// -// protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList) { -// createOrQuery(query, queryParam, mongoDbField, andBsonList, QueryValueType.STRING); -// } -// -// protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList, -// QueryValueType queryValueType) { -// if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { -// switch (queryValueType) { -// case INTEGER: -// createOrQuery(query.getAsIntegerList(queryParam), mongoDbField, andBsonList); -// break; -// default: -// createOrQuery(query.getAsStringList(queryParam), mongoDbField, andBsonList); -// } -// } -// } -// -// protected void createOrQuery(List queryValues, String mongoDbField, List andBsonList) { -// if (queryValues.size() == 1) { -// andBsonList.add(Filters.eq(mongoDbField, queryValues.get(0))); -// } else { -// List orBsonList = new ArrayList<>(queryValues.size()); -// for (T queryItem : queryValues) { -// orBsonList.add(Filters.eq(mongoDbField, queryItem)); -// } -// andBsonList.add(Filters.or(orBsonList)); -// } -// } + + + protected QueryOptions addPrivateExcludeOptions(QueryOptions options) { + return addPrivateExcludeOptions(options, "_id,_chunkIds"); + } + + protected QueryOptions addPrivateExcludeOptions(QueryOptions options, String csvFields) { + if (options != null) { + if (options.get("exclude") == null) { + options.put("exclude", csvFields); + } else { + String exclude = options.getString("exclude"); + if (exclude.contains(csvFields)) { + return options; + } else { + options.put("exclude", exclude + "," + csvFields); + } + } + } else { + options = new QueryOptions("exclude", csvFields); + } + return options; + } + + protected void createRegionQuery(Query query, String queryParam, List andBsonList) { + if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { + List regions = Region.parseRegions(query.getString(queryParam)); + if (regions != null && regions.size() > 0) { + // if there is only one region we add the AND filter directly to the andBsonList passed + if (regions.size() == 1) { + Bson chromosome = Filters.eq("chromosome", regions.get(0).getChromosome()); + Bson start = Filters.lte("start", regions.get(0).getEnd()); + Bson end = Filters.gte("end", regions.get(0).getStart()); + andBsonList.add(Filters.and(chromosome, start, end)); + } else { + // when multiple regions then we create and OR list before add it to andBsonList + List orRegionBsonList = new ArrayList<>(regions.size()); + for (Region region : regions) { + Bson chromosome = Filters.eq("chromosome", region.getChromosome()); + Bson start = Filters.lte("start", region.getEnd()); + Bson end = Filters.gte("end", region.getStart()); + orRegionBsonList.add(Filters.and(chromosome, start, end)); + } + andBsonList.add(Filters.or(orRegionBsonList)); + } + } else { + logger.warn("Region query no created, region object is null or empty."); + } + } + } + + protected void createRegionQuery(Query query, String queryParam, int chunkSize, List andBsonList) { + if (chunkSize <= 0) { + // if chunkSize is not valid we call to the default method + createRegionQuery(query, queryParam, andBsonList); + } else { + if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { + List regions = Region.parseRegions(query.getString(queryParam)); + if (regions != null && regions.size() > 0) { + if (regions.size() == 1) { + Bson chunkQuery = createChunkQuery(regions.get(0), chunkSize); + andBsonList.add(chunkQuery); + } else { + // if multiple regions we add them first to a OR list + List orRegionBsonList = new ArrayList<>(regions.size()); + for (Region region : regions) { + Bson chunkQuery = createChunkQuery(region, chunkSize); + orRegionBsonList.add(chunkQuery); + } + andBsonList.add(Filters.or(orRegionBsonList)); + } + } + } + } + } + + private Bson createChunkQuery(Region region, int chunkSize) { + int startChunkId = getChunkId(region.getStart(), chunkSize); + int endChunkId = getChunkId(region.getEnd(), chunkSize); + + List chunkIds = new ArrayList<>(endChunkId - startChunkId + 1); + for (int chunkId = startChunkId; chunkId <= endChunkId; chunkId++) { + chunkIds.add(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); + logger.debug(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); + } + + Bson chunk = Filters.in("_chunkIds", chunkIds); + Bson start = Filters.lte("start", region.getEnd()); + Bson end = Filters.gte("end", region.getStart()); + return Filters.and(chunk, start, end); + } + + protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList) { + createOrQuery(query, queryParam, mongoDbField, andBsonList, QueryValueType.STRING); + } + + protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList, + QueryValueType queryValueType) { + if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { + switch (queryValueType) { + case INTEGER: + createOrQuery(query.getAsIntegerList(queryParam), mongoDbField, andBsonList); + break; + default: + createOrQuery(query.getAsStringList(queryParam), mongoDbField, andBsonList); + } + } + } + + protected void createOrQuery(List queryValues, String mongoDbField, List andBsonList) { + if (queryValues.size() == 1) { + andBsonList.add(Filters.eq(mongoDbField, queryValues.get(0))); + } else { + List orBsonList = new ArrayList<>(queryValues.size()); + for (T queryItem : queryValues) { + orBsonList.add(Filters.eq(mongoDbField, queryItem)); + } + andBsonList.add(Filters.or(orBsonList)); + } + } protected CellBaseDataResult groupBy(Bson query, String groupByField, String featureIdField, QueryOptions options) { Boolean count = options.getBoolean("count", false); @@ -202,80 +207,80 @@ protected CellBaseDataResult groupBy(Bson query, List groupByField, Stri return new CellBaseDataResult<>(mongoDBCollection.aggregate(groupBy, options)); } -// public CellBaseDataResult getIntervalFrequencies(Bson query, Region region, int intervalSize, QueryOptions options) { -// int interval = 50000; -// if (intervalSize > 0) { -// interval = intervalSize; -// } -// -// Bson match = Aggregates.match(query); -// BsonArray divide1 = new BsonArray(); -// divide1.add(new BsonString("$start")); -// divide1.add(new BsonInt32(interval)); -// -// BsonArray divide2 = new BsonArray(); -// divide2.add(new BsonDocument("$mod", divide1)); -// divide2.add(new BsonInt32(interval)); -// -// BsonArray subtractList = new BsonArray(); -// subtractList.add(new BsonDocument("$divide", divide1)); -// subtractList.add(new BsonDocument("$divide", divide2)); -// -// Document substract = new Document("$subtract", subtractList); -// Document totalCount = new Document("$sum", 1); -// -// Document g = new Document("_id", substract); -// g.append("features_count", totalCount); -// Document group = new Document("$group", g); -// -// Document sort = new Document("$sort", new Document("_id", 1)); -// -// CellBaseDataResult aggregationOutput = new CellBaseDataResult<>(mongoDBCollection.aggregate( -// Arrays.asList(match, group, sort), options)); -// -// Map ids = new HashMap<>(); -// for (Document intervalObj : aggregationOutput.getResults()) { -// Long id = Math.round((Double) intervalObj.get("_id")); //is double -// -// Document intervalVisited = ids.get(id); -// if (intervalVisited == null) { -// intervalObj.put("_id", id); -// intervalObj.put("chromosome", region.getChromosome()); -// intervalObj.put("start", getChunkStart(id.intValue(), interval)); -// intervalObj.put("end", getChunkEnd(id.intValue(), interval)); -// intervalObj.put("features_count", intervalObj.getInteger("features_count")); -// ids.put(id, intervalObj); -// } else { -// int sum = intervalVisited.getInteger("features_count") + intervalObj.getInteger("features_count"); -// intervalVisited.put("features_count", sum); -// } -// } -// -// List resultList = new ArrayList<>(); -// int firstChunkId = getChunkId(region.getStart(), interval); -// int lastChunkId = getChunkId(region.getEnd(), interval); -// Document intervalObj; -// for (int chunkId = firstChunkId; chunkId <= lastChunkId; chunkId++) { -// intervalObj = ids.get((long) chunkId); -// if (intervalObj == null) { -// intervalObj = new Document(); -// intervalObj.put("_id", chunkId); -// intervalObj.put("chromosome", region.getChromosome()); -// intervalObj.put("start", getChunkStart(chunkId, interval)); -// intervalObj.put("end", getChunkEnd(chunkId, interval)); -// intervalObj.put("features_count", 0); -// } else { -// intervalObj.put("features_count", Math.log(intervalObj.getInteger("features_count"))); -// } -// resultList.add(intervalObj); -// } -// -// CellBaseDataResult cellBaseDataResult = new CellBaseDataResult(); -// cellBaseDataResult.setResults(resultList); -// cellBaseDataResult.setId(region.toString()); -// cellBaseDataResult.setResultType("frequencies"); -// return cellBaseDataResult; -// } + public CellBaseDataResult getIntervalFrequencies(Bson query, Region region, int intervalSize, QueryOptions options) { + int interval = 50000; + if (intervalSize > 0) { + interval = intervalSize; + } + + Bson match = Aggregates.match(query); + BsonArray divide1 = new BsonArray(); + divide1.add(new BsonString("$start")); + divide1.add(new BsonInt32(interval)); + + BsonArray divide2 = new BsonArray(); + divide2.add(new BsonDocument("$mod", divide1)); + divide2.add(new BsonInt32(interval)); + + BsonArray subtractList = new BsonArray(); + subtractList.add(new BsonDocument("$divide", divide1)); + subtractList.add(new BsonDocument("$divide", divide2)); + + Document substract = new Document("$subtract", subtractList); + Document totalCount = new Document("$sum", 1); + + Document g = new Document("_id", substract); + g.append("features_count", totalCount); + Document group = new Document("$group", g); + + Document sort = new Document("$sort", new Document("_id", 1)); + + CellBaseDataResult aggregationOutput = new CellBaseDataResult<>(mongoDBCollection.aggregate( + Arrays.asList(match, group, sort), options)); + + Map ids = new HashMap<>(); + for (Document intervalObj : aggregationOutput.getResults()) { + Long id = Math.round((Double) intervalObj.get("_id")); //is double + + Document intervalVisited = ids.get(id); + if (intervalVisited == null) { + intervalObj.put("_id", id); + intervalObj.put("chromosome", region.getChromosome()); + intervalObj.put("start", getChunkStart(id.intValue(), interval)); + intervalObj.put("end", getChunkEnd(id.intValue(), interval)); + intervalObj.put("features_count", intervalObj.getInteger("features_count")); + ids.put(id, intervalObj); + } else { + int sum = intervalVisited.getInteger("features_count") + intervalObj.getInteger("features_count"); + intervalVisited.put("features_count", sum); + } + } + + List resultList = new ArrayList<>(); + int firstChunkId = getChunkId(region.getStart(), interval); + int lastChunkId = getChunkId(region.getEnd(), interval); + Document intervalObj; + for (int chunkId = firstChunkId; chunkId <= lastChunkId; chunkId++) { + intervalObj = ids.get((long) chunkId); + if (intervalObj == null) { + intervalObj = new Document(); + intervalObj.put("_id", chunkId); + intervalObj.put("chromosome", region.getChromosome()); + intervalObj.put("start", getChunkStart(chunkId, interval)); + intervalObj.put("end", getChunkEnd(chunkId, interval)); + intervalObj.put("features_count", 0); + } else { + intervalObj.put("features_count", Math.log(intervalObj.getInteger("features_count"))); + } + resultList.add(intervalObj); + } + + CellBaseDataResult cellBaseDataResult = new CellBaseDataResult(); + cellBaseDataResult.setResults(resultList); + cellBaseDataResult.setId(region.toString()); + cellBaseDataResult.setResultType("frequencies"); + return cellBaseDataResult; + } protected CellBaseDataResult executeDistinct(Object id, String fields, Document query) { CellBaseDataResult cellBaseDataResult = new CellBaseDataResult<>(mongoDBCollection.distinct(fields, query)); @@ -367,159 +372,159 @@ protected List executeAggregationist2(List return cellBaseDataResults; } -// protected String getChunkIdPrefix(String chromosome, int position, int chunkSize) { -// return chromosome + "_" + position / chunkSize + "_" + chunkSize / 1000 + "k"; -// } -// -// protected int getChunkId(int position, int chunkSize) { -// return position / chunkSize; -// } -// -// private int getChunkStart(int id, int chunkSize) { -// return (id == 0) ? 1 : id * chunkSize; -// } -// -// private int getChunkEnd(int id, int chunkSize) { -// return (id * chunkSize) + chunkSize - 1; -// } -// -// public CellBaseDataResult next(String chromosome, int position, QueryOptions options, MongoDBCollection mongoDBCollection) { -// QueryBuilder builder; -// if (options.getString("strand") == null || options.getString("strand").equals("") -// || (options.getString("strand").equals("1") || options.getString("strand").equals("+"))) { -// builder = QueryBuilder.start("chromosome").is(chromosome).and("start").greaterThanEquals(position); -// options.put("sort", new HashMap().put("start", "asc")); -// options.put("limit", 1); -// } else { -// builder = QueryBuilder.start("chromosome").is(chromosome).and("end").lessThanEquals(position); -// options.put("sort", new HashMap().put("end", "desc")); -// options.put("limit", 1); -// } -// return executeQuery("result", new Document(builder.get().toMap()), options, mongoDBCollection); -// } - -// @Deprecated -// protected QueryOptions addIncludeReturnFields(String returnField, QueryOptions options) { -// if (options != null) { //&& !options.getBoolean(returnField, true) -// if (options.getList("include") != null) { -//// options.put("include", options.get("include") + "," + returnField); -// options.getList("include").add(returnField); -// } else { -// options.put("include", Arrays.asList(returnField)); -// } -// } else { -// options = new QueryOptions("include", Arrays.asList(returnField)); -// } -// return options; -// } -// -// @Deprecated -// protected QueryOptions addExcludeReturnFields(String returnField, QueryOptions options) { -// if (options != null) { //&& !options.getBoolean(returnField, true)) { -// if (options.getList("exclude") != null) { -// List arr = options.getList("exclude"); -// arr.add(returnField); -// options.put("exclude", arr); -// } else { -// options.put("exclude", Arrays.asList(returnField)); -// } -// } else { -// options = new QueryOptions("exclude", Arrays.asList(returnField)); -// } -// return options; -// } -// -// @Deprecated -// protected Document getReturnFields(QueryOptions options) { -// // Select which fields are excluded and included in MongoDB query -// Document returnFields = new Document("_id", 0); -// if (options != null) { -// // Read and process 'exclude' field from 'options' object -// if (options != null && options.getList("include") != null && options.getList("include").size() > 0) { -// for (Object field : options.getList("include")) { -// returnFields.put(field.toString(), 1); -// } -// } else { -// if (options != null && options.getList("exclude") != null && options.getList("exclude").size() > 0) { -// for (Object field : options.getList("exclude")) { -// returnFields.put(field.toString(), 0); -// } -// } -// } -// } -// return returnFields; -// } -// -// /* -// * For histograms -// */ -// protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList, -// int numFeatures, double maxSnpsInterval) { -// -// int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; -// List intervalFeatureFrequenciesList = new ArrayList<>(numIntervals); -// -// float maxNormValue = 1; -// -// if (numFeatures != 0) { -// maxNormValue = (float) maxSnpsInterval / numFeatures; -// } -// -// int start = region.getStart(); -// int end = start + interval; -// for (int i = 0, j = 0; i < numIntervals; i++) { -// if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { -// if (numFeatures != 0) { -// intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, -// ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), -// (float) Math.log(((BigInteger) objectList.get(j)[1]).doubleValue()) / numFeatures / maxNormValue)); -// } else { // no features for this chromosome -// intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, -// ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), 0)); -// } -// j++; -// } else { -// intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); -// } -// start += interval; -// end += interval; -// } -// -// return intervalFeatureFrequenciesList; -// } -// -// -// protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList) { -// -// int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; -// List intervalFeatureFrequenciesList = new ArrayList(numIntervals); -// -// BigInteger max = new BigInteger("-1"); -// for (int i = 0; i < objectList.size(); i++) { -// if (((BigInteger) objectList.get(i)[1]).compareTo(max) > 0) { -// max = (BigInteger) objectList.get(i)[1]; -// } -// } -// -// int start = region.getStart(); -// int end = start + interval; -// for (int i = 0, j = 0; i < numIntervals; i++) { -// if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { -// intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, -// ((BigInteger) objectList.get(j)[0]).intValue(), -// ((BigInteger) objectList.get(j)[1]).intValue(), -// ((BigInteger) objectList.get(j)[1]).floatValue() / max.floatValue())); -// j++; -// } else { -// intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); -// } -// start += interval; -// end += interval; -// } -// -// return intervalFeatureFrequenciesList; -// } -// + protected String getChunkIdPrefix(String chromosome, int position, int chunkSize) { + return chromosome + "_" + position / chunkSize + "_" + chunkSize / 1000 + "k"; + } + + protected int getChunkId(int position, int chunkSize) { + return position / chunkSize; + } + + private int getChunkStart(int id, int chunkSize) { + return (id == 0) ? 1 : id * chunkSize; + } + + private int getChunkEnd(int id, int chunkSize) { + return (id * chunkSize) + chunkSize - 1; + } + + public CellBaseDataResult next(String chromosome, int position, QueryOptions options, MongoDBCollection mongoDBCollection) { + QueryBuilder builder; + if (options.getString("strand") == null || options.getString("strand").equals("") + || (options.getString("strand").equals("1") || options.getString("strand").equals("+"))) { + builder = QueryBuilder.start("chromosome").is(chromosome).and("start").greaterThanEquals(position); + options.put("sort", new HashMap().put("start", "asc")); + options.put("limit", 1); + } else { + builder = QueryBuilder.start("chromosome").is(chromosome).and("end").lessThanEquals(position); + options.put("sort", new HashMap().put("end", "desc")); + options.put("limit", 1); + } + return executeQuery("result", new Document(builder.get().toMap()), options, mongoDBCollection); + } + + @Deprecated + protected QueryOptions addIncludeReturnFields(String returnField, QueryOptions options) { + if (options != null) { //&& !options.getBoolean(returnField, true) + if (options.getList("include") != null) { +// options.put("include", options.get("include") + "," + returnField); + options.getList("include").add(returnField); + } else { + options.put("include", Arrays.asList(returnField)); + } + } else { + options = new QueryOptions("include", Arrays.asList(returnField)); + } + return options; + } + + @Deprecated + protected QueryOptions addExcludeReturnFields(String returnField, QueryOptions options) { + if (options != null) { //&& !options.getBoolean(returnField, true)) { + if (options.getList("exclude") != null) { + List arr = options.getList("exclude"); + arr.add(returnField); + options.put("exclude", arr); + } else { + options.put("exclude", Arrays.asList(returnField)); + } + } else { + options = new QueryOptions("exclude", Arrays.asList(returnField)); + } + return options; + } + + @Deprecated + protected Document getReturnFields(QueryOptions options) { + // Select which fields are excluded and included in MongoDB query + Document returnFields = new Document("_id", 0); + if (options != null) { + // Read and process 'exclude' field from 'options' object + if (options != null && options.getList("include") != null && options.getList("include").size() > 0) { + for (Object field : options.getList("include")) { + returnFields.put(field.toString(), 1); + } + } else { + if (options != null && options.getList("exclude") != null && options.getList("exclude").size() > 0) { + for (Object field : options.getList("exclude")) { + returnFields.put(field.toString(), 0); + } + } + } + } + return returnFields; + } + + /* + * For histograms + */ + protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList, + int numFeatures, double maxSnpsInterval) { + + int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; + List intervalFeatureFrequenciesList = new ArrayList<>(numIntervals); + + float maxNormValue = 1; + + if (numFeatures != 0) { + maxNormValue = (float) maxSnpsInterval / numFeatures; + } + + int start = region.getStart(); + int end = start + interval; + for (int i = 0, j = 0; i < numIntervals; i++) { + if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { + if (numFeatures != 0) { + intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, + ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), + (float) Math.log(((BigInteger) objectList.get(j)[1]).doubleValue()) / numFeatures / maxNormValue)); + } else { // no features for this chromosome + intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, + ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), 0)); + } + j++; + } else { + intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); + } + start += interval; + end += interval; + } + + return intervalFeatureFrequenciesList; + } + + + protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList) { + + int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; + List intervalFeatureFrequenciesList = new ArrayList(numIntervals); + + BigInteger max = new BigInteger("-1"); + for (int i = 0; i < objectList.size(); i++) { + if (((BigInteger) objectList.get(i)[1]).compareTo(max) > 0) { + max = (BigInteger) objectList.get(i)[1]; + } + } + + int start = region.getStart(); + int end = start + interval; + for (int i = 0, j = 0; i < numIntervals; i++) { + if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { + intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, + ((BigInteger) objectList.get(j)[0]).intValue(), + ((BigInteger) objectList.get(j)[1]).intValue(), + ((BigInteger) objectList.get(j)[1]).floatValue() / max.floatValue())); + j++; + } else { + intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); + } + start += interval; + end += interval; + } + + return intervalFeatureFrequenciesList; + } + public String getSpecies() { return species; diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java index f16ed65499..9d221e90c2 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinMongoDBAdaptor.java @@ -208,10 +208,10 @@ public CellBaseDataResult getVariantAnnotation(String return cellBaseDataResult; } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { @@ -242,10 +242,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, document)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinProteinInteractionMongoDBAdaptor.java index 89039f94b0..0efd4de939 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/ProteinProteinInteractionMongoDBAdaptor.java @@ -44,10 +44,10 @@ public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, logger.debug("ProteinProteinInteractionMongoDBAdaptor: in 'constructor'"); } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { @@ -74,10 +74,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, parseQuery(query))); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java index 34bae1502f..9bf67b770f 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RegulationMongoDBAdaptor.java @@ -57,10 +57,10 @@ public CellBaseDataResult nativeNext(Query query, QueryOptions options) { return null; } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { @@ -101,10 +101,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult<>(mongoDBCollection.distinct(field, bsonDocument)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions inputOptions) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java index ec0b88adcc..623567949a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/RepeatsMongoDBAdaptor.java @@ -62,10 +62,10 @@ public CellBaseDataResult distinct(Query query, String field) { return null; } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions inputOptions) { @@ -95,10 +95,10 @@ public Iterator nativeIterator(Query query, QueryOptions options) { return null; } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java index fd3ffff374..edf69f76ce 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/TranscriptMongoDBAdaptor.java @@ -16,21 +16,23 @@ package org.opencb.cellbase.lib.impl.core; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; +import org.apache.commons.lang3.StringUtils; import org.bson.Document; import org.bson.conversions.Bson; +import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.Transcript; import org.opencb.cellbase.core.api.core.TranscriptDBAdaptor; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDataStore; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.function.Consumer; /** @@ -45,33 +47,55 @@ public TranscriptMongoDBAdaptor(String species, String assembly, MongoDataStore logger.debug("TranscriptMongoDBAdaptor: in 'constructor'"); } - @Override - public CellBaseDataResult getCdna(String id) { - Bson bson = Filters.eq("transcripts.xrefs.id", id); - Bson elemMatch = Projections.elemMatch("transcripts", Filters.eq("xrefs.id", id)); - Bson include = Projections.include("transcripts.cDnaSequence"); - // elemMatch and include are combined to reduce the data sent from the server - Bson projection = Projections.fields(elemMatch, include); - CellBaseDataResult result = new CellBaseDataResult<>(mongoDBCollection.find(bson, projection, new QueryOptions())); - - String sequence = null; - if (result != null && !result.getResults().isEmpty()) { - List transcripts = (List) result.getResults().get(0).get("transcripts"); - sequence = transcripts.get(0).getString("cDnaSequence"); - } - return new CellBaseDataResult<>(id, result.getTime(), result.getEvents(), result.getNumResults(), - Collections.singletonList(sequence), 1); - } +// @Override +// public CellBaseDataResult getCdna(String id) { +// Bson bson = Filters.eq("transcripts.xrefs.id", id); +// Bson elemMatch = Projections.elemMatch("transcripts", Filters.eq("xrefs.id", id)); +// Bson include = Projections.include("transcripts.cDnaSequence"); +// // elemMatch and include are combined to reduce the data sent from the server +// Bson projection = Projections.fields(elemMatch, include); +// CellBaseDataResult result = new CellBaseDataResult<>(mongoDBCollection.find(bson, projection, new QueryOptions())); +// +// String sequence = null; +// if (result != null && !result.getResults().isEmpty()) { +// List transcripts = (List) result.getResults().get(0).get("transcripts"); +// sequence = transcripts.get(0).getString("cDnaSequence"); +// } +// return new CellBaseDataResult<>(id, result.getTime(), result.getEvents(), result.getNumResults(), +// Collections.singletonList(sequence), 1); +// } @Override public CellBaseDataResult update(List objectList, String field, String[] innerFields) { return null; } - public CellBaseDataResult count(Bson match, Bson include, Bson unwind, Bson match2, Bson project, Bson group) { - CellBaseDataResult cellBaseDataResult = - new CellBaseDataResult<>(mongoDBCollection.aggregate(Arrays.asList(match, include, unwind, match2, project, group), - null)); + @Override + public CellBaseDataResult count(Query query) { + Bson document = parseQuery(query); + Bson match = Aggregates.match(document); + + List includeFields = new ArrayList<>(); + for (String s : query.keySet()) { + if (StringUtils.isNotEmpty(query.getString(s))) { + includeFields.add(s); + } + } + + Bson include; + if (includeFields.size() > 0) { + include = Aggregates.project(Projections.include(includeFields)); + } else { + include = Aggregates.project(Projections.include("transcripts.id")); + } + + Bson unwind = Aggregates.unwind("$transcripts"); + Bson match2 = Aggregates.match(document); + Bson project = Aggregates.project(new Document("transcripts", "$transcripts.id")); + Bson group = Aggregates.group("transcripts", Accumulators.sum("count", 1)); + + CellBaseDataResult cellBaseDataResult = + new CellBaseDataResult<>(mongoDBCollection.aggregate(Arrays.asList(match, include, unwind, match2, project, group), null)); Number number = (Number) cellBaseDataResult.first().get("count"); Long count = number.longValue(); return new CellBaseDataResult<>(null, cellBaseDataResult.getTime(), cellBaseDataResult.getEvents(), @@ -84,10 +108,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult(mongoDBCollection.distinct(field, bsonDocument)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { @@ -117,22 +141,22 @@ public void forEach(Query query, Consumer action, QueryOptions options) { } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } + @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; + public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { + Bson bsonQuery = parseQuery(query); + return groupBy(bsonQuery, field, "name", options); } -// @Override -// public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { -// Bson bsonQuery = parseQuery(query); -// return groupBy(bsonQuery, field, "name", options); -// } -// -// @Override -// public CellBaseDataResult groupBy(Query query, List fields, QueryOptions options) { -// Bson bsonQuery = parseQuery(query); -// return groupBy(bsonQuery, fields, "name", options); -// } + @Override + public CellBaseDataResult groupBy(Query query, List fields, QueryOptions options) { + Bson bsonQuery = parseQuery(query); + return groupBy(bsonQuery, fields, "name", options); + } @Override public CellBaseDataResult next(Query query, QueryOptions options) { @@ -144,15 +168,136 @@ public CellBaseDataResult nativeNext(Query query, QueryOptions options) { return null; } -// @Override -// public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, QueryOptions options) { -// if (query.getString("region") != null) { -// Region region = Region.parseRegion(query.getString("region")); -// Bson bsonDocument = parseQuery(query); -// return getIntervalFrequencies(bsonDocument, region, intervalSize, options); + @Override + public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, QueryOptions options) { + if (query.getString("region") != null) { + Region region = Region.parseRegion(query.getString("region")); + Bson bsonDocument = parseQuery(query); + return getIntervalFrequencies(bsonDocument, region, intervalSize, options); + } + return null; + } + + private Bson parseQuery(Query query) { + List andBsonList = new ArrayList<>(); + + createRegionQuery(query, TranscriptDBAdaptor.QueryParams.REGION.key(), MongoDBCollectionConfiguration.GENE_CHUNK_SIZE, andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.ID.key(), "transcripts.id", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.NAME.key(), "transcripts.name", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.BIOTYPE.key(), "transcripts.biotype", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.XREFS.key(), "transcripts.xrefs.id", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.TFBS_NAME.key(), "transcripts.tfbs.name", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.ANNOTATION_FLAGS.key(), "transcripts.annotationFlags", andBsonList); + if (andBsonList.size() > 0) { + return Filters.and(andBsonList); + } else { + return new Document(); + } + } + + private Bson parseQueryUnwindTranscripts(Query query) { + List andBsonList = new ArrayList<>(); + + createRegionQuery(query, TranscriptDBAdaptor.QueryParams.REGION.key(), andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.ID.key(), "id", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.NAME.key(), "name", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.BIOTYPE.key(), "biotype", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.XREFS.key(), "xrefs.id", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.TFBS_NAME.key(), "tfbs.name", andBsonList); + createOrQuery(query, TranscriptDBAdaptor.QueryParams.ANNOTATION_FLAGS.key(), "annotationFlags", andBsonList); + + if (andBsonList.size() > 0) { + return Filters.and(andBsonList); + } else { + return new Document(); + } + } + + private List unwindAndMatchTranscripts(Query query, QueryOptions options) { + List aggregateList = new ArrayList<>(); + + Bson bson = parseQuery(query); + Bson match = Aggregates.match(bson); + + Bson include = null; + if (options != null && options.containsKey("include")) { + List includeList = new ArrayList<>(); + List optionsAsStringList = options.getAsStringList("include"); + for (String s : optionsAsStringList) { + if (s.startsWith("transcripts")) { + includeList.add(s); + } + } + + if (includeList.size() > 0) { + include = Projections.include(includeList); + } + } + + if (include == null) { + include = Projections.include("transcripts"); + } + Bson excludeAndInclude = Aggregates.project(Projections.fields(Projections.excludeId(), include)); + Bson unwind = Aggregates.unwind("$transcripts"); + + // This project the three fields of Xref to the top of the object + Document document = new Document("id", "$transcripts.id"); + document.put("name", "$transcripts.name"); + document.put("biotype", "$transcripts.biotype"); + document.put("status", "$transcripts.status"); + document.put("chromosome", "$transcripts.chromosome"); + document.put("start", "$transcripts.start"); + document.put("end", "$transcripts.end"); + document.put("strand", "$transcripts.strand"); + document.put("genomicCodingStart", "$transcripts.genomicCodingStart"); + document.put("genomicCodingEnd", "$transcripts.genomicCodingEnd"); + document.put("cdnaCodingStart", "$transcripts.cdnaCodingStart"); + document.put("cdnaCodingEnd", "$transcripts.cdnaCodingEnd"); + document.put("cdsLength", "$transcripts.cdsLength"); + document.put("proteinID", "$transcripts.proteinID"); + document.put("proteinSequence", "$transcripts.proteinSequence"); + document.put("cDnaSequence", "$transcripts.cDnaSequence"); + document.put("xrefs", "$transcripts.xrefs"); + document.put("exons", "$transcripts.exons"); + document.put("annotationFlags", "$transcripts.annotationFlags"); + Bson project = Aggregates.project(document); + + Bson match2 = Aggregates.match(bson); + + aggregateList.add(match); + aggregateList.add(unwind); + aggregateList.add(match2); + aggregateList.add(excludeAndInclude); + aggregateList.add(project); + + return aggregateList; + } + +// +// private CellBaseDataResult getCdna(String id) { +// Bson bson = Filters.eq("transcripts.xrefs.id", id); +// Bson elemMatch = Projections.elemMatch("transcripts", Filters.eq("xrefs.id", id)); +// Bson include = Projections.include("transcripts.cDnaSequence"); +// // elemMatch and include are combined to reduce the data sent from the server +// Bson projection = Projections.fields(elemMatch, include); +//// CellBaseDataResult result = new CellBaseDataResult<>(mongoDBCollection.find(bson, projection, new QueryOptions())); +// +// CellBaseDataResult result = new CellBaseDataResult<>(mongoDBCollection.find(bson, projection, new QueryOptions())); +// +// String sequence = null; +// if (result != null && !result.getResults().isEmpty()) { +// List transcripts = (List) result.getResults().get(0).get("transcripts"); +// sequence = transcripts.get(0).getString("cDnaSequence"); // } -// return null; +// return new CellBaseDataResult<>(id, result.getTime(), result.getEvents(), result.getNumResults(), +// Collections.singletonList(sequence), 1); // } - +// private List> getCdna(List idList) { +// List> cellBaseDataResults = new ArrayList<>(); +// for (String id : idList) { +// cellBaseDataResults.add(getCdna(id)); +// } +// return cellBaseDataResults; +// } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java index 7c00085e03..4fd66584a6 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/VariantMongoDBAdaptor.java @@ -136,10 +136,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult(mongoDBCollection.distinct(field, document)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions inputOptions) { @@ -202,10 +202,10 @@ public void forEach(Query query, Consumer action, QueryOptions o } } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java index 5bd67f2147..4f2d0b25fd 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/core/XRefMongoDBAdaptor.java @@ -79,10 +79,10 @@ public CellBaseDataResult distinct(Query query, String field) { return new CellBaseDataResult(mongoDBCollection.distinct(field, bson)); } - @Override - public CellBaseDataResult stats(Query query) { - return null; - } +// @Override +// public CellBaseDataResult stats(Query query) { +// return null; +// } @Override public CellBaseDataResult get(Query query, QueryOptions options) { @@ -124,10 +124,10 @@ public Iterator nativeIterator(Query query, QueryOptions options) { return mongoDBCollection.nativeQuery().find(bson, options); } - @Override - public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { - return null; - } +// @Override +// public CellBaseDataResult rank(Query query, String field, int numResults, boolean asc) { +// return null; +// } @Override public CellBaseDataResult groupBy(Query query, String field, QueryOptions options) { diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java index 0cfdd069c6..3000b3ceda 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/AbstractManager.java @@ -17,25 +17,16 @@ package org.opencb.cellbase.lib.managers; import com.fasterxml.jackson.databind.ObjectWriter; -import com.mongodb.QueryBuilder; -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Filters; -import org.bson.*; -import org.bson.conversions.Bson; -import org.opencb.biodata.models.core.Region; import org.opencb.cellbase.core.api.core.DBAdaptorFactory; -import org.opencb.cellbase.core.common.IntervalFeatureFrequency; import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.impl.core.MongoDBAdaptorFactory; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; -import java.util.*; +import java.util.ArrayList; +import java.util.List; public class AbstractManager { @@ -49,8 +40,6 @@ public class AbstractManager { protected Logger logger; - enum QueryValueType {INTEGER, STRING} - public static final int DEFAULT_LIMIT = 10; protected int histogramIntervalSize = 200000; public AbstractManager(CellBaseConfiguration configuration) { @@ -118,302 +107,4 @@ protected boolean hasHistogramQueryParam(QueryOptions queryOptions) { return Boolean.parseBoolean(getHistogramParameter(queryOptions)); } - - - protected QueryOptions addPrivateExcludeOptions(QueryOptions options) { - return addPrivateExcludeOptions(options, "_id,_chunkIds"); - } - - protected QueryOptions addPrivateExcludeOptions(QueryOptions options, String csvFields) { - if (options != null) { - if (options.get("exclude") == null) { - options.put("exclude", csvFields); - } else { - String exclude = options.getString("exclude"); - if (exclude.contains(csvFields)) { - return options; - } else { - options.put("exclude", exclude + "," + csvFields); - } - } - } else { - options = new QueryOptions("exclude", csvFields); - } - return options; - } - - protected void createRegionQuery(Query query, String queryParam, List andBsonList) { - if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { - List regions = Region.parseRegions(query.getString(queryParam)); - if (regions != null && regions.size() > 0) { - // if there is only one region we add the AND filter directly to the andBsonList passed - if (regions.size() == 1) { - Bson chromosome = Filters.eq("chromosome", regions.get(0).getChromosome()); - Bson start = Filters.lte("start", regions.get(0).getEnd()); - Bson end = Filters.gte("end", regions.get(0).getStart()); - andBsonList.add(Filters.and(chromosome, start, end)); - } else { - // when multiple regions then we create and OR list before add it to andBsonList - List orRegionBsonList = new ArrayList<>(regions.size()); - for (Region region : regions) { - Bson chromosome = Filters.eq("chromosome", region.getChromosome()); - Bson start = Filters.lte("start", region.getEnd()); - Bson end = Filters.gte("end", region.getStart()); - orRegionBsonList.add(Filters.and(chromosome, start, end)); - } - andBsonList.add(Filters.or(orRegionBsonList)); - } - } else { - logger.warn("Region query no created, region object is null or empty."); - } - } - } - - protected void createRegionQuery(Query query, String queryParam, int chunkSize, List andBsonList) { - if (chunkSize <= 0) { - // if chunkSize is not valid we call to the default method - createRegionQuery(query, queryParam, andBsonList); - } else { - if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { - List regions = Region.parseRegions(query.getString(queryParam)); - if (regions != null && regions.size() > 0) { - if (regions.size() == 1) { - Bson chunkQuery = createChunkQuery(regions.get(0), chunkSize); - andBsonList.add(chunkQuery); - } else { - // if multiple regions we add them first to a OR list - List orRegionBsonList = new ArrayList<>(regions.size()); - for (Region region : regions) { - Bson chunkQuery = createChunkQuery(region, chunkSize); - orRegionBsonList.add(chunkQuery); - } - andBsonList.add(Filters.or(orRegionBsonList)); - } - } - } - } - } - - private Bson createChunkQuery(Region region, int chunkSize) { - int startChunkId = getChunkId(region.getStart(), chunkSize); - int endChunkId = getChunkId(region.getEnd(), chunkSize); - - List chunkIds = new ArrayList<>(endChunkId - startChunkId + 1); - for (int chunkId = startChunkId; chunkId <= endChunkId; chunkId++) { - chunkIds.add(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); - logger.debug(region.getChromosome() + "_" + chunkId + "_" + chunkSize / 1000 + "k"); - } - - Bson chunk = Filters.in("_chunkIds", chunkIds); - Bson start = Filters.lte("start", region.getEnd()); - Bson end = Filters.gte("end", region.getStart()); - return Filters.and(chunk, start, end); - } - - protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList) { - createOrQuery(query, queryParam, mongoDbField, andBsonList, QueryValueType.STRING); - } - - protected void createOrQuery(Query query, String queryParam, String mongoDbField, List andBsonList, - QueryValueType queryValueType) { - if (query != null && query.getString(queryParam) != null && !query.getString(queryParam).isEmpty()) { - switch (queryValueType) { - case INTEGER: - createOrQuery(query.getAsIntegerList(queryParam), mongoDbField, andBsonList); - break; - default: - createOrQuery(query.getAsStringList(queryParam), mongoDbField, andBsonList); - } - } - } - - protected void createOrQuery(List queryValues, String mongoDbField, List andBsonList) { - if (queryValues.size() == 1) { - andBsonList.add(Filters.eq(mongoDbField, queryValues.get(0))); - } else { - List orBsonList = new ArrayList<>(queryValues.size()); - for (T queryItem : queryValues) { - orBsonList.add(Filters.eq(mongoDbField, queryItem)); - } - andBsonList.add(Filters.or(orBsonList)); - } - } - - public CellBaseDataResult getIntervalFrequencies(Bson query, Region region, int intervalSize, QueryOptions options) { - int interval = 50000; - if (intervalSize > 0) { - interval = intervalSize; - } - - Bson match = Aggregates.match(query); - BsonArray divide1 = new BsonArray(); - divide1.add(new BsonString("$start")); - divide1.add(new BsonInt32(interval)); - - BsonArray divide2 = new BsonArray(); - divide2.add(new BsonDocument("$mod", divide1)); - divide2.add(new BsonInt32(interval)); - - BsonArray subtractList = new BsonArray(); - subtractList.add(new BsonDocument("$divide", divide1)); - subtractList.add(new BsonDocument("$divide", divide2)); - - Document substract = new Document("$subtract", subtractList); - Document totalCount = new Document("$sum", 1); - - Document g = new Document("_id", substract); - g.append("features_count", totalCount); - Document group = new Document("$group", g); - - Document sort = new Document("$sort", new Document("_id", 1)); - - CellBaseDataResult aggregationOutput = new CellBaseDataResult<>(mongoDBCollection.aggregate( - Arrays.asList(match, group, sort), options)); - - Map ids = new HashMap<>(); - for (Document intervalObj : aggregationOutput.getResults()) { - Long id = Math.round((Double) intervalObj.get("_id")); //is double - - Document intervalVisited = ids.get(id); - if (intervalVisited == null) { - intervalObj.put("_id", id); - intervalObj.put("chromosome", region.getChromosome()); - intervalObj.put("start", getChunkStart(id.intValue(), interval)); - intervalObj.put("end", getChunkEnd(id.intValue(), interval)); - intervalObj.put("features_count", intervalObj.getInteger("features_count")); - ids.put(id, intervalObj); - } else { - int sum = intervalVisited.getInteger("features_count") + intervalObj.getInteger("features_count"); - intervalVisited.put("features_count", sum); - } - } - - List resultList = new ArrayList<>(); - int firstChunkId = getChunkId(region.getStart(), interval); - int lastChunkId = getChunkId(region.getEnd(), interval); - Document intervalObj; - for (int chunkId = firstChunkId; chunkId <= lastChunkId; chunkId++) { - intervalObj = ids.get((long) chunkId); - if (intervalObj == null) { - intervalObj = new Document(); - intervalObj.put("_id", chunkId); - intervalObj.put("chromosome", region.getChromosome()); - intervalObj.put("start", getChunkStart(chunkId, interval)); - intervalObj.put("end", getChunkEnd(chunkId, interval)); - intervalObj.put("features_count", 0); - } else { - intervalObj.put("features_count", Math.log(intervalObj.getInteger("features_count"))); - } - resultList.add(intervalObj); - } - - CellBaseDataResult cellBaseDataResult = new CellBaseDataResult(); - cellBaseDataResult.setResults(resultList); - cellBaseDataResult.setId(region.toString()); - cellBaseDataResult.setResultType("frequencies"); - return cellBaseDataResult; - } - - protected String getChunkIdPrefix(String chromosome, int position, int chunkSize) { - return chromosome + "_" + position / chunkSize + "_" + chunkSize / 1000 + "k"; - } - - protected int getChunkId(int position, int chunkSize) { - return position / chunkSize; - } - - private int getChunkStart(int id, int chunkSize) { - return (id == 0) ? 1 : id * chunkSize; - } - - private int getChunkEnd(int id, int chunkSize) { - return (id * chunkSize) + chunkSize - 1; - } - - public CellBaseDataResult next(String chromosome, int position, QueryOptions options, MongoDBCollection mongoDBCollection) { - QueryBuilder builder; - if (options.getString("strand") == null || options.getString("strand").equals("") - || (options.getString("strand").equals("1") || options.getString("strand").equals("+"))) { - builder = QueryBuilder.start("chromosome").is(chromosome).and("start").greaterThanEquals(position); - options.put("sort", new HashMap().put("start", "asc")); - options.put("limit", 1); - } else { - builder = QueryBuilder.start("chromosome").is(chromosome).and("end").lessThanEquals(position); - options.put("sort", new HashMap().put("end", "desc")); - options.put("limit", 1); - } - return executeQuery("result", new Document(builder.get().toMap()), options, mongoDBCollection); - } - - - /* - * For histograms - */ - protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList, - int numFeatures, double maxSnpsInterval) { - - int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; - List intervalFeatureFrequenciesList = new ArrayList<>(numIntervals); - - float maxNormValue = 1; - - if (numFeatures != 0) { - maxNormValue = (float) maxSnpsInterval / numFeatures; - } - - int start = region.getStart(); - int end = start + interval; - for (int i = 0, j = 0; i < numIntervals; i++) { - if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { - if (numFeatures != 0) { - intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, - ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), - (float) Math.log(((BigInteger) objectList.get(j)[1]).doubleValue()) / numFeatures / maxNormValue)); - } else { // no features for this chromosome - intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, - ((BigInteger) objectList.get(j)[0]).intValue(), ((BigInteger) objectList.get(j)[1]).intValue(), 0)); - } - j++; - } else { - intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); - } - start += interval; - end += interval; - } - - return intervalFeatureFrequenciesList; - } - - - protected List getIntervalFeatureFrequencies(Region region, int interval, List objectList) { - - int numIntervals = (region.getEnd() - region.getStart()) / interval + 1; - List intervalFeatureFrequenciesList = new ArrayList(numIntervals); - - BigInteger max = new BigInteger("-1"); - for (int i = 0; i < objectList.size(); i++) { - if (((BigInteger) objectList.get(i)[1]).compareTo(max) > 0) { - max = (BigInteger) objectList.get(i)[1]; - } - } - - int start = region.getStart(); - int end = start + interval; - for (int i = 0, j = 0; i < numIntervals; i++) { - if (j < objectList.size() && ((BigInteger) objectList.get(j)[0]).intValue() == i) { - intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, ((BigInteger) objectList.get(j)[0]).intValue(), - ((BigInteger) objectList.get(j)[1]).intValue(), - ((BigInteger) objectList.get(j)[1]).floatValue() / max.floatValue())); - j++; - } else { - intervalFeatureFrequenciesList.add(new IntervalFeatureFrequency(start, end, i, 0, 0.0f)); - } - start += interval; - end += interval; - } - - return intervalFeatureFrequenciesList; - } - - } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java index 6fc46aa15a..15e04ac416 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/CellBaseManagerFactory.java @@ -39,6 +39,7 @@ public class CellBaseManagerFactory { private Map regulatoryManagers; private Map xrefManagers; private Map repeatsManagers; + private MetaManager metaManager; private Logger logger; public CellBaseManagerFactory(CellBaseConfiguration configuration) { @@ -62,18 +63,17 @@ private String getMultiKey(String species, String assembly) { public GeneManager getGeneManager(String species) throws CellbaseException { if (species == null) { - logger.error("Failed to get GeneManager. Species is required."); - return null; + throw new CellbaseException("Failed to get GeneManager. Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getGeneManager(species, assembly.getName()); } - public GeneManager getGeneManager(String species, String assembly) { + public GeneManager getGeneManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!geneManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } geneManagers.put(multiKey, new GeneManager(species, assembly, configuration)); } @@ -104,18 +104,17 @@ private boolean validateSpeciesAssembly(String species, String assembly) { public TranscriptManager getTranscriptManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getTranscriptManager(species, assembly.getName()); } - public TranscriptManager getTranscriptManager(String species, String assembly) { + public TranscriptManager getTranscriptManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!transcriptManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } transcriptManagers.put(multiKey, new TranscriptManager(species, assembly, configuration)); } @@ -124,18 +123,17 @@ public TranscriptManager getTranscriptManager(String species, String assembly) { public VariantManager getVariantManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getVariantManager(species, assembly.getName()); } - public VariantManager getVariantManager(String species, String assembly) { + public VariantManager getVariantManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!variantManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } variantManagers.put(multiKey, new VariantManager(species, assembly, configuration)); } @@ -144,18 +142,17 @@ public VariantManager getVariantManager(String species, String assembly) { public ProteinManager getProteinManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getProteinManager(species, assembly.getName()); } - public ProteinManager getProteinManager(String species, String assembly) { + public ProteinManager getProteinManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!proteinManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } proteinManagers.put(multiKey, new ProteinManager(species, assembly, configuration)); } @@ -164,18 +161,17 @@ public ProteinManager getProteinManager(String species, String assembly) { public GenomeManager getGenomeManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getGenomeManager(species, assembly.getName()); } - public GenomeManager getGenomeManager(String species, String assembly) { + public GenomeManager getGenomeManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!genomeManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } genomeManagers.put(multiKey, new GenomeManager(species, assembly, configuration)); } @@ -184,18 +180,17 @@ public GenomeManager getGenomeManager(String species, String assembly) { public ClinicalManager getClinicalManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getClinicalManager(species, assembly.getName()); } - public ClinicalManager getClinicalManager(String species, String assembly) { + public ClinicalManager getClinicalManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!clinicalManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } clinicalManagers.put(multiKey, new ClinicalManager(species, assembly, configuration)); } @@ -204,18 +199,17 @@ public ClinicalManager getClinicalManager(String species, String assembly) { public RegulatoryManager getRegulatoryManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getRegulatoryManager(species, assembly.getName()); } - public RegulatoryManager getRegulatoryManager(String species, String assembly) { + public RegulatoryManager getRegulatoryManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!regulatoryManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } regulatoryManagers.put(multiKey, new RegulatoryManager(species, assembly, configuration)); } @@ -224,18 +218,17 @@ public RegulatoryManager getRegulatoryManager(String species, String assembly) { public XrefManager getXrefManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getXrefManager(species, assembly.getName()); } - public XrefManager getXrefManager(String species, String assembly) { + public XrefManager getXrefManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!xrefManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } xrefManagers.put(multiKey, new XrefManager(species, assembly, configuration)); } @@ -244,21 +237,24 @@ public XrefManager getXrefManager(String species, String assembly) { public RepeatsManager getRepeatsManager(String species) throws CellbaseException { if (species == null) { - logger.error("Species is required."); - return null; + throw new CellbaseException("Species is required."); } SpeciesConfiguration.Assembly assembly = SpeciesUtils.getDefaultAssembly(configuration, species); return getRepeatsManager(species, assembly.getName()); } - public RepeatsManager getRepeatsManager(String species, String assembly) { + public RepeatsManager getRepeatsManager(String species, String assembly) throws CellbaseException { String multiKey = getMultiKey(species, assembly); if (!repeatsManagers.containsKey(multiKey)) { if (!validateSpeciesAssembly(species, assembly)) { - return null; + throw new CellbaseException("Invalid species " + species + " or assembly " + assembly); } repeatsManagers.put(multiKey, new RepeatsManager(species, assembly, configuration)); } return repeatsManagers.get(multiKey); } + + public MetaManager getMetaManager() { + return metaManager; + } } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/GeneManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/GeneManager.java index f80460d3f2..adbe32c0f3 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/GeneManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/GeneManager.java @@ -141,4 +141,7 @@ public List getGeneByEnsemblId(QueryOptions queryOptions, St return queryResults; } + + + } diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/MetaManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/MetaManager.java new file mode 100644 index 0000000000..925425dac3 --- /dev/null +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/MetaManager.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.cellbase.lib.managers; + +import org.opencb.cellbase.core.api.core.CellBaseDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; + +public class MetaManager extends AbstractManager { + + private CellBaseDBAdaptor cellBaseDBAdaptor; + + public MetaManager(CellBaseConfiguration configuration) { + super(configuration); + this.init(); + } + + private void init() { + cellBaseDBAdaptor = dbAdaptorFactory.getMetaDBAdaptor(species, assembly); + } + + public CellBaseDataResult getVersions(QueryOptions queryOptions) { + return cellBaseDBAdaptor.nativeGet(new Query(), queryOptions); + } +} diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java index 8675b07c39..07954651c1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/managers/TranscriptManager.java @@ -16,21 +16,12 @@ package org.opencb.cellbase.lib.managers; -import com.mongodb.client.model.Accumulators; -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Gene; -import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.Transcript; import org.opencb.cellbase.core.api.core.GeneDBAdaptor; import org.opencb.cellbase.core.api.core.TranscriptDBAdaptor; import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.result.CellBaseDataResult; -import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -54,16 +45,14 @@ private void init() { // geneDBAdaptor = dbAdaptorFactory.getGeneDBAdaptor(this.species, this.assembly); } - - public CellBaseDataResult getCdna(String id) { GeneDBAdaptor geneDBAdaptor = dbAdaptorFactory.getGeneDBAdaptor(this.species, this.assembly); CellBaseDataResult gene = geneDBAdaptor .get(new Query("xrefs", id), new QueryOptions(QueryOptions.INCLUDE, "transcripts.id,transcripts.cDnaSequence")); - if (gene.getResults().get(0).getTranscripts().size() != 1) { - // check id exists - } +// if (gene.getResults().get(0).getTranscripts().size() != 1) { +// // check id exists +// } String cdnaSequence = null; for (Transcript transcript: gene.getResults().get(0).getTranscripts()) { @@ -76,6 +65,8 @@ public CellBaseDataResult getCdna(String id) { return new CellBaseDataResult<>(id, gene.getTime(), gene.getEvents(), gene.getNumResults(), Collections.singletonList(cdnaSequence), 1); + + // Bson bson = Filters.eq("transcripts.xrefs.id", id); // Bson elemMatch = Projections.elemMatch("transcripts", Filters.eq("xrefs.id", id)); // Bson include = Projections.include("transcripts.cDnaSequence"); @@ -92,6 +83,13 @@ public CellBaseDataResult getCdna(String id) { // Collections.singletonList(sequence), 1); } + private List> getCdna(List idList) { + List> cellBaseDataResults = new ArrayList<>(); + for (String id : idList) { + cellBaseDataResults.add(getCdna(id)); + } + return cellBaseDataResults; + } public CellBaseDataResult search(Query query, QueryOptions queryOptions) { CellBaseDataResult queryResult = transcriptDBAdaptor.nativeGet(query, queryOptions); @@ -115,9 +113,9 @@ public List info(Query query, QueryOptions queryOptions, Str return queryResults; } - public List getSequence(String id) { + public List> getSequence(String id) { List transcriptsList = Arrays.asList(id.split(",")); - List queryResult = transcriptDBAdaptor.getCdna(transcriptsList); + List> queryResult = getCdna(transcriptsList); for (int i = 0; i < transcriptsList.size(); i++) { queryResult.get(i).setId(transcriptsList.get(i)); } @@ -133,142 +131,5 @@ public List getByRegion(Query query, QueryOptions queryOptio return queryResults; } - public CellBaseDataResult count(Query query) { - GeneDBAdaptor geneDBAdaptor = dbAdaptorFactory.getGeneDBAdaptor(this.species); - CellBaseDataResult cellBaseDataResult1 = geneDBAdaptor.get(query, new QueryOptions(QueryOptions.COUNT)); - return cellBaseDataResult1.getNumMatches(); - - Bson document = parseQuery(query); - Bson match = Aggregates.match(document); - - List includeFields = new ArrayList<>(); - for (String s : query.keySet()) { - if (StringUtils.isNotEmpty(query.getString(s))) { - includeFields.add(s); - } - } - - Bson include; - if (includeFields.size() > 0) { - include = Aggregates.project(Projections.include(includeFields)); - } else { - include = Aggregates.project(Projections.include("transcripts.id")); - } - - Bson unwind = Aggregates.unwind("$transcripts"); - Bson match2 = Aggregates.match(document); - Bson project = Aggregates.project(new Document("transcripts", "$transcripts.id")); - Bson group = Aggregates.group("transcripts", Accumulators.sum("count", 1)); - - CellBaseDataResult cellBaseDataResult = transcriptDBAdaptor.count(match, include, unwind, match2, project, group); - Number number = (Number) cellBaseDataResult.first().get("count"); - Long count = number.longValue(); - return new CellBaseDataResult<>(null, cellBaseDataResult.getTime(), cellBaseDataResult.getEvents(), - cellBaseDataResult.getNumResults(), Collections.singletonList(count), cellBaseDataResult.getNumMatches()); - } - - private Bson parseQuery(Query query) { - List andBsonList = new ArrayList<>(); - - createRegionQuery(query, TranscriptDBAdaptor.QueryParams.REGION.key(), MongoDBCollectionConfiguration.GENE_CHUNK_SIZE, andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.ID.key(), "transcripts.id", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.NAME.key(), "transcripts.name", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.BIOTYPE.key(), "transcripts.biotype", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.XREFS.key(), "transcripts.xrefs.id", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.TFBS_NAME.key(), "transcripts.tfbs.name", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.ANNOTATION_FLAGS.key(), "transcripts.annotationFlags", andBsonList); - if (andBsonList.size() > 0) { - return Filters.and(andBsonList); - } else { - return new Document(); - } - } - - private Bson parseQueryUnwindTranscripts(Query query) { - List andBsonList = new ArrayList<>(); - - createRegionQuery(query, TranscriptDBAdaptor.QueryParams.REGION.key(), andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.ID.key(), "id", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.NAME.key(), "name", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.BIOTYPE.key(), "biotype", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.XREFS.key(), "xrefs.id", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.TFBS_NAME.key(), "tfbs.name", andBsonList); - createOrQuery(query, TranscriptDBAdaptor.QueryParams.ANNOTATION_FLAGS.key(), "annotationFlags", andBsonList); - - if (andBsonList.size() > 0) { - return Filters.and(andBsonList); - } else { - return new Document(); - } - } - - private List unwindAndMatchTranscripts(Query query, QueryOptions options) { - List aggregateList = new ArrayList<>(); - - Bson bson = parseQuery(query); - Bson match = Aggregates.match(bson); - - Bson include = null; - if (options != null && options.containsKey("include")) { - List includeList = new ArrayList<>(); - List optionsAsStringList = options.getAsStringList("include"); - for (String s : optionsAsStringList) { - if (s.startsWith("transcripts")) { - includeList.add(s); - } - } - - if (includeList.size() > 0) { - include = Projections.include(includeList); - } - } - - if (include == null) { - include = Projections.include("transcripts"); - } - Bson excludeAndInclude = Aggregates.project(Projections.fields(Projections.excludeId(), include)); - Bson unwind = Aggregates.unwind("$transcripts"); - - // This project the three fields of Xref to the top of the object - Document document = new Document("id", "$transcripts.id"); - document.put("name", "$transcripts.name"); - document.put("biotype", "$transcripts.biotype"); - document.put("status", "$transcripts.status"); - document.put("chromosome", "$transcripts.chromosome"); - document.put("start", "$transcripts.start"); - document.put("end", "$transcripts.end"); - document.put("strand", "$transcripts.strand"); - document.put("genomicCodingStart", "$transcripts.genomicCodingStart"); - document.put("genomicCodingEnd", "$transcripts.genomicCodingEnd"); - document.put("cdnaCodingStart", "$transcripts.cdnaCodingStart"); - document.put("cdnaCodingEnd", "$transcripts.cdnaCodingEnd"); - document.put("cdsLength", "$transcripts.cdsLength"); - document.put("proteinID", "$transcripts.proteinID"); - document.put("proteinSequence", "$transcripts.proteinSequence"); - document.put("cDnaSequence", "$transcripts.cDnaSequence"); - document.put("xrefs", "$transcripts.xrefs"); - document.put("exons", "$transcripts.exons"); - document.put("annotationFlags", "$transcripts.annotationFlags"); - Bson project = Aggregates.project(document); - - Bson match2 = Aggregates.match(bson); - - aggregateList.add(match); - aggregateList.add(unwind); - aggregateList.add(match2); - aggregateList.add(excludeAndInclude); - aggregateList.add(project); - - return aggregateList; - } - - public CellBaseDataResult getIntervalFrequencies(Query query, int intervalSize, QueryOptions options) { - if (query.getString("region") != null) { - Region region = Region.parseRegion(query.getString("region")); - Bson bsonDocument = parseQuery(query); - return getIntervalFrequencies(bsonDocument, region, intervalSize, options); - } - return null; - } } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java index ea7b433989..c0e4b20a63 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/GenericRestWSServer.java @@ -159,7 +159,8 @@ private void init() throws IOException, CellbaseException { cellBaseManagerFactory = new CellBaseManagerFactory(cellBaseConfiguration); // Initialize Monitor - monitor = new Monitor(dbAdaptorFactory); + // TODO move to manager +// monitor = new Monitor(dbAdaptorFactory); } } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java index 350772ee08..7a80c7dafb 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/MetaWSServer.java @@ -18,13 +18,14 @@ import io.swagger.annotations.*; import org.apache.commons.lang3.StringUtils; -import org.opencb.cellbase.core.api.core.CellBaseDBAdaptor; +import org.opencb.cellbase.core.ParamConstants; import org.opencb.cellbase.core.common.GitRepositoryState; import org.opencb.cellbase.core.config.DownloadProperties; import org.opencb.cellbase.core.config.SpeciesProperties; import org.opencb.cellbase.core.exception.CellbaseException; import org.opencb.cellbase.core.monitor.HealthStatus; import org.opencb.cellbase.core.result.CellBaseDataResult; +import org.opencb.cellbase.lib.managers.MetaManager; import org.opencb.cellbase.server.exception.SpeciesException; import org.opencb.cellbase.server.exception.VersionException; import org.opencb.cellbase.server.rest.clinical.ClinicalWSServer; @@ -37,8 +38,6 @@ import org.opencb.cellbase.server.rest.genomic.VariantWSServer; import org.opencb.cellbase.server.rest.regulatory.RegulatoryWSServer; import org.opencb.cellbase.server.rest.regulatory.TfWSServer; -import org.opencb.commons.datastore.core.Query; -import org.opencb.commons.datastore.core.QueryOptions; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -58,6 +57,8 @@ @Api(value = "Meta", description = "Meta RESTful Web Services API") public class MetaWSServer extends GenericRestWSServer { + private MetaManager metaManager; + private static final String PONG = "pong"; private static final String STATUS = "status"; private static final String HEALTH = "health"; @@ -69,22 +70,21 @@ public MetaWSServer(@PathParam("version") @Context UriInfo uriInfo, @Context HttpServletRequest hsr) throws VersionException, SpeciesException, IOException, CellbaseException { super(version, uriInfo, hsr); + + metaManager = cellBaseManagerFactory.getMetaManager(); } @GET @Path("/{species}/versions") @ApiOperation(httpMethod = "GET", value = "Returns source version metadata, including source urls from which " - + "data files were downloaded.", - response = DownloadProperties.class, responseContainer = "QueryResponse") + + "data files were downloaded.", response = DownloadProperties.class, responseContainer = "QueryResponse") public Response getVersion(@PathParam("species") - @ApiParam(name = "species", - value = "Name of the species, e.g.: hsapiens. For a full list of potentially" - + "available species ids, please refer to: " - + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species", - required = true) String species) { - CellBaseDBAdaptor metaDBAdaptor = dbAdaptorFactory.getMetaDBAdaptor(species, this.assembly); - return createOkResponse(metaDBAdaptor.nativeGet(new Query(), new QueryOptions())); + @ApiParam(name = "species", value = ParamConstants.SPECIES_DESCRIPTION, required = true) String species) + throws CellbaseException { + parseQueryParams(); + CellBaseDataResult queryResults = metaManager.getVersions(queryOptions); + return createOkResponse(queryResults); } @GET diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java index 0434bc7977..2bedab7109 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/IdWSServer.java @@ -22,7 +22,6 @@ import org.bson.Document; import org.opencb.biodata.models.core.Xref; import org.opencb.cellbase.core.ParamConstants; -import org.opencb.cellbase.core.api.core.XRefDBAdaptor; import org.opencb.cellbase.core.exception.CellbaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.managers.GeneManager; diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java index 83399783a9..8c133e3859 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/feature/TranscriptWSServer.java @@ -247,7 +247,7 @@ public Response getSequencesByIdList(@PathParam("transcripts") @ApiParam(name = required = true) String id) { try { parseQueryParams(); - List queryResults = transcriptManager.getSequence(id); + List> queryResults = transcriptManager.getSequence(id); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java index 7965425555..3436571f9b 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/genomic/VariantWSServer.java @@ -18,18 +18,14 @@ import io.swagger.annotations.*; import org.opencb.biodata.models.variant.Variant; -import org.opencb.biodata.models.variant.avro.Score; import org.opencb.biodata.models.variant.avro.VariantAnnotation; -import org.opencb.cellbase.core.CellBaseDataResponse; import org.opencb.cellbase.core.ParamConstants; -import org.opencb.cellbase.core.api.core.VariantDBAdaptor; import org.opencb.cellbase.core.exception.CellbaseException; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.lib.managers.VariantManager; import org.opencb.cellbase.server.exception.SpeciesException; import org.opencb.cellbase.server.exception.VersionException; import org.opencb.cellbase.server.rest.GenericRestWSServer; -import org.opencb.commons.datastore.core.QueryOptions; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -38,7 +34,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/network/ProteinProteinInteractionWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/network/ProteinProteinInteractionWSServer.java index 0d81d2d600..fe99a3b794 100644 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/network/ProteinProteinInteractionWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/network/ProteinProteinInteractionWSServer.java @@ -17,23 +17,21 @@ package org.opencb.cellbase.server.rest.network; import org.opencb.biodata.models.protein.Interaction; -import org.opencb.cellbase.core.api.core.ProteinProteinInteractionDBAdaptor; import org.opencb.cellbase.core.exception.CellbaseException; -import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.cellbase.server.exception.SpeciesException; import org.opencb.cellbase.server.exception.VersionException; import org.opencb.cellbase.server.rest.GenericRestWSServer; -import org.opencb.commons.datastore.core.Query; - import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.io.IOException; -import java.util.List; /** * Created with IntelliJ IDEA. @@ -58,53 +56,53 @@ public Response getModel() { return createModelResponse(Interaction.class); } - @GET - @Path("/all") - public Response getAllPPI(@DefaultValue("") @QueryParam("interactor") String interactor, - @DefaultValue("") @QueryParam("type") String type, - @DefaultValue("") @QueryParam("database") String database, - @DefaultValue("") @QueryParam("status") String status, - @DefaultValue("") @QueryParam("detectionMethod") String detectionMethod) { - try { - parseQueryParams(); - ProteinProteinInteractionDBAdaptor ppiDBAdaptor = - dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); - return createOkResponse(ppiDBAdaptor.nativeGet(query, queryOptions)); - } catch (Exception e) { - return createErrorResponse(e); - } - } - - @GET - @Path("/{interaction}/info") - public Response getPPIByInteractionId(@PathParam("interaction") String interaction) { - try { - parseQueryParams(); - ProteinProteinInteractionDBAdaptor ppiDBAdaptor = - dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); - List queries = createQueries(interaction, ProteinProteinInteractionDBAdaptor.QueryParams.INTERACTOR_A_XREFS.key()); - List queryResults = ppiDBAdaptor.nativeGet(queries, queryOptions); - return createOkResponse(queryResults); - } catch (Exception e) { - return createErrorResponse(e); - } - } - - // TODO Correct method - @GET - @Path("/{interaction}/interactors") - public Response getInteractorsByInteractionId(@PathParam("interaction") String interaction, - @DefaultValue("interactorA,interactorB") @QueryParam("include") String include) { - try { - parseQueryParams(); - ProteinProteinInteractionDBAdaptor ppiDBAdaptor = - dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); - queryOptions.put("include", "interactorA,interactorB"); - List queries = createQueries(interaction, ProteinProteinInteractionDBAdaptor.QueryParams.INTERACTOR_A_XREFS.key()); - List queryResults = ppiDBAdaptor.nativeGet(queries, queryOptions); - return createOkResponse(queryResults); - } catch (Exception e) { - return createErrorResponse(e); - } - } +// @GET +// @Path("/all") +// public Response getAllPPI(@DefaultValue("") @QueryParam("interactor") String interactor, +// @DefaultValue("") @QueryParam("type") String type, +// @DefaultValue("") @QueryParam("database") String database, +// @DefaultValue("") @QueryParam("status") String status, +// @DefaultValue("") @QueryParam("detectionMethod") String detectionMethod) { +// try { +// parseQueryParams(); +// ProteinProteinInteractionDBAdaptor ppiDBAdaptor = +// dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); +// return createOkResponse(ppiDBAdaptor.nativeGet(query, queryOptions)); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } +// +// @GET +// @Path("/{interaction}/info") +// public Response getPPIByInteractionId(@PathParam("interaction") String interaction) { +// try { +// parseQueryParams(); +// ProteinProteinInteractionDBAdaptor ppiDBAdaptor = +// dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); +// List queries = createQueries(interaction, ProteinProteinInteractionDBAdaptor.QueryParams.INTERACTOR_A_XREFS.key()); +// List queryResults = ppiDBAdaptor.nativeGet(queries, queryOptions); +// return createOkResponse(queryResults); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } +// +// // TODO Correct method +// @GET +// @Path("/{interaction}/interactors") +// public Response getInteractorsByInteractionId(@PathParam("interaction") String interaction, +// @DefaultValue("interactorA,interactorB") @QueryParam("include") String include) { +// try { +// parseQueryParams(); +// ProteinProteinInteractionDBAdaptor ppiDBAdaptor = +// dbAdaptorFactory.getProteinProteinInteractionDBAdaptor(this.species, this.assembly); +// queryOptions.put("include", "interactorA,interactorB"); +// List queries = createQueries(interaction, ProteinProteinInteractionDBAdaptor.QueryParams.INTERACTOR_A_XREFS.key()); +// List queryResults = ppiDBAdaptor.nativeGet(queries, queryOptions); +// return createOkResponse(queryResults); +// } catch (Exception e) { +// return createErrorResponse(e); +// } +// } } diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java index 40f047c673..d33e9c8a5d 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/rest/regulatory/TfWSServer.java @@ -54,8 +54,8 @@ public TfWSServer(@PathParam("apiVersion") throws VersionException, SpeciesException, IOException, CellbaseException { super(apiVersion, species, uriInfo, hsr); - regulatoryManager = cellBaseManagerFactory.getRegulatoryManager(); - geneManager = cellBaseManagerFactory.getGeneManager(); + regulatoryManager = cellBaseManagerFactory.getRegulatoryManager(species, assembly); + geneManager = cellBaseManagerFactory.getGeneManager(species, assembly); } @GET @@ -80,7 +80,7 @@ public Response getAllByTfbs(@PathParam("tf") @ApiParam(name = "tf", value = Par parseIncludesAndExcludes(exclude, include, sort); parseLimitAndSkip(limit, skip); parseQueryParams(); - List queryResults = regulatoryManager.getAllByTfbs(query, queryOptions, species, assembly, tf); + List queryResults = regulatoryManager.getAllByTfbs(query, queryOptions, tf); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e); @@ -122,7 +122,7 @@ public Response getEnsemblGenes(@PathParam("tf") @ApiParam(name = "tf", value = parseIncludesAndExcludes(exclude, include, sort); parseLimitAndSkip(limit, skip); parseQueryParams(); - List queryResults = geneManager.getByTf(query, queryOptions, species, assembly, tf); + List queryResults = geneManager.getByTf(query, queryOptions, tf); return createOkResponse(queryResults); } catch (Exception e) { return createErrorResponse(e);