Skip to content

Commit

Permalink
storage: Fix wrong columns at sample+file sql query build. #TASK-5876
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-coll
j-coll committed Apr 18, 2024
1 parent b10b5c7 commit 4d20481
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 26 deletions.
7 changes: 7 additions & 0 deletions ...e-core/src/main/java/org/opencb/opencga/storage/core/variant/query/VariantQueryUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1542,6 +1542,13 @@ public static String printQuery(Query query) {
query.put(ANNOT_GENE_REGIONS.key(), "numGeneRegions : " + ((Collection<?>) geneRegions).size());
}
}
if (isValidParam(query, ID_INTERSECT)) {
query = new Query(query);
Object idIntersect = query.get(ID_INTERSECT.key());
if (idIntersect instanceof Collection) {
query.put(ID_INTERSECT.key(), "numIdIntersect : " + ((Collection<?>) idIntersect).size());
}
}
try {
return QUERY_MAPPER.writeValueAsString(query);
} catch (JsonProcessingException e) {
Expand Down
8 changes: 5 additions & 3 deletions ...rg/opencb/opencga/storage/core/variant/query/projection/VariantQueryProjectionParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,15 @@ public VariantQueryProjection parseVariantQueryProjection(Query query, QueryOpti
for (CohortMetadata cohort : metadataManager.getCalculatedOrInvalidCohorts(studyId)) {
cohorts.add(cohort.getId());
if (cohort.isInvalid()) {
String message = "Cohort '" + cohort.getName() + "' has outdated variant stats.";
String message = "Please note that the Cohort Stats for "
+ "'" + study.getName() + ":" + cohort.getName() + "' are currently outdated.";
int numSampmles = cohort.getSamples().size();
int invalidStatsNumSamples = cohort.getAttributes().getInt(CohortMetadata.INVALID_STATS_NUM_SAMPLES, -1);
if (invalidStatsNumSamples > 0) {
message += " Existing stats are calculated with " + invalidStatsNumSamples + " samples, "
+ "but the cohort currently has " + numSampmles + " samples.";
message += " The statistics have been calculated with " + invalidStatsNumSamples + " samples, "
+ "while the total number of samples in the cohort is " + numSampmles + ".";
}
message += " To display updated statistics, please execute variant-stats-index.";
events.add(new Event(Event.Type.WARNING, message));
}
}
Expand Down
4 changes: 3 additions & 1 deletion ...test/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManagerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,9 @@ public void queryInvalidStats() throws Exception {

VariantQueryResult<Variant> result = variantStorageEngine.get(new Query(), new QueryOptions(QueryOptions.LIMIT, 1));
assertEquals(1, result.getEvents().size());
assertEquals("Cohort 'cohort2' has outdated variant stats", result.getEvents().get(0).getMessage());
assertEquals("Please note that the Cohort Stats for '1000g:cohort2' are currently outdated." +
" The statistics have been calculated with 2 samples, while the total number of samples in the cohort is 4." +
" To display updated statistics, please execute variant-stats-index.", result.getEvents().get(0).getMessage());
}

@Test
Expand Down
40 changes: 20 additions & 20 deletions ...ava/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantSqlQueryParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -974,7 +974,8 @@ protected void addVariantFilters(ParsedVariantQuery variantQuery, QueryOptions o
sampleFiles.addAll(fileIdsFromSampleId);
} else {
for (Pair<Integer, Integer> fileIdPair : fileIds) {
if (fileIdPair.getKey().equals(studyId)) {
if (fileIdPair.getKey().equals(studyId)
&& sampleMetadata.getFiles().contains(fileIdPair.getValue())) {
sampleFiles.add(fileIdPair.getValue());
}
}
Expand Down Expand Up @@ -1231,27 +1232,24 @@ private void addSampleDataFilter(ParsedVariantQuery query, List<String> filters,
SampleMetadata sampleMetadata = sampleDataFilter.getKey();

boolean multiFileSample = VariantStorageEngine.SplitData.MULTI.equals(sampleMetadata.getSplitData());
// First file does not have the fileID in the column name
Integer firstSampleFile = sampleMetadata.getFiles().get(0);
List<Integer> sampleFiles = new ArrayList<>();
if (multiFileSample) {
if (fileIds.isEmpty()) {
sampleFiles.add(null); // First file does not have the fileID in the column name
List<Integer> fileIdsFromSampleId = sampleMetadata.getFiles();
sampleFiles.addAll(fileIdsFromSampleId.subList(1, fileIdsFromSampleId.size()));
sampleFiles.addAll(fileIdsFromSampleId);
} else {
for (Pair<Integer, Integer> fileIdPair : fileIds) {
if (fileIdPair.getKey().equals(sampleMetadata.getStudyId())) {
Integer fileId = fileIdPair.getValue();
int idx = sampleMetadata.getFiles().indexOf(fileId);
if (idx == 0) {
sampleFiles.add(null); // First file does not have the fileID in the column name
} else if (idx > 0) {
sampleFiles.add(fileId); // First file does not have the fileID in the column name
}
if (fileIdPair.getKey().equals(sampleMetadata.getStudyId())
&& sampleMetadata.getFiles().contains(fileIdPair.getValue())) {
sampleFiles.add(fileIdPair.getValue());
}
}
}
} else {
sampleFiles.add(null); // First file does not have the fileID in the column name
// Non multi file sample
sampleFiles.add(firstSampleFile);
}
for (Integer sampleFile : sampleFiles) {
List<String> sampleFileFilters = new LinkedList<>();
Expand All @@ -1271,10 +1269,11 @@ private void addSampleDataFilter(ParsedVariantQuery query, List<String> filters,
sb.append("TO_NUMBER(");
}
sb.append('"');
if (sampleFile != null) {
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb);
} else {
if (Objects.equals(sampleFile, firstSampleFile)) {
// Special scenario for the first file. Column name does not contain the fileId
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sb);
} else {
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb);
}
sb.append('"');

Expand All @@ -1288,10 +1287,11 @@ private void addSampleDataFilter(ParsedVariantQuery query, List<String> filters,

if (op.startsWith(">>") || op.startsWith("<<")) {
sb.append(" OR \"");
if (sampleFile != null) {
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb);
} else {
if (Objects.equals(sampleFile, firstSampleFile)) {
// Special scenario for the first file. Column name does not contain the fileId
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sb);
} else {
buildSampleColumnKey(sampleMetadata.getStudyId(), sampleMetadata.getId(), sampleFile, sb);
}
sb.append('"');

Expand All @@ -1308,8 +1308,8 @@ private void addSampleDataFilter(ParsedVariantQuery query, List<String> filters,
if (multiFileSample) {
// The first file is null. Get the actual fileId
Integer actualFileId;
if (sampleFile == null) {
actualFileId = sampleMetadata.getFiles().get(0);
if (Objects.equals(sampleFile, firstSampleFile)) {
actualFileId = firstSampleFile;
} else {
actualFileId = sampleFile;
}
Expand Down
4 changes: 2 additions & 2 deletions ...src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,8 +369,8 @@ public void regenerateSampleIndex() throws Exception {
}

public VariantQueryResult<Variant> dbAdaptorQuery(Query query, QueryOptions options) {
query = variantStorageEngine.preProcessQuery(query, options);
return dbAdaptor.get(query, options);
ParsedVariantQuery variantQuery = variantStorageEngine.parseQuery(query, options);
return dbAdaptor.get(variantQuery);
}

@Test
Expand Down

0 comments on commit 4d20481

Please sign in to comment.