Skip to content

Commit

Permalink
Merge branch 'release-3.2.x' into TASK-5964
Browse files Browse the repository at this point in the history
  • Loading branch information
pfurio committed Jul 19, 2024
2 parents 04be875 + 9fae390 commit c21ef79
Show file tree
Hide file tree
Showing 41 changed files with 121 additions and 82 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
- name: Maven build
run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
- name: Run Junit tests
run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
- name: Publish Test Report on GitHub
uses: scacap/action-surefire-report@v1
env:
Expand Down
2 changes: 1 addition & 1 deletion opencga-analysis/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.testclassification.duration.MediumTests;
import org.opencb.opencga.core.tools.result.ExecutionResult;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery;
import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;

import java.io.File;
Expand Down Expand Up @@ -138,8 +139,11 @@ public void customAnalysisFromClinicalAnalysisTest() throws Exception {
.setConfig(config);

ExecutionResult result = customAnalysis.start();

checkInterpretation(238, result);
int expected = opencga.getVariantStorageManager().get(new VariantQuery(query),
new QueryOptions(QueryOptions.LIMIT, 500), clinicalTest.token)
.getResults().size();
assertNotEquals(0, expected);
checkInterpretation(expected, result);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -532,13 +532,20 @@ public void testCellbaseConfigure() throws Exception {
assertEquals("GRCh38", cellBaseUtils.getAssembly());

String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/";
String newCellbaseVersion = "v5.8";
String newCellbaseVersion = "v5.2";
String newCellbaseDataRelease = "1";

assertNotEquals(newCellbase, cellBaseUtils.getURL());
assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion());
assertNotEquals(newCellbaseDataRelease, cellBaseUtils.getDataRelease());

variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token);
variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, newCellbaseDataRelease, ""), false, null, token);
CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase();

assertEquals(newCellbase, cellbaseConfiguration.getUrl());
assertEquals(newCellbaseVersion, cellbaseConfiguration.getVersion());
assertEquals(newCellbaseDataRelease, cellbaseConfiguration.getDataRelease());

// assertTrue(family.getPedigreeGraph() != null);
}

Expand Down
2 changes: 1 addition & 1 deletion opencga-app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
10 changes: 1 addition & 9 deletions opencga-catalog/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down Expand Up @@ -200,14 +200,6 @@
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-client-runtime</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-batch</artifactId>
</dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt-jackson</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,13 @@ private void validateProjectForCreation(String organizationId, Project project)
throw new CatalogParameterException("Missing mandatory organism information");
}
try {
//TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints.
String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38")
? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38
: null;
CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(),
new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION,
ParamConstants.CELLBASE_DATA_RELEASE, ParamConstants.CELLBASE_APIKEY));
defaultDataRelease, ParamConstants.CELLBASE_APIKEY));
cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, project.getOrganism().getScientificName(),
project.getOrganism().getAssembly(), true);
project.setCellbase(cellBaseConfiguration);
Expand Down
2 changes: 1 addition & 1 deletion opencga-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion opencga-clinical/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand Down
2 changes: 1 addition & 1 deletion opencga-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public class ParamConstants {

public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase";
public static final String CELLBASE_VERSION = "v5.8";
public static final String CELLBASE_DATA_RELEASE = "7";
public static final String CELLBASE_DATA_RELEASE_GRCH38 = "7";
public static final String CELLBASE_APIKEY = "";

public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException
logger.warn("DataRelease not supported on version '" + serverVersion + ".x'");
} else {
String dataRelease = getDataRelease();
if (dataRelease == null) {
if (StringUtils.isEmpty(dataRelease)) {
if (autoComplete) {
cellBaseConfiguration.setDataRelease(getDefaultDataRelease());
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public class CellBaseConfiguration {
private String apiKey;

public CellBaseConfiguration() {
this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE,
this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE_GRCH38,
ParamConstants.CELLBASE_APIKEY);
}

Expand Down
2 changes: 1 addition & 1 deletion opencga-master/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>opencga</artifactId>
<groupId>org.opencb.opencga</groupId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public void testKill() throws Exception {
for (int i = 0; i < 10; i++) {
String jobId = "jobId-" + i;
// System.out.println("Checking status of job " + jobId);
while(!localExecutor.getStatus(jobId).equals("RUNNING")) {
Thread.sleep(10);
}
do {
Thread.sleep(100);
} while (!localExecutor.getStatus(jobId).equals("RUNNING"));
assertEquals("RUNNING", localExecutor.getStatus(jobId));
assertTrue(localExecutor.kill(jobId));
}
Expand Down
2 changes: 1 addition & 1 deletion opencga-server/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion opencga-storage/opencga-storage-app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion opencga-storage/opencga-storage-benchmark/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>opencga-storage</artifactId>
<groupId>org.opencb.opencga</groupId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion opencga-storage/opencga-storage-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public abstract class VariantStorageEngineBNDTest extends VariantStorageBaseTest
public void before() throws Exception {
variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
if (!loaded) {
clearDB(DB_NAME);
loadFiles();
Expand All @@ -58,7 +58,7 @@ public void before() throws Exception {
protected void loadFiles() throws Exception {
variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
studyMetadata = new StudyMetadata(1, "s1");
// variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical");
input1 = getResourceUri("variant-test-bnd.vcf");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public void before() throws Exception {
VariantStorageEngine variantStorageEngine = getVariantStorageEngine();
variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
dbAdaptor = variantStorageEngine.getDBAdaptor();
if (!loaded) {
loadFile();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,31 @@ public abstract class VariantQueryExecutorTest extends VariantStorageBaseTest {
private DBAdaptorVariantQueryExecutor dbQueryExecutor;
private List<VariantQueryExecutor> variantQueryExecutors;

@ClassRule
public static VariantSolrExternalResource solr = new VariantSolrExternalResource();
public static VariantSolrExternalResource solr = null;


public void initSolr() throws Exception {
if (solr != null) {
solr = new VariantSolrExternalResource();
solr.before();
}
}

@AfterClass
public static void afterClass() {
if (solr != null) {
solr.after();
}
}

@Before
public void setUp() throws Exception {

initSolr();
VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor();
VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager();
solr.configure(variantStorageEngine);
if (solr != null) {
solr.configure(variantStorageEngine);
}
if (!fileIndexed) {
studyMetadata = newStudyMetadata();
// variantSource = new VariantSource(smallInputUri.getPath(), "testAlias", "testStudy", "Study for testing purposes");
Expand Down Expand Up @@ -96,9 +112,11 @@ public void setUp() throws Exception {
variantStorageEngine.calculateStats(studyMetadata.getName(),
new ArrayList<>(cohorts.keySet()), options);

solr.configure(variantStorageEngine);
variantStorageEngine.secondaryIndex();
Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive());
if (solr != null) {
solr.configure(variantStorageEngine);
variantStorageEngine.secondaryIndex();
Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive());
}

variantQueryExecutors = variantStorageEngine.getVariantQueryExecutors();
dbQueryExecutor = null;
Expand Down Expand Up @@ -135,8 +153,8 @@ public void testXRefRs() throws StorageEngineException {
with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem(
with("GeneName", ConsequenceType::getGeneName,
is("TEX13B"))))));
matchers.put("COSV60260399", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem(
with("Cosmic", EvidenceEntry::getId, is("COSV60260399"))))));
matchers.put("RCV000155534", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem(
with("Clinvar", EvidenceEntry::getId, is("RCV000155534"))))));
matchers.put("ENST00000341832.11(ENSG00000248333):c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G")))));
matchers.put("ENST00000341832.11:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,15 +187,16 @@ public void testSpecialCharacter() throws Exception {
List<Variant> variants = getVariants(limit);
List<Variant> annotatedVariants = annotatedVariants(variants);

String studyId = "abyu12";
String fileId = "a.vcf";
String study = "abyu12";
String file = "a.vcf";

variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(fileId);
variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(file);
System.out.println(variants.get(0).getStudies().get(0).getFiles().get(0).getFileId());
//System.exit(-1);

scm.createStudy(studyId);

int studyId = scm.createStudy(study).getId();
int fileId = scm.registerFile(studyId, file, Arrays.asList("A-A", "B", "C", "D"));
scm.addIndexedFiles(studyId, Collections.singletonList(fileId));
String collection = solr.coreName;
variantSearchManager.create(collection);

Expand All @@ -204,13 +205,13 @@ public void testSpecialCharacter() throws Exception {
samplePosition.put("B", 1);
samplePosition.put("C", 2);
samplePosition.put("D", 3);
annotatedVariants.get(0).getStudies().get(0).setStudyId(studyId).setSortedSamplesPosition(samplePosition);
annotatedVariants.get(0).getStudies().get(0).setStudyId(study).setSortedSamplesPosition(samplePosition);
variantSearchManager.insert(collection, annotatedVariants);

Query query = new Query();
query.put(VariantQueryParam.STUDY.key(), studyId);
query.put(VariantQueryParam.STUDY.key(), study);
// query.put(VariantQueryParam.SAMPLE.key(), samplePosition.keySet().toArray()[0]);
query.put(VariantQueryParam.FILE.key(), fileId);
query.put(VariantQueryParam.FILE.key(), file);
query.put(VariantQueryParam.FILTER.key(), "PASS");
query.put(VariantQueryParam.ANNOT_CLINICAL_SIGNIFICANCE.key(), "benign");
VariantQueryResult<Variant> results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(query,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage-hadoop-compat</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage-hadoop-compat</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage-hadoop-compat</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage-hadoop-compat</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
<parent>
<groupId>org.opencb.opencga</groupId>
<artifactId>opencga-storage-hadoop</artifactId>
<version>3.2.0-SNAPSHOT</version>
<version>3.2.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
Loading

0 comments on commit c21ef79

Please sign in to comment.