Skip to content

Commit

Permalink
Merge branch 'develop' into TASK-6281
Browse files Browse the repository at this point in the history
  • Loading branch information
juanfeSanahuja committed Jun 10, 2024
2 parents 79eafc9 + 83ca8d0 commit 7622d7c
Show file tree
Hide file tree
Showing 269 changed files with 6,883 additions and 5,460 deletions.
17 changes: 11 additions & 6 deletions .github/workflows/manual-deploy-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,15 @@ on:
type: string
required: true
hadoop:
type: string
type: choice
description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
required: false
default: "hdp3.1"
default: hdp3.1
options:
- hdp3.1
- hdi5.1
- emr6.1
- emr6.13

jobs:
build:
Expand All @@ -23,12 +28,12 @@ jobs:
outputs:
version: ${{ steps.get_project_version.outputs.version }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: '10'
ref: "${{ inputs.branch }}"
- name: Set up JDK 8
uses: actions/setup-java@v3
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '8'
Expand All @@ -40,8 +45,8 @@ jobs:
./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
fi
- name: Maven Build (skip tests)
run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }},RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
- uses: actions/upload-artifact@v3
run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip
- uses: actions/upload-artifact@v4
with:
name: build-folder
path: build
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/manual-deploy-ext-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ jobs:
outputs:
version: ${{ steps.get_project_version.outputs.version }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: '10'
ref: "${{ inputs.branch }}"
- name: Set up JDK 8
uses: actions/setup-java@v3
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '8'
Expand All @@ -41,7 +41,7 @@ jobs:
fi
- name: Maven Build (skip tests)
run: mvn -T 2 clean install -DskipTests
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v4
with:
name: build-folder
path: build
Expand Down
77 changes: 64 additions & 13 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,39 +7,90 @@ on:
workflow_dispatch:

jobs:
build:
# Add the build-hdp job
build-hdp:
uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
with:
maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
build_folder: build-folder

deploy-maven:
uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
needs: build
# Add the deploy-docker-hdp job that depends on the build-hdp job
deploy-docker-hdp:
uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
needs: build-hdp
with:
maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
build_folder: build-folder
secrets: inherit

deploy-docker:
uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
needs: build
# Add the deploy-maven and deploy-python jobs they depend on the build-hdp job
deploy-maven:
uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
needs: build-hdp
with:
cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }}
maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
secrets: inherit

deploy-python:
uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
needs: build
needs: build-hdp
with:
cli: ./clients/python/python-build.sh push
artifact: build-folder
secrets: inherit

# Add the release job that depends on all the previous jobs
release:
uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
needs: [ build, deploy-maven, deploy-docker, deploy-python ]
needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python]
with:
artifact: build-folder
file: |
opencga-client-${{ needs.build.outputs.version }}.tar.gz
clients/R/opencgaR_${{ needs.build.outputs.version }}.tar.gz
opencga-client-${{ needs.build-hdp.outputs.version }}.tar.gz
clients/R/opencgaR_${{ needs.build-hdp.outputs.version }}.tar.gz
# Compile and deploy other hadoop flavours
# Add the build-hdi and deploy-docker-hdi jobs
build-hdi:
uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
with:
maven_opts: -P hdi5.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
build_folder: build-folder-hdi

deploy-docker-hdi:
uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
needs: build-hdi
with:
cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdi.outputs.version }}-hdi5.1"
build_folder: build-folder-hdi
secrets: inherit

# Add the build-emr and deploy-docker-emr jobs
build-emr:
uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
with:
maven_opts: -P emr6.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
build_folder: build-folder-emr

deploy-docker-emr:
uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
needs: build-emr
with:
cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr.outputs.version }}-emr6.1"
build_folder: build-folder-emr
secrets: inherit

# Add the build-emr613 and deploy-docker-emr613 jobs
build-emr613:
uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
with:
maven_opts: -P emr6.13,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
build_folder: build-folder-emr613

deploy-docker-emr613:
uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
needs: build-emr613
with:
cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr613.outputs.version }}-emr6.13"
build_folder: build-folder-emr613
secrets: inherit
6 changes: 6 additions & 0 deletions opencga-analysis/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,12 @@
<version>${jetty-for-hadoop-test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.http2</groupId>
<artifactId>http2-hpack</artifactId>
<version>${jetty-for-hadoop-test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
import org.opencb.opencga.core.models.job.Job;
import org.opencb.opencga.core.response.OpenCGAResult;

import java.io.*;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;

Expand Down Expand Up @@ -46,6 +48,26 @@ public static File getBamFileBySampleId(String sampleId, String studyId, FileMan
return (fileQueryResult.getNumResults() == 0) ? null : fileQueryResult.first();
}

public static File getBwFileBySampleId(String sampleId, String studyId, FileManager fileManager, String token) throws ToolException {
// Look for the bam file for each sample
OpenCGAResult<File> fileQueryResult;

Query query = new Query(FileDBAdaptor.QueryParams.FORMAT.key(), File.Format.BIGWIG)
.append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sampleId);
try {
fileQueryResult = fileManager.search(studyId, query, QueryOptions.empty(), token);
} catch (CatalogException e) {
throw new ToolException(e);
}

// Sanity check
if (fileQueryResult.getNumResults() > 1) {
throw new ToolException("Found more than one BIGWIG files (" + fileQueryResult.getNumResults() + ") for sample " + sampleId);
}

return (fileQueryResult.getNumResults() == 0) ? null : fileQueryResult.first();
}

public static File getBamFile(String filename, String sampleId, String studyId, FileManager fileManager, String token) throws ToolException {
// Look for the bam file for each sample
OpenCGAResult<File> fileQueryResult;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
* Copyright 2015-2020 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.opencb.opencga.analysis.alignment;

import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.file.*;
import org.opencb.opencga.core.response.OpenCGAResult;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;

public class AlignmentAnalysisUtils {

public static File linkAndUpdate(File bamCatalogFile, Path outPath, String jobId, String study, CatalogManager catalogManager, String token)
throws CatalogException, ToolException {
// Link BW file and update sample info
FileLinkParams fileLinkParams = new FileLinkParams()
.setUri(outPath.toString())
.setPath(Paths.get(jobId).resolve(outPath.getFileName()).toString());
OpenCGAResult<File> fileResult = catalogManager.getFileManager().link(study, fileLinkParams, true, token);
if (fileResult.getNumResults() != 1) {
throw new ToolException("It could not link OpenCGA file catalog file for '" + outPath + "'");
}
File outCatalogFile = fileResult.first();

// Updating file: samples, related file
FileUpdateParams updateParams = new FileUpdateParams()
.setSampleIds(bamCatalogFile.getSampleIds())
.setRelatedFiles(Collections.singletonList(new SmallRelatedFileParams()
.setFile(bamCatalogFile.getId())
.setRelation(FileRelatedFile.Relation.ALIGNMENT)));
try {
OpenCGAResult<File> updateResult = catalogManager.getFileManager().update(study, outCatalogFile.getId(), updateParams, null,
token);
if (updateResult.getNumUpdated() != 1) {
catalogManager.getFileManager().unlink(study, outCatalogFile.getId(), token);
throw new ToolException("It could not update OpenCGA file catalog (" + outCatalogFile.getId()
+ ") from alignment file ID '" + bamCatalogFile.getId() + "'");
}
} catch (CatalogException e) {
catalogManager.getFileManager().unlink(study, outCatalogFile.getId(), token);
throw e;
}
return outCatalogFile;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright 2015-2020 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.opencb.opencga.analysis.alignment;

public class AlignmentConstants {

public static final String BAM_EXTENSION = ".bam";
public static final String BAI_EXTENSION = ".bai";
public static final String CRAM_EXTENSION = ".cram";
public static final String CRAI_EXTENSION = ".crai";
public static final String BIGWIG_EXTENSION = ".bw";
}
Loading

0 comments on commit 7622d7c

Please sign in to comment.