diff --git a/.github/workflows/check-junit-test.yml b/.github/workflows/check-junit-test.yml new file mode 100644 index 00000000000..4331f156cf5 --- /dev/null +++ b/.github/workflows/check-junit-test.yml @@ -0,0 +1,55 @@ +name: Check junits + +on: + workflow_dispatch: + inputs: + short_tests: + type: boolean + required: false + default: true + medium_tests: + type: boolean + required: false + default: false + long_tests: + type: boolean + required: false + default: false + mvn_opts: + type: string + required: false + default: "" + +jobs: + get_profiles: + name: Test JUnit + runs-on: ubuntu-22.04 + outputs: + profiles: ${{ steps.getter.outputs.profiles }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: '10' + - name: Building string profiles to run + id: getter + run: | + if [ -f "./.github/workflows/scripts/get_profiles.sh" ]; then + chmod +x ./.github/workflows/scripts/get_profiles.sh + echo "profiles=$(./.github/workflows/scripts/get_profiles.sh ${{ inputs.short_tests }} ${{ inputs.medium_tests }} ${{ inputs.long_tests }})" >> $GITHUB_OUTPUT + fi + + echo_log: + needs: get_profiles + runs-on: ubuntu-22.04 + steps: + - name: Echo profiles + id: log + run: echo "Executing testing profiles -> ${{ needs.get_profiles.outputs.profiles }}" + + test: + needs: [ get_profiles, echo_log ] + uses: ./.github/workflows/test-analysis.yml + with: + test_profile: ${{ needs.get_profiles.outputs.profiles }} + mvn_opts: ${{ inputs.mvn_opts }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index a6d37f1e39c..0e8f622a2b6 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -9,34 +9,26 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' - deploy-docker-ext-tools: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x - needs: build - with: - cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }} - secrets: inherit - test: uses: ./.github/workflows/test-analysis.yml needs: deploy-docker-ext-tools - secrets: inherit with: test_profile: runShortTests,runMediumTests - report_context: development + secrets: inherit deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop needs: test with: maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga secrets: inherit deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml index 98157167216..b44a777134d 100644 --- a/.github/workflows/docker-aws-emr.yml +++ b/.github/workflows/docker-aws-emr.yml @@ -8,15 +8,13 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: maven_opts: -P storage-hadoop,emr6.1 -Dopencga.war.name=opencga deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init secrets: inherit -# DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} -# DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} diff --git a/.github/workflows/docker-tools.yml b/.github/workflows/docker-ext-tools.yml similarity index 81% rename from .github/workflows/docker-tools.yml rename to .github/workflows/docker-ext-tools.yml index 32e938c5a2d..07062f03278 100644 --- a/.github/workflows/docker-tools.yml +++ b/.github/workflows/docker-ext-tools.yml @@ -1,4 +1,4 @@ -name: Deploy Docker Tools +name: Deploy Docker Ext-Tools on: push: @@ -12,13 +12,12 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x - with: - maven_opts: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop deploy-docker-ext-tools: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }} secrets: inherit + diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml index f516eb25029..ef610d74bda 100644 --- a/.github/workflows/long-test-analysis.yml +++ b/.github/workflows/long-test-analysis.yml @@ -1,6 +1,7 @@ name: Long test for run only on weekend -#Every Saturday at 01:10 + +## Every Saturday at 01:10 on: schedule: - cron: '10 1 * * 6' @@ -13,3 +14,4 @@ jobs: test_profile: runLongTests,runMediumTests,runShortTests report_context: development + diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index ff9fbdc3c1e..3e3e5bb46b9 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -3,14 +3,17 @@ name: "Pull Request Merge Workflow" on: pull_request: branches: - - "develop" - - "release-*" + - develop + - release-* types: - closed jobs: + build: + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop + delete-docker: - uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop with: cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }} secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c69bdad248..204f13d2b72 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,52 +8,35 @@ on: jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' - deploy-docker-ext-tools: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x - needs: build - with: - cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ needs.build.outputs.version }} - secrets: inherit - - test: - uses: ./.github/workflows/test-analysis.yml - needs: deploy-docker-ext-tools - secrets: inherit - with: - test_profile: runShortTests,runMediumTests - report_context: xetabase - deploy-maven: - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x - needs: test + uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop + needs: build with: maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga secrets: inherit deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x - needs: test + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop + needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init secrets: inherit - - deploy-python: - uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@release-4.8.x - needs: test + uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop + needs: build with: cli: ./clients/python/python-build.sh push artifact: build-folder secrets: inherit release: - uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@release-4.8.x - needs: [ build,test ] + uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop + needs: [ build, deploy-maven, deploy-docker, deploy-python ] with: artifact: build-folder file: | diff --git a/.github/workflows/scripts/get_profiles.sh b/.github/workflows/scripts/get_profiles.sh new file mode 100755 index 00000000000..fc205968984 --- /dev/null +++ b/.github/workflows/scripts/get_profiles.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ $# -eq 0 ]; then + echo "The arguments must be 3" + exit 1 +fi +PROFILE="" + +if [ $1 == "true" ]; then + PROFILE="${PROFILE}runShortTests," +fi +if [ $2 == "true" ]; then + PROFILE="${PROFILE}runMediumTests," +fi +if [ $3 == "true" ]; then + PROFILE="${PROFILE}runLongTests" +fi + +if [[ "${PROFILE}" == *"," ]]; then + PROFILE="${PROFILE%?}" +fi + +if [ -z "${PROFILE}" ]; then + echo "There must be at least one active profile" + exit 1 +fi + +echo "${PROFILE}" +exit 0 + diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh new file mode 100644 index 00000000000..f217d21e70c --- /dev/null +++ b/.github/workflows/scripts/get_same_branch.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +BRANCH_NAME=$1 + +if [[ -z $BRANCH_NAME ]]; then + echo "The first parameter is mandatory and must be a valid branch name." + exit 1 +fi + +if [[ $BRANCH_NAME != "TASK-"* ]]; then + echo "No need to check dependencies." + exit 0 +fi + +function install(){ + local REPO=$1 + cd /home/runner/work/ || exit 2 + git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME" + if [ -d "./$REPO" ]; then + cd "$REPO" || exit 2 + echo "Branch name $BRANCH_NAME already exists." + mvn clean install -DskipTests + else + echo "$CURRENT Branch is NOT EQUALS $BRANCH_NAME " + fi +} + +install "java-common-libs" +install "biodata" +install "cellbase" diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 1b522b40511..51888a9179d 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -6,9 +6,11 @@ on: - TASK-* workflow_dispatch: +# WARNING Develop branch needed for prod + jobs: build: - uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' @@ -18,37 +20,10 @@ jobs: secrets: inherit with: test_profile: runShortTests - report_context: development - - deploy-maven: - name: Deploy in maven only for renamed versions - uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@release-4.8.x - needs: [ build, test ] - if: contains( needs.build.outputs.version ,'TASK') - secrets: inherit deploy-docker: - uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@release-4.8.x + uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop needs: test with: cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ github.ref_name }} secrets: inherit - - # The following jobs are to see that the previous 'if' doesn't fail and that the maven deploy is executed because it is true - snapshot-version: - name: Check SNAPSHOT version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'SNAPSHOT') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} - - task-version: - name: Check TASK version - needs: [ build, test ] - if: contains(needs.build.outputs.version ,'TASK') - runs-on: ubuntu-22.04 - steps: - - name: test-version-from-check - run: echo "Project version is " ${{ needs.build.outputs.version }} diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 8c72b649a33..732d92a4e56 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -1,26 +1,19 @@ name: Build and test the project + on: workflow_call: inputs: test_profile: type: string required: true - report_context: + mvn_opts: type: string - required: true + required: false + default: "" secrets: SONAR_TOKEN: required: true - SSH_TESTING_SERVER_HOST: - required: true - SSH_TESTING_SERVER_PORT: - required: true - SSH_TESTING_SERVER_USER: - required: true - SSH_TESTING_SERVER_PASSWORD: - required: true -env: - xb_version: "1.6.2" + jobs: analysis: name: Execute Sonar Analysis @@ -35,11 +28,20 @@ jobs: distribution: 'temurin' java-version: '11' cache: 'maven' + - name: Install dependencies branches + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + else + echo "./.github/workflows/scripts/get_same_branch.sh does not exist." + fi - name: Test and Analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: mvn -T 2 clean install -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga + test: name: Execute JUnit and Jacoco tests runs-on: ubuntu-22.04 @@ -53,6 +55,14 @@ jobs: distribution: 'temurin' java-version: '8' cache: 'maven' + - name: Install dependencies branches + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + else + echo "./.github/workflows/scripts/get_same_branch.sh does not exist." + fi - name: Install Samtools run: sudo apt-get install samtools python3-deeptools - name: Start MongoDB v6.0 @@ -61,31 +71,14 @@ jobs: mongodb-version: 6.0 mongodb-replica-set: rs-test - name: Run Junit tests - run: mvn -B verify surefire-report:report -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' - - name: Upload result dir - uses: actions/upload-artifact@v3 - with: - name: workdir - path: "**/target/site" - publish-test: - name: Publish test results - runs-on: ubuntu-22.04 - needs: test - strategy: - matrix: - module: ["opencga-app", "opencga-catalog", "opencga-client", "opencga-clinical", "opencga-core", "opencga-master", "opencga-server", "opencga-storage", "opencga-storage/opencga-storage-app", "opencga-storage/opencga-storage-benchmark", "opencga-storage/opencga-storage-core", "opencga-storage/opencga-storage-hadoop", "opencga-storage/opencga-storage-server", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core", "opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps"] - steps: - - name: Download result dir - uses: actions/download-artifact@v3 - with: - name: workdir - - name: Deploy unit tests web recursively to remote - uses: garygrossgarten/github-action-scp@release + run: mvn -B verify surefire-report:report --fail-never -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' ${{ inputs.mvn_opts }} + - name: Publish Test Report + uses: scacap/action-surefire-report@v1 + ## Skip cancelled() + ## https://docs.github.com/en/actions/learn-github-actions/expressions#cancelled + if: success() || failure() with: - local: ${{ matrix.module }}/target/site - remote: /var/www/html/reports/${{ inputs.report_context }}/${{ env.xb_version }}/${{ github.ref_name }}/opencga/${{ github.sha }}/unit/${{ matrix.module }} - host: ${{ secrets.SSH_TESTING_SERVER_HOST}} - port: ${{ secrets.SSH_TESTING_SERVER_PORT}} - username: ${{ secrets.SSH_TESTING_SERVER_USER }} - password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }} - concurrency: 2 + check_name: "Surefire tests report" + report_paths: './**/surefire-reports/TEST-*.xml' + commit: '${{ github.sha }}' + fail_on_test_failures: true diff --git a/.github/workflows/test-long-test-analysis.yml b/.github/workflows/test-long-test-analysis.yml deleted file mode 100644 index fe516b75a98..00000000000 --- a/.github/workflows/test-long-test-analysis.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: LONG TEST ANALYSIS - -on: - push: - branches: - - TASK-4149 - -jobs: - test: - uses: ./.github/workflows/test-analysis.yml - secrets: inherit - with: - test_profile: runLongTests,runMediumTests,runShortTests - report_context: development diff --git a/misc/sync_dependency.sh b/misc/sync_dependency.sh deleted file mode 100755 index 316bbc39d3b..00000000000 --- a/misc/sync_dependency.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -function yellow (){ - echo "$(tput setaf 3)$1$(tput setaf 7)" -} -function green (){ - echo "$(tput setaf 2)$1$(tput setaf 7)" -} -function cyan (){ - echo "$(tput setaf 6)$1$(tput setaf 7)" -} - -function printUsage(){ - echo "" - yellow "Release an OpenCB project." - echo "" - echo "Usage: $(basename $0) --biodata|-b|--java-common-libs|-j" - echo "" - cyan "Options:" - green " -j --java-common-libs STRING Update java-common-libs dependency" - green " -b --biodata STRING Update biodata dependency" - echo "" -} - -## Check if the repo status is clean. -function check_repo_clean() { - GIT_STATUS=$(git status --short) - if [ -n "$GIT_STATUS" ]; then - yellow "Repository is not clean:" - yellow "$GIT_STATUS" - exit - fi -} - -## This function removes TASK-XXX- if exists, otherwise it adds it. -function toggle_version() { - local BRANCH=$1 - if [[ "$POM_DEPENDENCY_VERSION" == *"$BRANCH"* ]]; then - ## Remove TASK-XXX- from the current version - ## Example: remove 'TASK-1234-' from 2.6.0-TASK-1234-SNAPSHOT - NEW_VERSION=${POM_DEPENDENCY_VERSION/"$BRANCH-"} - else - ## Add 'TASK-XXX-' to the current version - ## Example: 2.6.0-SNAPSHOT --> 2.6.0-TASK-1234-SNAPSHOT - CLEAN_RELEASE_VERSION=$(echo "$POM_DEPENDENCY_VERSION" | cut -d "-" -f 1) - TAG_VERSION=$(echo "$POM_DEPENDENCY_VERSION" | cut -d "-" -f 2) - NEW_VERSION="$CLEAN_RELEASE_VERSION-$BRANCH-$TAG_VERSION" - fi -} - -## Change version in the dependency. -## Usage: update_dependency "$DEPENDENCY_REPO" "$NEW_VERSION" "$BRANCH_NAME" -function update_dependency() { - ## Save current directory - local pwd=$PWD - cd "$1" || exit 2 - check_repo_clean - git checkout "$3" - ## Check branch exists - local BRANCH=$(git branch --show-current) - if [ "$BRANCH" != "$3" ]; then - yellow "Branch '$3' does not exist" - exit - fi - ## Rename and commit new version - mvn versions:set -DnewVersion="$2" -DgenerateBackupPoms=false - git commit -am "Update version to $2" - ## Restore directory - cd "$pwd" || exit 2 -} - -## At least one parameter is required. -if [ -z "$1" ]; then - printUsage - exit 1 -fi - -while [[ $# -gt 0 ]]; do - key="$1" - if [ -n "$2" ]; then - DEPENDENCY_REPO="$2" - fi - case $key in - -h | --help) - printUsage - exit 0 - ;; - -j | --java-common-libs) - LIB="JAVA_COMMONS_LIB" - if [ -z "$DEPENDENCY_REPO" ]; then - DEPENDENCY_REPO="../java-common-libs" - else - shift - fi - shift # past argument - ;; - -b | --biodata) - LIB="BIODATA" - if [ -z "$DEPENDENCY_REPO" ]; then - DEPENDENCY_REPO="../biodata" - else - shift - fi - shift # past argument - ;; - *) # unknown option - echo "Unknown option $key" - printUsage - exit 1 - ;; - esac -done - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -CURRENT_DIR=$PWD -cd "$SCRIPT_DIR" || exit 2 -cd .. -BRANCH_NAME=$(git branch --show-current) -if [[ "$BRANCH_NAME" == "TASK-"* ]]; then - check_repo_clean "$BRANCH_NAME" -else - yellow "[$BRANCH_NAME] The branch name must start with TASK-" - yellow "$GIT_STATUS" - exit -fi - -function update_library(){ - local LIBRARY="$1" - POM_DEPENDENCY_VERSION=$(grep -m 1 "$LIBRARY" pom.xml | cut -d ">" -f 2 | cut -d "<" -f 1) - toggle_version "$BRANCH_NAME" - update_dependency "$DEPENDENCY_REPO" "$NEW_VERSION" "$BRANCH_NAME" - mvn versions:set-property -Dproperty=java-common-libs.version -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false - git commit -am "Update '$LIBRARY' dependency to $NEW_VERSION" -} - - -if [ "$LIB" = "JAVA_COMMONS_LIB" ];then - update_library java-common-libs.version -fi -if [ "$LIB" = "BIODATA" ];then - update_library biodata.version -fi - -yellow "The new dependency version is $NEW_VERSION" -cd "$CURRENT_DIR" || exit 2 diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java index 02ab7d9b5d8..4fe57ad701b 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/InterpretationAnalysis.java @@ -79,8 +79,8 @@ protected InterpretationMethod getInterpretationMethod(String name) { .setName(name) .setDependencies(Collections.singletonList(new Software() .setName("OpenCGA") - .setVersion(GitRepositoryState.get().getBuildVersion()) - .setCommit(GitRepositoryState.get().getCommitId()))); + .setVersion(GitRepositoryState.getInstance().getBuildVersion()) + .setCommit(GitRepositoryState.getInstance().getCommitId()))); return method; } @@ -114,8 +114,8 @@ protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnaly // Interpretation method InterpretationMethod method = new InterpretationMethod(getId(), null, null, - Collections.singletonList(new Software().setName("OpenCGA").setVersion(GitRepositoryState.get().getBuildVersion()) - .setCommit(GitRepositoryState.get().getCommitId()))); + Collections.singletonList(new Software().setName("OpenCGA").setVersion(GitRepositoryState.getInstance().getBuildVersion()) + .setCommit(GitRepositoryState.getInstance().getCommitId()))); // Analyst ClinicalAnalyst analyst = clinicalInterpretationManager.getAnalyst(token); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java index 07e35a78504..39fa4fd581c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserInterpretationAnalysis.java @@ -137,8 +137,8 @@ protected void run() throws ToolException { protected void saveInterpretation(String studyId, ClinicalAnalysis clinicalAnalysis) throws ToolException, StorageEngineException, CatalogException, IOException { // Interpretation method - InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.get().getBuildVersion(), - GitRepositoryState.get().getCommitId(), Collections.singletonList( + InterpretationMethod method = new InterpretationMethod(getId(), GitRepositoryState.getInstance().getBuildVersion(), + GitRepositoryState.getInstance().getCommitId(), Collections.singletonList( new Software() .setName("Exomiser") .setRepository("Docker: " + ExomiserWrapperAnalysisExecutor.DOCKER_IMAGE_NAME) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java index cd2ada565fa..40f829ecd6c 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/circos/CircosLocalAnalysisExecutor.java @@ -58,7 +58,7 @@ public class CircosLocalAnalysisExecutor extends CircosAnalysisExecutor implements StorageToolExecutor { public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" - + GitRepositoryState.get().getBuildVersion(); + + GitRepositoryState.getInstance().getBuildVersion(); private VariantStorageManager storageManager; private File snvsFile; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java index 027bb8dc9a8..4246e142d55 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotLocalAnalysisExecutor.java @@ -61,7 +61,7 @@ public class GenomePlotLocalAnalysisExecutor extends GenomePlotAnalysisExecutor implements StorageToolExecutor { public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" - + GitRepositoryState.get().getBuildVersion(); + + GitRepositoryState.getInstance().getBuildVersion(); private GenomePlotConfig plotConfig; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java index 4c96ad7b3b0..f85e0a1873a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java @@ -16,31 +16,21 @@ package org.opencb.opencga.analysis.variant.hrdetect; -import htsjdk.samtools.reference.BlockCompressedIndexedFastaSequenceFile; -import htsjdk.samtools.reference.FastaSequenceIndex; -import htsjdk.samtools.reference.ReferenceSequence; -import htsjdk.samtools.util.GZIIndex; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; -import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.commons.datastore.core.QueryResultWriter; import org.opencb.commons.exec.Command; import org.opencb.commons.utils.DockerUtils; -import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; -import org.opencb.opencga.core.response.OpenCGAResult; -import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.ToolExecutor; import org.opencb.opencga.core.tools.variant.HRDetectAnalysisExecutor; -import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; @@ -53,14 +43,12 @@ import java.nio.file.Paths; import java.util.*; -import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT; - @ToolExecutor(id="opencga-local", tool = HRDetectAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) public class HRDetectLocalAnalysisExecutor extends HRDetectAnalysisExecutor implements StorageToolExecutor { - public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion(); + public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); private final static String CNV_FILENAME = "cnv.tsv"; private final static String INDEL_FILENAME = "indel.vcf"; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index f2bf45cf487..c542521329a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -64,11 +64,12 @@ public class MutationalSignatureLocalAnalysisExecutor extends MutationalSignatureAnalysisExecutor implements StorageToolExecutor { - public static final String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion(); - private static final String SVCLASS = "SVCLASS"; private static final String EXT_SVTYPE = "EXT_SVTYPE"; + public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + + GitRepositoryState.getInstance().getBuildVersion(); + private Path opencgaHome; private Logger logger = LoggerFactory.getLogger(this.getClass()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java index 27d6c54d7b3..21f619cc5a4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/executors/DockerWrapperAnalysisExecutor.java @@ -32,7 +32,7 @@ public String getDockerImageName() { } public String getDockerImageVersion() { - return GitRepositoryState.get().getBuildVersion(); + return GitRepositoryState.getInstance().getBuildVersion(); } private Logger privateLogger = LoggerFactory.getLogger(DockerWrapperAnalysisExecutor.class); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java index 0884482ef80..f45e8e77ec2 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java @@ -5,10 +5,8 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.NodeConfig; -import org.apache.solr.core.SolrResourceLoader; import org.junit.rules.ExternalResource; import org.opencb.commons.datastore.solr.SolrManager; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.config.storage.StorageConfiguration; @@ -40,8 +38,8 @@ protected void before() throws Throwable { Path rootDir = getTmpRootDir(); - String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.get().getBuildVersion(); - String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.get().getBuildVersion(); + String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.getInstance().getBuildVersion(); + String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.getInstance().getBuildVersion(); copyConfigSetConfiguration(mainConfigSet, "managed-schema"); copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema"); diff --git a/opencga-app/app/cloud/docker/opencga-base/Dockerfile b/opencga-app/app/cloud/docker/opencga-base/Dockerfile index 3d2c1e4db7c..d3eee2266d2 100644 --- a/opencga-app/app/cloud/docker/opencga-base/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-base/Dockerfile @@ -1,5 +1,6 @@ -## Based on Debian 11 (bullseye) -FROM openjdk:8-jre +## Based on Ubuntu 22.04 (jammy) +## We are now using OpenJDK 8u372 to support "cgroup v2", see https://developers.redhat.com/articles/2023/04/19/openjdk-8u372-feature-cgroup-v2-support# +FROM eclipse-temurin:8u372-b07-jre-jammy ARG BUILD_PATH="." @@ -7,14 +8,20 @@ ENV OPENCGA_HOME=/opt/opencga ENV OPENCGA_CONFIG_DIR=${OPENCGA_HOME}/conf RUN apt-get update && apt-get -y upgrade && apt-get install -y lsb-release sshpass ca-certificates curl gnupg jq ncurses-bin && \ - # Install Docker repository - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ - # Install MongoDB repository - wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - && \ - echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.2 main" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list && \ - apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io mongodb-org-shell && \ + ## Install Docker repository + install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + chmod a+r /etc/apt/keyrings/docker.gpg && \ + echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "jammy")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + ## Install MongoDB 6.0 repository + curl -fsSL https://pgp.mongodb.com/server-6.0.asc | gpg --dearmor -o /usr/share/keyrings/mongodb-server-6.0.gpg && \ + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-6.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \ + ## Install docker and mongodb packages + apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io mongodb-mongosh && \ + ## Install yq + wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && \ + chmod +x /usr/local/bin/yq && \ + ## Clean downloaded packages to make images smaller rm -rf /var/lib/apt/lists/* && \ adduser --disabled-password --uid 1001 opencga diff --git a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile index 364a2994a24..5558259e3f5 100644 --- a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile @@ -33,4 +33,4 @@ RUN git fetch origin --tags && \ rm -rf /var/lib/apt/lists/* /tmp/* /opt/opencga/signature.tools.lib/.git && \ strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5 -WORKDIR /opt/opencga \ No newline at end of file +WORKDIR /opt/opencga diff --git a/opencga-app/app/cloud/docker/opencga-init/Dockerfile b/opencga-app/app/cloud/docker/opencga-init/Dockerfile index f7efd322ac9..60b60f278b7 100644 --- a/opencga-app/app/cloud/docker/opencga-init/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-init/Dockerfile @@ -8,25 +8,22 @@ ARG INIT_PATH=cloud/docker/opencga-init/ COPY ${INIT_PATH} /opt/opencga/init/ COPY ${BUILD_PATH}/conf/* /opt/opencga/init/test/ - -# Mount volume to copy config into +## Mount volume to copy config into VOLUME /opt/volume USER root -# Install local dependencies -RUN apt install python3 && \ - echo "deb http://ftp.de.debian.org/debian bullseye main" | tee -a /etc/apt/sources.list.d/sources.list && \ - apt-get update && apt-get -y upgrade && \ - apt install -y python3-pip && \ + +## Install local dependencies +RUN apt-get update && apt-get -y upgrade && apt-get install -y python3 python3-pip && \ pip3 install --upgrade pip setuptools && \ pip3 install -r /opt/opencga/init/requirements.txt && \ chmod +x /opt/opencga/init/setup.sh /opt/opencga/init/setup-hadoop.sh && \ echo ">Running init container configuration tests" && \ cd /opt/opencga/init/test && python3 test_override_yaml.py -v && rm -r /opt/opencga/init/test - +## Run Docker images as non root USER opencga -# It is the responsibility of the setup.sh -# script to initialise the volume correctly -# and apply any runtime config transforms. -ENTRYPOINT [ "/bin/sh","/opt/opencga/init/setup.sh" ] + +## It is the responsibility of the setup.sh script to initialise the volume correctly +## and apply any runtime config transforms. +ENTRYPOINT [ "/bin/sh", "/opt/opencga/init/setup.sh" ] diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java index 89832e53260..28787847b65 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminCliOptionsParser.java @@ -800,8 +800,8 @@ public void printUsage() { if (parsedCommand.isEmpty()) { System.err.println(""); System.err.println("Program: OpenCGA Admin (OpenCB)"); - System.err.println("Version: " + GitRepositoryState.get().getBuildVersion()); - System.err.println("Git commit: " + GitRepositoryState.get().getCommitId()); + System.err.println("Version: " + GitRepositoryState.getInstance().getBuildVersion()); + System.err.println("Git commit: " + GitRepositoryState.getInstance().getCommitId()); System.err.println("Description: Big Data platform for processing and analysing NGS data"); System.err.println(""); System.err.println("Usage: opencga-admin.sh [-h|--help] [--version] [options]"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminMain.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminMain.java index 2bb78fb51c3..2dc1f8d638b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminMain.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/AdminMain.java @@ -30,7 +30,7 @@ */ public class AdminMain { - public static final String VERSION = GitRepositoryState.get().getBuildVersion(); + public static final String VERSION = GitRepositoryState.getInstance().getBuildVersion(); public static void main(String[] args) { @@ -71,8 +71,8 @@ public static void main(String[] args) { String parsedCommand = cliOptionsParser.getCommand(); if (parsedCommand == null || parsedCommand.isEmpty()) { if (cliOptionsParser.getGeneralOptions().version) { - System.out.println("Version " + GitRepositoryState.get().getBuildVersion()); - System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); + System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion()); + System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); System.exit(0); } else if (cliOptionsParser.getGeneralOptions().help) { cliOptionsParser.printUsage(); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java index ceaab6ea98d..a360eaa2f1e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java @@ -167,7 +167,7 @@ private String parseVersion(String version) { public static String getDefaultVersion() { String version; - version = GitRepositoryState.get().getBuildVersion(); + version = GitRepositoryState.getInstance().getBuildVersion(); // Remove extra information version = version.split("-")[0]; return version; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java index f11d519c846..132ad854c82 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java @@ -468,8 +468,8 @@ public void printUsage() { if (parsedCommand.isEmpty()) { System.err.println(""); System.err.println("Program: OpenCGA Analysis (OpenCB)"); - System.err.println("Version: " + GitRepositoryState.get().getBuildVersion()); - System.err.println("Git commit: " + GitRepositoryState.get().getCommitId()); + System.err.println("Version: " + GitRepositoryState.getInstance().getBuildVersion()); + System.err.println("Git commit: " + GitRepositoryState.getInstance().getCommitId()); System.err.println("Description: Big Data platform for processing and analysing NGS data"); System.err.println(""); System.err.println("Usage: opencga-internal.sh [-h|--help] [--version] [options]"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalMain.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalMain.java index a1aaf7466f3..98d8b965357 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalMain.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalMain.java @@ -28,7 +28,7 @@ */ public class InternalMain { - public static final String VERSION = GitRepositoryState.get().getBuildVersion(); + public static final String VERSION = GitRepositoryState.getInstance().getBuildVersion(); public static void main(String[] args) { System.exit(privateMain(args)); @@ -48,8 +48,8 @@ public static int privateMain(String[] args) { String parsedCommand = cliOptionsParser.getCommand(); if (parsedCommand == null || parsedCommand.isEmpty()) { if (cliOptionsParser.getGeneralOptions().version) { - System.out.println("Version " + GitRepositoryState.get().getBuildVersion()); - System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); + System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion()); + System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); return 0; } else if (cliOptionsParser.getGeneralOptions().help) { cliOptionsParser.printUsage(); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaMain.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaMain.java index 4fa055dde8c..296b1be2d75 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaMain.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaMain.java @@ -17,6 +17,7 @@ package org.opencb.opencga.app.cli.main; import org.apache.commons.lang3.ArrayUtils; +import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.CliOptionsParser; import org.opencb.opencga.app.cli.GeneralCliOptions; import org.opencb.opencga.app.cli.main.processors.CommandProcessor; @@ -48,6 +49,7 @@ public static void main(String[] args) { parser.printUsage(); System.exit(0); } + PrintUtils.printSpace(); checkLogLevel(args); checkMode(args); logger.debug(Arrays.toString(args)); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/CommandLineUtils.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/CommandLineUtils.java index 46af3e8eeb9..4454887584b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/CommandLineUtils.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/CommandLineUtils.java @@ -18,16 +18,16 @@ public class CommandLineUtils { private static final Logger logger = LoggerFactory.getLogger(CommandLineUtils.class); public static String getVersionString() { - String res = PrintUtils.getKeyValueAsFormattedString("\tOpenCGA CLI version: ", "\t" + GitRepositoryState.get().getBuildVersion() + "\n"); - res += PrintUtils.getKeyValueAsFormattedString("\tGit version:", "\t\t" + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId() + "\n"); + String res = PrintUtils.getKeyValueAsFormattedString("\tOpenCGA CLI version: ", "\t" + GitRepositoryState.getInstance().getBuildVersion() + "\n"); + res += PrintUtils.getKeyValueAsFormattedString("\tGit version:", "\t\t" + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId() + "\n"); res += PrintUtils.getKeyValueAsFormattedString("\tProgram:", "\t\tOpenCGA (OpenCB)" + "\n"); res += PrintUtils.getKeyValueAsFormattedString("\tDescription: ", "\t\tBig Data platform for processing and analysing NGS data" + "\n"); return res; } public static String getHelpVersionString() { - String res = PrintUtils.getHelpVersionFormattedString("OpenCGA CLI version: ", "\t" + GitRepositoryState.get().getBuildVersion() + "\n"); - res += PrintUtils.getHelpVersionFormattedString("Git version:", "\t\t" + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId() + "\n"); + String res = PrintUtils.getHelpVersionFormattedString("OpenCGA CLI version: ", "\t" + GitRepositoryState.getInstance().getBuildVersion() + "\n"); + res += PrintUtils.getHelpVersionFormattedString("Git version:", "\t\t" + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId() + "\n"); res += PrintUtils.getHelpVersionFormattedString("Program:", "\t\tOpenCGA (OpenCB)" + "\n"); res += PrintUtils.getHelpVersionFormattedString("Description: ", "\t\tBig Data platform for processing and analysing NGS data" + "\n"); return res; @@ -90,7 +90,7 @@ public static String[] processShortCuts(String[] args) { break; case "--build-version": case "build-version": - println(GitRepositoryState.get().getBuildVersion()); + println(GitRepositoryState.getInstance().getBuildVersion()); break; case "logout": return ArrayUtils.addAll(new String[]{"users"}, args); @@ -99,7 +99,7 @@ public static String[] processShortCuts(String[] args) { if (args.length > 1 && args[1].equals("studies")) { println(String.join(", ", OpencgaMain.getShell().getSessionManager().getSession().getStudies()), Color.GREEN); } else { - printWarn("Opencga version " + GitRepositoryState.get().getBuildVersion() + " can only list studies"); + printWarn("Opencga version " + GitRepositoryState.getInstance().getBuildVersion() + " can only list studies"); } } else { printWarn("List studies is only available in Shell mode"); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/JobsTopManager.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/JobsTopManager.java index 1c2099ec471..15d7b63e6ec 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/JobsTopManager.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/utils/JobsTopManager.java @@ -150,7 +150,7 @@ public void print(JobTop top) { jobTable.restoreCursorPosition(); jobTable.println("OpenCGA jobs TOP"); - jobTable.println(" Version " + GitRepositoryState.get().getBuildVersion()); + jobTable.println(" Version " + GitRepositoryState.getInstance().getBuildVersion()); jobTable.println(" " + TextOutputWriter.SIMPLE_DATE_FORMAT.format(Date.from(Instant.now()))); jobTable.println(); jobTable.print(Enums.ExecutionStatus.RUNNING + ": " + top.getStats().getRunning() + ", "); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/session/SessionManager.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/session/SessionManager.java index 640a687f7c3..f822716041e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/session/SessionManager.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/session/SessionManager.java @@ -111,7 +111,7 @@ private void init() { private Session createEmptySession() { Session session = new Session(); session.setHost(host); - session.setVersion(GitRepositoryState.get().getBuildVersion()); + session.setVersion(GitRepositoryState.getInstance().getBuildVersion()); session.setTimestamp(System.currentTimeMillis()); session.setStudies(new ArrayList()); session.setCurrentStudy(NO_STUDY); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java index d23d2345681..d9e270c3d9f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_2_0/catalog/issue_1849/CompleteStatusDataModelUtils.java @@ -35,8 +35,8 @@ public static void completeInternalStatus(Document document) { completeStatus(internal); Document status = internal.get("status", Document.class); - status.put("version", GitRepositoryState.get().getBuildVersion()); - status.put("commit", GitRepositoryState.get().getCommitId()); + status.put("version", GitRepositoryState.getInstance().getBuildVersion()); + status.put("commit", GitRepositoryState.getInstance().getCommitId()); String id = status.getString("id"); if (StringUtils.isEmpty(id)) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index 65077320bb7..f74d1a843b7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -25,6 +25,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; @@ -287,9 +288,28 @@ OpenCGAResult removeUsersFromAllGroups(long studyId, List users) * @param groupList List containing possible groups that are synced and where the user should be added to. * @param authOrigin Authentication origin of the synced groups. * @return OpenCGAResult object. - * @throws CatalogDBException CatalogDBException. + * @throws CatalogDBException CatalogDBException + * @throws CatalogParameterException CatalogParameterException + * @throws CatalogAuthorizationException CatalogAuthorizationException */ - OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) throws CatalogDBException; + OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; + + /** + * ADD or REMOVE user to list of provided groups. + * + * @param user User id. + * @param studyUids List of study uids. + * @param groupList List of group ids. + * @param action Update action [ADD, REMOVE] + * @return OpenCGAResult object. + * @throws CatalogDBException CatalogDBException + * @throws CatalogParameterException CatalogParameterException + * @throws CatalogAuthorizationException CatalogAuthorizationException + */ + OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, + ParamUtils.AddRemoveAction action) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException; /** * Create the permission rule to the list of permission rules defined for the entry in the studyId. diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index edab86e2f36..fff9a1ed3b3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -522,8 +522,8 @@ private void addToMembersGroupInStudy(long studyId, List members, Client .collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(userList)) { // We first add the member to the @members group in case they didn't belong already - dbAdaptorFactory.getCatalogStudyDBAdaptor().addUsersToGroup(studyId, CatalogAuthorizationManager.MEMBERS_GROUP, - userList, clientSession); + dbAdaptorFactory.getCatalogStudyDBAdaptor().addUsersToGroup(clientSession, studyId, CatalogAuthorizationManager.MEMBERS_GROUP, + userList); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java index bb542490cdc..295a84f2093 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java @@ -55,7 +55,7 @@ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements MetaDBAdaptor Filters.eq(ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID), Filters.eq(OLD_ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID)); private final MongoDBCollection metaCollection; - private static final String VERSION = GitRepositoryState.get().getBuildVersion(); + private static final String VERSION = GitRepositoryState.getInstance().getBuildVersion(); public MetaMongoDBAdaptor(MongoDBCollection metaMongoDBCollection, Configuration configuration, MongoDBAdaptorFactory dbAdaptorFactory) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index 0a3539870bb..34510e96868 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -29,7 +29,6 @@ import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; -import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; import org.opencb.opencga.catalog.db.mongodb.converters.VariableSetConverter; @@ -40,6 +39,7 @@ import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.cohort.Cohort; @@ -482,8 +482,8 @@ public OpenCGAResult setUsersToGroup(long studyId, String groupId, List(result); } - void addUsersToGroup(long studyId, String groupId, List members, ClientSession clientSession) throws CatalogDBException { - if (ListUtils.isEmpty(members)) { + void addUsersToGroup(ClientSession clientSession, long studyId, String groupId, List members) throws CatalogDBException { + if (CollectionUtils.isEmpty(members)) { return; } @@ -491,7 +491,7 @@ void addUsersToGroup(long studyId, String groupId, List members, ClientS .append(PRIVATE_UID, studyId) .append(QueryParams.GROUP_ID.key(), groupId); Document update = new Document("$addToSet", new Document("groups.$.userIds", new Document("$each", members))); - DataResult result = studyCollection.update(clientSession, query, update, null); + DataResult result = studyCollection.update(clientSession, query, update, null); if (result.getNumMatches() != 1) { throw new CatalogDBException("Unable to add members to group " + groupId + ". The group does not exist."); @@ -500,7 +500,7 @@ void addUsersToGroup(long studyId, String groupId, List members, ClientS @Override public OpenCGAResult addUsersToGroup(long studyId, String groupId, List members) throws CatalogDBException { - if (ListUtils.isEmpty(members)) { + if (CollectionUtils.isEmpty(members)) { throw new CatalogDBException("List of 'members' is missing or empty."); } @@ -589,56 +589,145 @@ public OpenCGAResult syncGroup(long studyId, String groupId, Group.Sync s return new OpenCGAResult<>(studyCollection.update(query, updates, null)); } - // TODO: Make this transactional @Override public OpenCGAResult resyncUserWithSyncedGroups(String user, List groupList, String authOrigin) - throws CatalogDBException { + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { if (StringUtils.isEmpty(user)) { throw new CatalogDBException("Missing user field"); } - // 1. Take the user out from all synced groups - Document query = new Document() - .append(QueryParams.GROUPS.key(), new Document("$elemMatch", new Document() - .append("userIds", user) - .append("syncedFrom.authOrigin", authOrigin) - )); - Bson pull = Updates.pull("groups.$.userIds", user); - - // Pull the user while it still belongs to a synced group - QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); - DataResult update; - do { - update = studyCollection.update(query, pull, multi); - } while (update.getNumUpdated() > 0); - - // 2. Add user to all synced groups - if (groupList != null && groupList.size() > 0) { - // Add the user to all the synced groups matching - query = new Document() + return runTransaction(clientSession -> { + // 1. Take the user out from all synced groups + Document query = new Document() .append(QueryParams.GROUPS.key(), new Document("$elemMatch", new Document() - .append("userIds", new Document("$ne", user)) - .append("syncedFrom.remoteGroup", new Document("$in", groupList)) + .append("userIds", user) .append("syncedFrom.authOrigin", authOrigin) )); - Document push = new Document("$addToSet", new Document("groups.$.userIds", user)); + Bson pull = Updates.pull("groups.$.userIds", user); + + // Pull the user while it still belongs to a synced group + QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); + DataResult update; do { - update = studyCollection.update(query, push, multi); + update = studyCollection.update(clientSession, query, pull, multi); } while (update.getNumUpdated() > 0); - // We need to be updated with the internal @members group, so we fetch all the studies where the user has been added - // and attempt to add it to the each @members group - query = new Document() - .append(QueryParams.GROUP_USER_IDS.key(), user) - .append(QueryParams.GROUP_SYNCED_FROM_AUTH_ORIGIN.key(), authOrigin); - DataResult studyDataResult = studyCollection.find(query, studyConverter, new QueryOptions(QueryOptions.INCLUDE, - QueryParams.UID.key())); - for (Study study : studyDataResult.getResults()) { - addUsersToGroup(study.getUid(), "@members", Arrays.asList(user)); + // 2. Add user to all synced groups + if (groupList != null && groupList.size() > 0) { + // Add the user to all the synced groups matching + query = new Document() + .append(QueryParams.GROUPS.key(), new Document("$elemMatch", new Document() + .append("userIds", new Document("$ne", user)) + .append("syncedFrom.remoteGroup", new Document("$in", groupList)) + .append("syncedFrom.authOrigin", authOrigin) + )); + Document push = new Document("$addToSet", new Document("groups.$.userIds", user)); + do { + update = studyCollection.update(clientSession, query, push, multi); + } while (update.getNumUpdated() > 0); + + // We need to be updated with the internal @members group, so we fetch all the studies where the user has been added + // and attempt to add it to all @members groups + query = new Document() + .append(QueryParams.GROUP_USER_IDS.key(), user) + .append(QueryParams.GROUP_SYNCED_FROM_AUTH_ORIGIN.key(), authOrigin); + DataResult studyDataResult = studyCollection.find(clientSession, query, studyConverter, + new QueryOptions(QueryOptions.INCLUDE, QueryParams.UID.key())); + for (Study study : studyDataResult.getResults()) { + addUsersToGroup(clientSession, study.getUid(), "@members", Collections.singletonList(user)); + } } + + return OpenCGAResult.empty(Group.class); + }); + } + + @Override + public OpenCGAResult updateUserFromGroups(String user, List studyUids, List groupList, + ParamUtils.AddRemoveAction action) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + + if (StringUtils.isEmpty(user)) { + throw new CatalogParameterException("Missing user parameter"); + } + if (action == null) { + throw new CatalogParameterException("Missing action parameter"); + } + if (CollectionUtils.isEmpty(groupList)) { + throw new CatalogParameterException("Missing list of groups"); } - return OpenCGAResult.empty(); + // Fix group ids + List fixedGroupList = groupList.stream() + .map((group) -> { + if (!group.startsWith("@")) { + return "@" + group; + } + return group; + }) + .collect(Collectors.toList()); + + QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); + + return runTransaction(clientSession -> { + switch (action) { + case ADD: + Document addQuery = new Document() + // Do not apply changes in the admin study + .append(QueryParams.FQN.key(), new Document("$ne", ParamConstants.ADMIN_STUDY_FQN)) + // Add the user to all the groups matching the list + .append(QueryParams.GROUPS.key(), new Document("$elemMatch", new Document() + .append("userIds", new Document("$ne", user)) + .append("id", new Document("$in", fixedGroupList)) + )); + if (studyUids != null) { + addQuery.put(QueryParams.UID.key(), new Document("$in", studyUids)); + } + + Document push = new Document("$addToSet", new Document("groups.$.userIds", user)); + DataResult update; + do { + update = studyCollection.update(clientSession, addQuery, push, multi); + } while (update.getNumUpdated() > 0); + + // We need to be updated with the internal @members group, so we fetch all the studies where the user has been added + // and attempt to add it to all @members groups + addQuery = new Document() + .append(QueryParams.GROUP_USER_IDS.key(), user) + .append(QueryParams.GROUP_ID.key(), new Document("$in", fixedGroupList)); + if (studyUids != null) { + addQuery.put(QueryParams.UID.key(), new Document("$in", studyUids)); + } + + DataResult studyDataResult = studyCollection.find(clientSession, addQuery, studyConverter, + new QueryOptions(QueryOptions.INCLUDE, QueryParams.UID.key())); + for (Study study : studyDataResult.getResults()) { + addUsersToGroup(clientSession, study.getUid(), "@members", Collections.singletonList(user)); + } + break; + case REMOVE: + // 1. Take the user out from all groups + Document removeQuery = new Document() + .append(QueryParams.GROUPS.key(), new Document("$elemMatch", new Document() + .append("userIds", user) + .append("id", new Document("$in", fixedGroupList)) + )); + if (studyUids != null) { + removeQuery.put(QueryParams.UID.key(), new Document("$in", studyUids)); + } + Bson pull = Updates.pull("groups.$.userIds", user); + DataResult pullUpdate; + do { + pullUpdate = studyCollection.update(clientSession, removeQuery, pull, multi); + } while (pullUpdate.getNumUpdated() > 0); + + break; + default: + break; + } + + return OpenCGAResult.empty(Group.class); + }); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java index 3bffa773511..b57426ebf04 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AdminManager.java @@ -1,9 +1,9 @@ package org.opencb.opencga.catalog.managers; +import org.apache.commons.collections4.CollectionUtils; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.UserDBAdaptor; @@ -12,12 +12,18 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.study.Group; +import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + public class AdminManager extends AbstractManager { private final CatalogIOManager catalogIOManager; @@ -72,4 +78,97 @@ public OpenCGAResult userSearch(Query query, QueryOptions options, String } } + public OpenCGAResult updateGroups(String userId, List studyIds, List groupIds, + ParamUtils.AddRemoveAction action, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("userId", userId) + .append("studyIds", studyIds) + .append("groupIds", groupIds) + .append("action", action) + .append("token", token); + String authenticatedUser = catalogManager.getUserManager().getUserId(token); + try { + authorizationManager.checkIsInstallationAdministrator(authenticatedUser); + + // Check userId exists + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userId); + OpenCGAResult count = userDBAdaptor.count(query); + if (count.getNumMatches() == 0) { + throw new CatalogException("User '" + userId + "' not found."); + } + + boolean membersPassed = groupIds.stream().anyMatch(g -> g.startsWith("@") + ? ParamConstants.MEMBERS_GROUP.equals(g) + : ParamConstants.MEMBERS_GROUP.equals("@" + g)); + if (action.equals(ParamUtils.AddRemoveAction.REMOVE) && membersPassed) { + // TODO: If @members is added for REMOVAL, we should also automatically remove all permissions. + throw new CatalogException("Operation not supported for '@members' group."); + } + + // Check studyIds exist + List studies = catalogManager.getStudyManager().resolveIds(studyIds, authenticatedUser); + List studyUids = new ArrayList<>(studies.size()); + for (Study study : studies) { + if (ParamConstants.ADMIN_STUDY_FQN.equals(study.getFqn())) { + if (CollectionUtils.isNotEmpty(studyIds)) { + // Only fail if the user is passing the list of study ids + throw new CatalogException("Cannot perform this operation on administration study '" + study.getFqn() + "'."); + } + } else { + studyUids.add(study.getUid()); + } + } + + OpenCGAResult result = studyDBAdaptor.updateUserFromGroups(userId, studyUids, groupIds, action); + + auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return result; + } catch (CatalogException e) { + auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + + /** + * Add a user to all the remote groups he/she may belong for a particular authentication origin. + * Also remove the user from other groups he/she may have been associated in the past for the same authentication origin. + * + * @param userId User id. + * @param remoteGroupIds List of group ids the user must be associated with. + * @param authenticationOriginId The authentication origin the groups must be associated to. + * @param token Administrator token. + * @return An empty OpenCGAResult. + * @throws CatalogException If the token is invalid or belongs to a user other thant the Administrator and if userId does not exist. + */ + public OpenCGAResult syncRemoteGroups(String userId, List remoteGroupIds, + String authenticationOriginId, String token) throws CatalogException { + ObjectMap auditParams = new ObjectMap() + .append("userId", userId) + .append("remoteGroupIds", remoteGroupIds) + .append("authenticationOriginId", authenticationOriginId) + .append("token", token); + String authenticatedUser = catalogManager.getUserManager().getUserId(token); + try { + authorizationManager.checkIsInstallationAdministrator(authenticatedUser); + + // Check userId exists + Query query = new Query(UserDBAdaptor.QueryParams.ID.key(), userId); + OpenCGAResult count = userDBAdaptor.count(query); + if (count.getNumMatches() == 0) { + throw new CatalogException("User '" + userId + "' not found."); + } + OpenCGAResult result = studyDBAdaptor.resyncUserWithSyncedGroups(userId, remoteGroupIds, authenticationOriginId); + + auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return result; + } catch (CatalogException e) { + auditManager.audit(userId, Enums.Action.UPDATE_USERS_FROM_STUDY_GROUP, Enums.Resource.STUDY, "", "", "", "", auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java index 1f0a9ae40c6..b95d3b0ff0e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AuditManager.java @@ -184,7 +184,7 @@ public void audit(String operationId, String userId, Enums.Action action, Enums. public void audit(String operationId, String userId, Enums.Action action, Enums.Resource resource, String resourceId, String resourceUuid, String studyId, String studyUuid, ObjectMap params, AuditRecord.Status status, ObjectMap attributes) { - String apiVersion = GitRepositoryState.get().getBuildVersion(); + String apiVersion = GitRepositoryState.getInstance().getBuildVersion(); Date date = TimeUtils.getDate(); String auditId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java index 6be7ad1be28..b1c2df87d1f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/stats/solr/CatalogSolrManager.java @@ -268,7 +268,7 @@ public DataResult facetedQuery(Study study, String collection, Query //***************** PRIVATE ****************/ private void populateConfigCollectionMap() { - String version = GitRepositoryState.get().getBuildVersion(); + String version = GitRepositoryState.getInstance().getBuildVersion(); CONFIGS_COLLECTION.put(DATABASE_PREFIX + COHORT_SOLR_COLLECTION, COHORT_CONF_SET + "-" + version); CONFIGS_COLLECTION.put(DATABASE_PREFIX + FILE_SOLR_COLLECTION, FILE_CONF_SET + "-" + version); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java index 16a638e0e2d..bb59584363b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java @@ -125,7 +125,7 @@ public void validate(TemplateManifest manifest) throws CatalogException { } // Check version - GitRepositoryState gitRepositoryState = GitRepositoryState.get(); + GitRepositoryState gitRepositoryState = GitRepositoryState.getInstance(); String version = gitRepositoryState.getBuildVersion(); String versionShort; if (version.contains("-")) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java index 6459083abab..b56966a44e8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/PedigreeGraphUtils.java @@ -25,7 +25,7 @@ public class PedigreeGraphUtils { public static final String PEDIGREE_JSON_FILENAME = "ped_coords.json"; public static final String PEDIGREE_TSV_FILENAME = "ped_coords.tsv"; - public static final String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion(); + public static final String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.getInstance().getBuildVersion(); public static PedigreeGraph getPedigreeGraph(Family family, Path openCgaHome, Path scratchDir) throws IOException { PedigreeGraph pedigreeGraph = new PedigreeGraph(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index f8c7b20e550..5791005394c 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -26,6 +26,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -208,7 +209,7 @@ public void removeUsersFromAllGroups() throws CatalogDBException, CatalogParamet } @Test - public void resyncUserWithSyncedGroups() throws CatalogDBException { + public void resyncUserWithSyncedGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { // We create synced groups and not synced groups in study 5 Group group = new Group("@notSyncedGroup", Arrays.asList("user1", "user2", "user3")); catalogStudyDBAdaptor.createGroup(5L, group); @@ -275,4 +276,74 @@ public void resyncUserWithSyncedGroups() throws CatalogDBException { .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup2", "@syncedGroup3", "@members"))); } + @Test + public void updateUserToGroups() throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + // We create synced groups and not synced groups in study 5 + Group group = new Group("@notSyncedGroup", Collections.emptyList()); + catalogStudyDBAdaptor.createGroup(5L, group); + group.setId("@syncedGroup1"); + group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); + catalogStudyDBAdaptor.createGroup(5L, group); + group.setId("@syncedGroup2"); + group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); + catalogStudyDBAdaptor.createGroup(5L, group); + group.setId("@syncedGroup3"); + group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); + catalogStudyDBAdaptor.createGroup(5L, group); + + // We create the same synced groups and not synced groups in study 9 + group = new Group("@notSyncedGroup", Collections.emptyList()); + catalogStudyDBAdaptor.createGroup(9L, group); + group.setId("@syncedGroup1"); + group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup1")); + catalogStudyDBAdaptor.createGroup(9L, group); + group.setId("@syncedGroup2"); + group.setSyncedFrom(new Group.Sync("origin1", "@syncedGroup2")); + catalogStudyDBAdaptor.createGroup(9L, group); + group.setId("@syncedGroup3"); + group.setSyncedFrom(new Group.Sync("otherOrigin", "@syncedGroup3")); + catalogStudyDBAdaptor.createGroup(9L, group); + group = new Group("@otherNotSyncedGroup", Collections.emptyList()); + catalogStudyDBAdaptor.createGroup(9L, group); + + catalogStudyDBAdaptor.updateUserFromGroups("user2", null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); + DataResult groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); + DataResult groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); + assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); + assertEquals(3, groupsStudy1.getNumResults()); + assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + assertEquals(3, groupsStudy2.getNumResults()); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + + catalogStudyDBAdaptor.updateUserFromGroups("user2", null, Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); + assertEquals(1, groupsStudy1.getNumResults()); + assertEquals(1, groupsStudy2.getNumResults()); + assertEquals("@members", groupsStudy1.first().getId()); + assertEquals("@members", groupsStudy2.first().getId()); + + catalogStudyDBAdaptor.updateUserFromGroups("user2", Arrays.asList(5L, 9L), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.ADD); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); + assertEquals(groupsStudy1.getNumResults(), groupsStudy2.getNumResults()); + assertEquals(3, groupsStudy1.getNumResults()); + assertTrue(groupsStudy1.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + assertEquals(3, groupsStudy2.getNumResults()); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + + catalogStudyDBAdaptor.updateUserFromGroups("user2", Collections.singletonList(5L), Arrays.asList("syncedGroup1", "notSyncedGroup"), ParamUtils.AddRemoveAction.REMOVE); + groupsStudy1 = catalogStudyDBAdaptor.getGroup(5L, null, Arrays.asList("user2")); + groupsStudy2 = catalogStudyDBAdaptor.getGroup(9L, null, Arrays.asList("user2")); + assertEquals(1, groupsStudy1.getNumResults()); + assertEquals("@members", groupsStudy1.first().getId()); + assertEquals(3, groupsStudy2.getNumResults()); + assertTrue(groupsStudy2.getResults().stream().map(Group::getId).collect(Collectors.toList()) + .containsAll(Arrays.asList("@notSyncedGroup", "@syncedGroup1", "@members"))); + } + } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java index cc0a758e602..677262d5cf8 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/stats/solr/SolrExternalResource.java @@ -41,7 +41,7 @@ public void before() throws Exception { Path rootDir = getOpencgaHome(); - String version = GitRepositoryState.get().getBuildVersion(); + String version = GitRepositoryState.getInstance().getBuildVersion(); // Copy configuration copyConfiguration("cohort-managed-schema", CatalogSolrManager.COHORT_CONF_SET + "-" + version); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index a2c5da2dc72..21f5c79fecd 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -17,7 +17,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; public class CellBaseValidator { @@ -94,17 +94,33 @@ public CellBaseConfiguration getCellBaseConfiguration() { return new CellBaseConfiguration(getURL(), getVersion(), getDataRelease(), getToken()); } - public String getLatestActiveDataRelease() throws IOException { + public String getDefaultDataRelease() throws IOException { if (supportsDataRelease()) { - Optional dataRelease = cellBaseClient.getMetaClient().dataReleases() - .allResults() - .stream() - .filter(DataRelease::isActive) - .max(Comparator.comparing(DataRelease::getDate)); - if (!dataRelease.isPresent()) { + List dataReleases = cellBaseClient.getMetaClient().dataReleases() + .allResults(); + DataRelease dataRelease = null; + if (supportsDataReleaseActiveByDefaultIn()) { + // ActiveByDefault versions are stored in form `v.` , i.e. v5.5 , v5.7, ... + String majorMinor = "v" + getVersionFromServerMajorMinor(); + List drs = dataReleases + .stream() + .filter(dr -> dr.getActiveByDefaultIn() != null && dr.getActiveByDefaultIn().contains(majorMinor)) + .collect(Collectors.toList()); + if (drs.size() == 1) { + dataRelease = drs.get(0); + } else if (drs.size() > 1) { + throw new IllegalArgumentException("More than one default active data releases found on cellbase " + this); + } + } else { + dataRelease = dataReleases + .stream() + .filter(DataRelease::isActive) + .max(Comparator.comparing(DataRelease::getDate)).orElse(null); + } + if (dataRelease == null) { throw new IllegalArgumentException("No active data releases found on cellbase " + this); } else { - return String.valueOf(dataRelease.get().getRelease()); + return String.valueOf(dataRelease.getRelease()); } } else { return null; @@ -149,7 +165,7 @@ public CellBaseConfiguration validate(boolean autoComplete) throws IOException { String dataRelease = getDataRelease(); if (dataRelease == null) { if (autoComplete) { - cellBaseConfiguration.setDataRelease(getLatestActiveDataRelease()); + cellBaseConfiguration.setDataRelease(getDefaultDataRelease()); } else { throw new IllegalArgumentException("Missing DataRelease for cellbase " + "url: '" + getURL() + "'" @@ -202,10 +218,19 @@ public boolean supportsDataRelease() throws IOException { } public static boolean supportsDataRelease(String serverVersion) { - // Data Release support starts at versio 5.1.0 + // Data Release support starts at version 5.1.0 return VersionUtils.isMinVersion("5.1.0", serverVersion); } + public boolean supportsDataReleaseActiveByDefaultIn() throws IOException { + return supportsDataReleaseActiveByDefaultIn(getVersionFromServer()); + } + + public static boolean supportsDataReleaseActiveByDefaultIn(String serverVersion) { + // Data Release Default Active In Version support starts at version 5.5.0 , TASK-4157 + return VersionUtils.isMinVersion("5.5.0", serverVersion, true); + } + public String getVersionFromServerMajor() throws IOException { return major(getVersionFromServer()); } @@ -217,10 +242,13 @@ public String getVersionFromServerMajorMinor() throws IOException { } private static String major(String version) { +// return String.valueOf(new VersionUtils.Version(version).getMajor()); return version.split("\\.")[0]; } private static String majorMinor(String version) { +// VersionUtils.Version v = new VersionUtils.Version(version); +// return v.getMajor() + "." + v.getMinor(); String[] split = version.split("\\."); if (split.length > 1) { version = split[0] + "." + split[1]; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/GitRepositoryState.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/GitRepositoryState.java index 80f1563dec6..790d8adcbe0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/GitRepositoryState.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/GitRepositoryState.java @@ -32,6 +32,7 @@ public class GitRepositoryState { public static final String DEFAULT_RESOURCE_NAME = "git.properties"; private static GitRepositoryState gitRepositoryState; + private static Properties properties; private static final Logger logger = LoggerFactory.getLogger(GitRepositoryState.class); private String tags; // =${git.tags} // comma separated tag names @@ -57,9 +58,16 @@ public class GitRepositoryState { private String buildHost; // =${git.build.host} private String buildVersion; // =${git.build.version} - public static GitRepositoryState get() { + public static String get(String key) { + if (properties == null) { + getInstance(); + } + return properties.getProperty(key); + } + + public static GitRepositoryState getInstance() { if (gitRepositoryState == null) { - Properties properties = new Properties(); + properties = new Properties(); InputStream stream = null; try { stream = GitRepositoryState.class.getClassLoader().getResourceAsStream(DEFAULT_RESOURCE_NAME); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/VersionUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/VersionUtils.java index fb70ed94cb9..4e32c6ada32 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/VersionUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/VersionUtils.java @@ -13,7 +13,11 @@ public static List order(List versions) { } public static boolean isMinVersion(String minVersion, String version) { - return new Version(minVersion).compareTo(new Version(version)) <= 0; + return isMinVersion(minVersion, version, false); + } + + public static boolean isMinVersion(String minVersion, String version, boolean ignorePreReleaseVersioning) { + return new Version(minVersion).compareTo(new Version(version), ignorePreReleaseVersioning) <= 0; } public static class Version implements Comparable { @@ -48,6 +52,12 @@ public static class Version implements Comparable { return o1.other.compareTo(o2.other); }); + public static final Comparator COMPARATOR_NO_PR = Comparator + .comparingInt(Version::getMajor) + .thenComparingInt(Version::getMinor) + .thenComparingInt(Version::getPatch) + .thenComparingInt(Version::getRepatch); + public Version(String version) { String[] split = StringUtils.split(version, ".", 4); major = Integer.parseInt(split[0]); @@ -81,6 +91,14 @@ public int compareTo(Version o) { return COMPARATOR.compare(this, o); } + public int compareTo(Version o, boolean ignorePreReleaseVersioning) { + if (ignorePreReleaseVersioning) { + return COMPARATOR_NO_PR.compare(this, o); + } else { + return COMPARATOR.compare(this, o); + } + } + public int getMajor() { return major; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/docs/DocBuilder.java b/opencga-core/src/main/java/org/opencb/opencga/core/docs/DocBuilder.java index 53f0014dd27..a647bbd8b43 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/docs/DocBuilder.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/docs/DocBuilder.java @@ -44,7 +44,7 @@ public static void main(String[] args) { config.setType(DocFactory.DocFactoryType.MARKDOWN); config.setOutputDir("/workspace/opencga/docs/data-models/"); config.setGithubServerURL("https://github.com/opencb/opencga/tree/" - + GitRepositoryState.get().getBranch() + "/opencga-core"); + + GitRepositoryState.getInstance().getBranch() + "/opencga-core"); config.setJsondir("/workspace/opencga/opencga-core/src/main/resources/doc/json"); config.setGitbookServerURL("https://docs.opencga.opencb.org/data-models"); try { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/Metadata.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/Metadata.java index a0a67ee823a..99fb1bedeaa 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/Metadata.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/Metadata.java @@ -31,7 +31,7 @@ public class Metadata { public Metadata() { - this(GitRepositoryState.get().getBuildVersion(), TimeUtils.getTime()); + this(GitRepositoryState.getInstance().getBuildVersion(), TimeUtils.getTime()); } public Metadata(String version, String creationDate) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserUpdateGroup.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserUpdateGroup.java new file mode 100644 index 00000000000..ce05d03ccc2 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/admin/UserUpdateGroup.java @@ -0,0 +1,43 @@ +package org.opencb.opencga.core.models.admin; + +import java.util.List; + +public class UserUpdateGroup { + private List studyIds; + private List groupIds; + + public UserUpdateGroup() { + } + + public UserUpdateGroup(List studyIds, List groupIds) { + this.studyIds = studyIds; + this.groupIds = groupIds; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("UserUpdateGroup{"); + sb.append("studyIds=").append(studyIds); + sb.append(", groupIds=").append(groupIds); + sb.append('}'); + return sb.toString(); + } + + public List getStudyIds() { + return studyIds; + } + + public UserUpdateGroup setStudyIds(List studyIds) { + this.studyIds = studyIds; + return this; + } + + public List getGroupIds() { + return groupIds; + } + + public UserUpdateGroup setGroupIds(List groupIds) { + this.groupIds = groupIds; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java index 732cb04adaf..a934efdc3ee 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/InternalStatus.java @@ -120,8 +120,8 @@ protected void init(String statusId, String statusName, String description) { super.name = statusName; super.description = description; super.date = TimeUtils.getTime(); - this.version = GitRepositoryState.get().getBuildVersion(); - this.commit = GitRepositoryState.get().getCommitId(); + this.version = GitRepositoryState.getInstance().getBuildVersion(); + this.commit = GitRepositoryState.getInstance().getCommitId(); } @Override diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/Status.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/Status.java index da1778bf650..dc811356a1a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/Status.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/Status.java @@ -61,8 +61,8 @@ public Status(Type name, String step, Date date) { public static Status initStatus() { Status status = new Status(); - status.version = GitRepositoryState.get().getBuildVersion(); - status.commit = GitRepositoryState.get().getCommitId(); + status.version = GitRepositoryState.getInstance().getBuildVersion(); + status.commit = GitRepositoryState.getInstance().getCommitId(); return status; } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/models/RestApi.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/models/RestApi.java index 84ae7f75aae..cff66f13e62 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/models/RestApi.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/models/RestApi.java @@ -28,7 +28,7 @@ public class RestApi { private List categories; public RestApi() { - this(GitRepositoryState.get().getBuildVersion(), GitRepositoryState.get().getCommitId(), new ArrayList<>()); + this(GitRepositoryState.getInstance().getBuildVersion(), GitRepositoryState.getInstance().getCommitId(), new ArrayList<>()); } public RestApi(String version, String commit, List categories) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java index e075e938983..528853faeb6 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java @@ -80,9 +80,9 @@ public MetaWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpSe public Response getAbout() { Map info = new HashMap<>(5); info.put("Program", "OpenCGA (OpenCB)"); - info.put("Version", GitRepositoryState.get().getBuildVersion()); - info.put("Git branch", GitRepositoryState.get().getBranch()); - info.put("Git commit", GitRepositoryState.get().getCommitId()); + info.put("Version", GitRepositoryState.getInstance().getBuildVersion()); + info.put("Git branch", GitRepositoryState.getInstance().getBranch()); + info.put("Git commit", GitRepositoryState.getInstance().getCommitId()); info.put("Description", "Big Data platform for processing and analysing NGS data"); OpenCGAResult queryResult = new OpenCGAResult(); queryResult.setTime(0); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java index d152cf17277..03a39102d9e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java @@ -260,8 +260,8 @@ static void init(String opencgaHomeStr) { } logger.info("| OpenCGA REST successfully started!"); - logger.info("| - Version " + GitRepositoryState.get().getBuildVersion()); - logger.info("| - Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); + logger.info("| - Version " + GitRepositoryState.getInstance().getBuildVersion()); + logger.info("| - Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); logger.info("========================================================================\n"); } @@ -428,8 +428,8 @@ private static void addErrorEvent(RestResponse response, Throwable e) { } public static void setFederationServer(OpenCGAResult result, String baseuri) { - result.setFederationNode(new FederationNode(baseuri, GitRepositoryState.get().getCommitId(), - GitRepositoryState.get().getBuildVersion())); + result.setFederationNode(new FederationNode(baseuri, GitRepositoryState.getInstance().getCommitId(), + GitRepositoryState.getInstance().getBuildVersion())); } public static void logResponse(Response.StatusType statusInfo, RestResponse queryResponse, long startTime, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java index f07c7ad3ecb..835c9632af1 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java @@ -16,8 +16,6 @@ package org.opencb.opencga.server.rest.admin; -import org.opencb.opencga.core.models.user.AuthenticationResponse; -import org.opencb.opencga.core.tools.annotations.*; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.ObjectMap; @@ -31,6 +29,7 @@ import org.opencb.opencga.analysis.sample.SampleIndexTask; import org.opencb.opencga.catalog.db.api.MetaDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.admin.*; @@ -39,8 +38,10 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.Account; +import org.opencb.opencga.core.models.user.AuthenticationResponse; import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.tools.annotations.*; import org.opencb.opencga.server.rest.OpenCGAWSServer; import javax.servlet.http.HttpServletRequest; @@ -153,6 +154,22 @@ public Response remoteImport(@ApiParam(value = "JSON containing the parameters", } } + @POST + @Path("/users/{user}/groups/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Add or remove users from existing groups", response = Group.class) + public Response updateGroups( + @ApiParam(value = ParamConstants.USER_DESCRIPTION) @PathParam(ParamConstants.USER) String user, + @ApiParam(value = "Action to be performed: ADD or REMOVE user to/from groups", allowableValues = "ADD,REMOVE", + defaultValue = "ADD") @QueryParam("action") ParamUtils.AddRemoveAction action, + @ApiParam(value = "JSON containing the parameters", required = true) UserUpdateGroup updateParams) { + try { + return createOkResponse(catalogManager.getAdminManager().updateGroups(user, updateParams.getStudyIds(), updateParams.getGroupIds(), action, token)); + } catch (Exception e) { + return createErrorResponse(e); + } + } + @POST @Path("/users/sync") @ApiOperation(value = "Synchronise a group of users from an authentication origin with a group in a study from catalog", response = Group.class, diff --git a/opencga-server/src/main/resources/parserAbstractMethods.template b/opencga-server/src/main/resources/parserAbstractMethods.template index a06cc76c2f1..4d25b76e871 100644 --- a/opencga-server/src/main/resources/parserAbstractMethods.template +++ b/opencga-server/src/main/resources/parserAbstractMethods.template @@ -18,8 +18,8 @@ if (parsedCommand.isEmpty()) { System.err.println(""); System.err.println("Program: OpenCGA (OpenCB)"); - System.err.println("Version: " + GitRepositoryState.get().getBuildVersion()); - System.err.println("Git commit: " + GitRepositoryState.get().getCommitId()); + System.err.println("Version: " + GitRepositoryState.getInstance().getBuildVersion()); + System.err.println("Git commit: " + GitRepositoryState.getInstance().getCommitId()); System.err.println("Description: Big Data platform for processing and analysing NGS data"); System.err.println(""); System.err.println("Usage: opencga.sh [-h|--help] [--version] [options]"); diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageAdminMain.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageAdminMain.java index 85313d4e4c3..0825354d734 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageAdminMain.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageAdminMain.java @@ -48,8 +48,8 @@ public static int privateMain(String[] args) { String parsedCommand = adminCliOptionsParser.getCommand(); if (parsedCommand == null || parsedCommand.isEmpty()) { if (adminCliOptionsParser.getGeneralOptions().version) { - System.out.println("Version " + GitRepositoryState.get().getBuildVersion()); - System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); + System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion()); + System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); return 0; } else if (adminCliOptionsParser.getGeneralOptions().help) { adminCliOptionsParser.printUsage(); diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageMain.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageMain.java index 05da1e97637..9f7ec6fd321 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageMain.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/StorageMain.java @@ -48,9 +48,9 @@ public static int privateMain(String[] args) { String parsedCommand = cliOptionsParser.getCommand(); if (parsedCommand == null || parsedCommand.isEmpty()) { if (cliOptionsParser.getGeneralOptions().version) { - System.out.println("Version " + GitRepositoryState.get().getBuildVersion()); - System.out.println("Git version: " + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); -// System.out.println(GitRepositoryState.get()); + System.out.println("Version " + GitRepositoryState.getInstance().getBuildVersion()); + System.out.println("Git version: " + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); +// System.out.println(GitRepositoryState.getInstance()); return 0; } else if (cliOptionsParser.getGeneralOptions().help) { cliOptionsParser.printUsage(); diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/ClientCliOptionsParser.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/ClientCliOptionsParser.java index 9c430b284c1..fb567276992 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/ClientCliOptionsParser.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/ClientCliOptionsParser.java @@ -82,8 +82,8 @@ public void printUsage() { if (parsedCommand.isEmpty()) { System.err.println(""); System.err.println("Program: OpenCGA Storage (OpenCB)"); - System.err.println("Version: " + GitRepositoryState.get().getBuildVersion()); - System.err.println("Git commit: " + GitRepositoryState.get().getCommitId()); + System.err.println("Version: " + GitRepositoryState.getInstance().getBuildVersion()); + System.err.println("Git commit: " + GitRepositoryState.getInstance().getCommitId()); System.err.println("Description: Big Data platform for processing and analysing NGS data"); System.err.println(""); System.err.println("Usage: opencga-storage.sh [-h|--help] [--version] [options]"); diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/server/AdminCliOptionsParser.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/server/AdminCliOptionsParser.java index 7aedfbd4e85..86a4bcca419 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/server/AdminCliOptionsParser.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/server/AdminCliOptionsParser.java @@ -76,8 +76,8 @@ public void printUsage() { if (parsedCommand.isEmpty()) { System.err.println(""); System.err.println("Program: OpenCGA Storage Admin (OpenCB)"); - System.err.println("Version: " + GitRepositoryState.get().getBuildVersion()); - System.err.println("Git commit: " + GitRepositoryState.get().getCommitId()); + System.err.println("Version: " + GitRepositoryState.getInstance().getBuildVersion()); + System.err.println("Git commit: " + GitRepositoryState.getInstance().getCommitId()); System.err.println("Description: Big Data platform for processing and analysing NGS data"); System.err.println(""); System.err.println("Usage: opencga-storage-admin.sh [-h|--help] [--version] [options]"); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverter.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverter.java index 1ec92d91ea9..d2114731bc1 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverter.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantMetadataConverter.java @@ -48,7 +48,7 @@ public VariantMetadata toVariantMetadata(VariantQueryProjection variantQueryProj // .setDate(Date.from(Instant.now()).toString()) .setCreationDate(TimeUtils.getTime()) .setStudies(studies) - .setVersion(GitRepositoryState.get().getDescribeShort()) + .setVersion(GitRepositoryState.getInstance().getDescribeShort()) .setSpecies(species) .build(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java index ccb8106a2c0..1f679ace47e 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java @@ -67,7 +67,8 @@ public static List data() { new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5.1", "grch38", "1"}, new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5.1", "grch38", "2"}, new Object[]{"https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "grch37", "1"}, - new Object[]{"https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "grch38", "2"}); + new Object[]{"https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "grch38", "2"}, + new Object[]{"https://uk.ws.zettagenomics.com/cellbase/", "v5.4", "grch38", "3"}); } @Parameter(0) diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index 122ef2da4af..031360d3298 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -63,7 +63,7 @@ protected void before() throws Throwable { Path rootDir = getTmpRootDir(); - String configSet = "opencga-variant-configset-" + GitRepositoryState.get().getBuildVersion(); + String configSet = "opencga-variant-configset-" + GitRepositoryState.getInstance().getBuildVersion(); // Copy configuration getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml"); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java index 7ba0a4faf85..ef8f97152ad 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/executors/MRExecutor.java @@ -51,7 +51,7 @@ public MRExecutor init(ObjectMap options) { public static String getJarWithDependencies(ObjectMap options) throws StorageEngineException { String jar = options.getString(MR_JAR_WITH_DEPENDENCIES.key(), null); if (jar == null) { - jar = "opencga-storage-hadoop-core-" + GitRepositoryState.get().getBuildVersion() + "-jar-with-dependencies.jar"; + jar = "opencga-storage-hadoop-core-" + GitRepositoryState.getInstance().getBuildVersion() + "-jar-with-dependencies.jar"; // throw new StorageEngineException("Missing option " + MR_JAR_WITH_DEPENDENCIES); } if (!Paths.get(jar).isAbsolute()) { diff --git a/opencga-test/src/main/java/org/opencb/opencga/test/cli/OptionsParser.java b/opencga-test/src/main/java/org/opencb/opencga/test/cli/OptionsParser.java index efd41c2b319..256df4315ee 100644 --- a/opencga-test/src/main/java/org/opencb/opencga/test/cli/OptionsParser.java +++ b/opencga-test/src/main/java/org/opencb/opencga/test/cli/OptionsParser.java @@ -76,8 +76,8 @@ public static void printVersion() { private static Map getVersionMap() { Map versionMap = new HashMap<>(); - versionMap.put("OpenCGA Test version:", GitRepositoryState.get().getBuildVersion()); - versionMap.put("Git version:", "" + GitRepositoryState.get().getBranch() + " " + GitRepositoryState.get().getCommitId()); + versionMap.put("OpenCGA Test version:", GitRepositoryState.getInstance().getBuildVersion()); + versionMap.put("Git version:", "" + GitRepositoryState.getInstance().getBranch() + " " + GitRepositoryState.getInstance().getCommitId()); versionMap.put("Program:", "OpenCGA-test (OpenCB)"); versionMap.put("Description:", "Data generation application for the openCGA platform"); return versionMap; diff --git a/pom.xml b/pom.xml index 778d57c8df0..4d35b06f169 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ 2.10.0-SNAPSHOT pom OpenCGA - OenCGA projects implements a big data platform for genomic data analysis + OpenCGA projects implements a big data platform for genomic data analysis http://docs.opencb.org/display/opencga/ @@ -42,12 +42,13 @@ - - 2.8.2.1 - 5.4.0 - 2.8.0 - 4.8.0 - 2.8.2 + 2.10.0_dev + 2.10.0_dev + 5.6.0-SNAPSHOT + 2.10.0-SNAPSHOT + 4.10.0-SNAPSHOT + 2.10.0-SNAPSHOT + 0.2.0 2.11.4 @@ -84,7 +85,7 @@ false false - 2.22.2 + 3.1.0