Merge branch 'develop' into TASK-5516

.github/workflows/manual-deploy-docker.yml

@@ -11,10 +11,15 @@ on:
         type: string
         required: true
       hadoop:
-        type: string
+        type: choice
         description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
         required: false
-        default: "hdp3.1"
+        default: hdp3.1
+        options:
+          - hdp3.1
+          - hdi5.1
+          - emr6.1
+          - emr6.13
 
 jobs:
   build:
@@ -40,7 +45,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }},RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder

.github/workflows/release.yml

@@ -7,39 +7,90 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  # Add the build-hdp job
+  build-hdp:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
       maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      build_folder: build-folder
 
-  deploy-maven:
-    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-    needs: build
+  # Add the deploy-docker-hdp job that depends on the build-hdp job
+  deploy-docker-hdp:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build-hdp
     with:
-      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
+      build_folder: build-folder
     secrets: inherit
 
-  deploy-docker:
-    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: build
+  # Add the deploy-maven and deploy-python jobs they depend on the build-hdp job
+  deploy-maven:
+    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
+    needs: build-hdp
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ needs.build.outputs.version }}
+      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-python:
     uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
-    needs: build
+    needs: build-hdp
     with:
       cli: ./clients/python/python-build.sh push
       artifact: build-folder
     secrets: inherit
 
+  # Add the release job that depends on all the previous jobs
   release:
     uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: [ build, deploy-maven, deploy-docker, deploy-python ]
+    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python]
     with:
       artifact: build-folder
       file: |
-        opencga-client-${{ needs.build.outputs.version }}.tar.gz
-        clients/R/opencgaR_${{ needs.build.outputs.version }}.tar.gz
+        opencga-client-${{ needs.build-hdp.outputs.version }}.tar.gz
+        clients/R/opencgaR_${{ needs.build-hdp.outputs.version }}.tar.gz
+
+  # Compile and deploy other hadoop flavours
+  # Add the build-hdi and deploy-docker-hdi jobs
+  build-hdi:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -P hdi5.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      build_folder: build-folder-hdi
+
+  deploy-docker-hdi:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build-hdi
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdi.outputs.version }}-hdi5.1"
+      build_folder: build-folder-hdi
+    secrets: inherit
+
+  # Add the build-emr and deploy-docker-emr jobs
+  build-emr:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -P emr6.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      build_folder: build-folder-emr
+
+  deploy-docker-emr:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build-emr
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr.outputs.version }}-emr6.1"
+      build_folder: build-folder-emr
+    secrets: inherit
+
+  # Add the build-emr613 and deploy-docker-emr613 jobs
+  build-emr613:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -P emr6.13,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+      build_folder: build-folder-emr613
 
+  deploy-docker-emr613:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build-emr613
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-emr613.outputs.version }}-emr6.13"
+      build_folder: build-folder-emr613
+    secrets: inherit

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.1.0-SNAPSHOT</version>
+        <version>3.2.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-app/app/misc/clients/copyright.txt

@@ -0,0 +1,13 @@
+Copyright 2015-2024 OpenCB
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

opencga-app/app/misc/clients/java_client_generator.py

@@ -36,19 +36,7 @@ def get_imports(self):
 
         headers = []
         headers.append('/*')
-        headers.append('* Copyright 2015-' + str(date.today().year) + ' OpenCB')
-        headers.append('*')
-        headers.append('* Licensed under the Apache License, Version 2.0 (the "License");')
-        headers.append('* you may not use this file except in compliance with the License.')
-        headers.append('* You may obtain a copy of the License at')
-        headers.append('*')
-        headers.append('*     http://www.apache.org/licenses/LICENSE-2.0')
-        headers.append('*')
-        headers.append('* Unless required by applicable law or agreed to in writing, software')
-        headers.append('* distributed under the License is distributed on an "AS IS" BASIS,')
-        headers.append('* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.')
-        headers.append('* See the License for the specific language governing permissions and')
-        headers.append('* limitations under the License.')
+        headers.append(self.get_copyright_message(prefix='* '))
         headers.append('*/')
         headers.append('')
         # We need to calculate the Java package and no use: headers.append('package org.opencb.opencga.client.rest.clients;')
@@ -99,7 +87,6 @@ def get_class_definition(self, category):
         text.append('/**')
         text.append(' * This class contains methods for the {} webservices.'.format(
             self.categories[self.get_category_name(category)]))
-        text.append(' *{}Client version: {}'.format(' ' * 4, self.version))
         text.append(' *{}PATH: {}'.format(' ' * 4, self.get_category_path(category)))
         text.append(' */')
         text.append('public class {}Client extends {} {{'.format(self.categories[self.get_category_name(category)], parentClientClass))

opencga-app/app/misc/clients/javascript_client_generator.py

@@ -28,18 +28,10 @@ def __init__(self, server_url, output_dir):
         }
 
     def get_imports(self):
+        copyright = self.get_copyright_message(prefix=' * ')
         auto_msg = "\n * ".join(self.get_autogenerated_message())
         return (f'/**\n'
-                f' * Copyright 2015-2020 OpenCB\n'
-                f' * Licensed under the Apache License, Version 2.0 (the "License");\n'
-                f' * you may not use this file except in compliance with the License.\n'
-                f' * You may obtain a copy of the License at\n'
-                f' * http://www.apache.org/licenses/LICENSE-2.0\n'
-                f' * Unless required by applicable law or agreed to in writing, software\n'
-                f' * distributed under the License is distributed on an "AS IS" BASIS,\n'
-                f' * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'
-                f' * See the License for the specific language governing permissions and\n'
-                f' * limitations under the License.\n'
+                f'{copyright}\n'
                 f' * {auto_msg}'
                 f' \n *\n**/\n\n'
                 f'import OpenCGAParentClass from "./../opencga-parent-class.js";\n\n')

opencga-app/app/misc/clients/python_client_generator.py

@@ -70,7 +70,6 @@ def get_class_definition(self, category):
         text.append('{}"""'.format(' ' * 4))
         text.append('{}This class contains methods for the \'{}\' webservices'.format(' ' * 4,
                                                                                       self.get_category_name(category)))
-        text.append('{}Client version: {}'.format(' ' * 4, self.version))
         text.append('{}PATH: {}'.format(' ' * 4, category['path']))
         text.append('{}"""'.format(' ' * 4))
         text.append('')

opencga-app/app/misc/clients/r_client_generator.py

@@ -75,6 +75,7 @@ def get_class_definition(self, category):
                                                        myEndpoint['path'],
                                                        ", ".join(endpoint_params)))
         path_params = set(class_path_params)
+        path_params = sorted(list(path_params))
         text.append("#'\n#' @md")
         text.append("#' @seealso \\url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation")
         text.append("#' \\url{http://bioinfo.hpc.cam.ac.uk/opencga-prod/webservices/}")

opencga-app/app/misc/clients/rest_client_generator.py

@@ -56,14 +56,39 @@ def __init__(self, rest_api_file, output_dir):
             'Admin': 'Admin'
         }
 
+    @staticmethod
+    def get_copyright_message(prefix=""):
+        # copyright = [
+        #     'Copyright 2015-2024 OpenCB',
+        #     '',
+        #     'Licensed under the Apache License, Version 2.0 (the "License");',
+        #     'you may not use this file except in compliance with the License.',
+        #     'You may obtain a copy of the License at',
+        #     '',
+        #     '    http://www.apache.org/licenses/LICENSE-2.0',
+        #     '',
+        #     'Unless required by applicable law or agreed to in writing, software',
+        #     'distributed under the License is distributed on an "AS IS" BASIS,',
+        #     'WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.',
+        #     'See the License for the specific language governing permissions and',
+        #     'limitations under the License.'
+        # ]
+        copyright_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'copyright.txt')
+        print("Reading from file: ", copyright_file)
+        with open(copyright_file, 'r') as file:
+            copyright = file.read().splitlines()
+        ## Add prefix to each line
+        if prefix:
+            for line in range(len(copyright)):
+                copyright[line] = (prefix + copyright[line]).rstrip()
+        return "\n".join(copyright)
+
     @staticmethod
     def get_autogenerated_message():
-        date_ = datetime.now().strftime("%Y-%m-%d")
         return [
             'WARNING: AUTOGENERATED CODE',
             '',
             'This code was generated by a tool.',
-            'Autogenerated on: ' + date_,
             '',
             'Manual changes to this file may cause unexpected behavior in your application.',
             'Manual changes to this file will be overwritten if the code is regenerated.'

opencga-app/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.1.0-SNAPSHOT</version>
+        <version>3.2.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java

@@ -6,7 +6,7 @@
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.opencga.app.cli.admin.options.MigrationCommandOptions;
 import org.opencb.opencga.app.cli.main.io.Table;
-import org.opencb.opencga.app.migrations.v3_0_0.OrganizationMigration;
+import org.opencb.opencga.app.migrations.v3.v3_0_0.OrganizationMigration;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.managers.CatalogManager;
 import org.opencb.opencga.catalog.migration.Migration;

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java

@@ -1,19 +1,3 @@
-/*
-* Copyright 2015-2024-04-10 OpenCB
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
 package org.opencb.opencga.app.cli.main;
 
 import org.jline.reader.Candidate;
@@ -85,7 +69,7 @@ public abstract class OpenCgaCompleter implements Completer {
             .map(Candidate::new)
             .collect(toList());
 
-    private List<Candidate> usersList = asList( "anonymous","create","login","password","info","configs","configs-update","filters","password-reset","update")
+    private List<Candidate> usersList = asList( "anonymous","create","login","password","search","info","configs","configs-update","filters","password-reset","update")
             .stream()
             .map(Candidate::new)
             .collect(toList());

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java

@@ -1,19 +1,3 @@
-/*
-* Copyright 2015-2024-04-10 OpenCB
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
 package org.opencb.opencga.app.cli.main;
 
 import com.beust.jcommander.JCommander;
@@ -229,6 +213,7 @@ public OpencgaCliOptionsParser() {
         usersSubCommands.addCommand("create", usersCommandOptions.createCommandOptions);
         usersSubCommands.addCommand("login", usersCommandOptions.loginCommandOptions);
         usersSubCommands.addCommand("password", usersCommandOptions.passwordCommandOptions);
+        usersSubCommands.addCommand("search", usersCommandOptions.searchCommandOptions);
         usersSubCommands.addCommand("info", usersCommandOptions.infoCommandOptions);
         usersSubCommands.addCommand("configs", usersCommandOptions.configsCommandOptions);
         usersSubCommands.addCommand("configs-update", usersCommandOptions.updateConfigsCommandOptions);

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java

@@ -262,7 +262,6 @@ private RestResponse<Sample> searchUsers() throws Exception {
         queryParams.putIfNotNull("count", commandOptions.count);
         queryParams.putIfNotEmpty("organization", commandOptions.organization);
         queryParams.putIfNotEmpty("user", commandOptions.user);
-        queryParams.putIfNotEmpty("account", commandOptions.account);
         queryParams.putIfNotEmpty("authenticationId", commandOptions.authenticationId);
 
         return openCGAClient.getAdminClient().searchUsers(queryParams);

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OrganizationsCommandExecutor.java

@@ -22,6 +22,7 @@
 import org.opencb.opencga.core.models.organizations.OrganizationConfiguration;
 import org.opencb.opencga.core.models.organizations.OrganizationCreateParams;
 import org.opencb.opencga.core.models.organizations.OrganizationUpdateParams;
+import org.opencb.opencga.core.models.organizations.TokenConfiguration;
 import org.opencb.opencga.core.response.QueryType;
 import org.opencb.opencga.core.response.RestResponse;
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java

@@ -77,6 +77,9 @@ public void execute() throws Exception {
             case "password":
                 queryResponse = password();
                 break;
+            case "search":
+                queryResponse = search();
+                break;
             case "info":
                 queryResponse = info();
                 break;
@@ -188,6 +191,24 @@ private RestResponse<User> password() throws Exception {
         return openCGAClient.getUserClient().password(passwordChangeParams);
     }
 
+    private RestResponse<User> search() throws Exception {
+        logger.debug("Executing search in Users command line");
+
+        UsersCommandOptions.SearchCommandOptions commandOptions = usersCommandOptions.searchCommandOptions;
+
+        ObjectMap queryParams = new ObjectMap();
+        queryParams.putIfNotEmpty("include", commandOptions.include);
+        queryParams.putIfNotEmpty("exclude", commandOptions.exclude);
+        queryParams.putIfNotNull("limit", commandOptions.limit);
+        queryParams.putIfNotNull("skip", commandOptions.skip);
+        queryParams.putIfNotNull("count", commandOptions.count);
+        queryParams.putIfNotEmpty("organization", commandOptions.organization);
+        queryParams.putIfNotEmpty("id", commandOptions.id);
+        queryParams.putIfNotEmpty("authenticationId", commandOptions.authenticationId);
+
+        return openCGAClient.getUserClient().search(queryParams);
+    }
+
     private RestResponse<User> info() throws Exception {
         logger.debug("Executing info in Users command line");
 
@@ -284,7 +305,6 @@ private RestResponse<User> update() throws Exception {
             ObjectMap beanParams = new ObjectMap();
             putNestedIfNotEmpty(beanParams, "name",commandOptions.name, true);
             putNestedIfNotEmpty(beanParams, "email",commandOptions.email, true);
-            putNestedIfNotEmpty(beanParams, "organization",commandOptions.organization, true);
             putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true);
 
             userUpdateParams = JacksonUtils.getDefaultObjectMapper().copy()