Merge branch 'geonetwork:main' into main.prevent.delete.multiple.onli…

…ne.resources.iso.19139.schema
wangf1122 · Feb 14, 2024 · 2ae75c4 · 2ae75c4
2 parents f1975e3 + 4bf24e8
commit 2ae75c4
Show file tree

Hide file tree

Showing 168 changed files with 5,350 additions and 1,800 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -55,7 +55,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -80,7 +80,7 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
 
     - name: Remove SNAPSHOT jars from repository
       run: |

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -35,7 +35,7 @@ jobs:
       working-directory: docs/manual
       run: pip install --upgrade pip && pip install -r requirements.txt
     - name: Set up Maven
-      uses: stCarolas/setup-maven@v4
+      uses: stCarolas/setup-maven@v5
       with:
         maven-version: 3.6.3
     - name: Build with Maven
@@ -61,7 +61,7 @@ jobs:
         java-version: 11
         cache: 'maven'
     - name: Set up Maven
-      uses: stCarolas/setup-maven@v4
+      uses: stCarolas/setup-maven@v5
       with:
         maven-version: 3.6.3
     - name: Test with maven

diff --git a/.github/workflows/mvn-dep-tree.yml b/.github/workflows/mvn-dep-tree.yml
@@ -30,4 +30,4 @@ jobs:
           cache: maven
 
       - name: Submit Dependency Snapshot
-        uses: advanced-security/maven-dependency-submission-action@v3
+        uses: advanced-security/maven-dependency-submission-action@v4
diff --git a/SECURITY.md b/SECURITY.md
@@ -2,7 +2,7 @@
 
 The GeoNetwork community takes the security of the software and all services based on the software product seriously. On this page you can find the versions for which the community provides security patches. 
 
-If you believe you have found a security vulnerability in the software or an implementation of the software, please report it to [email protected] as described below. Do not publish the vulnerability in any public forums (such as Twitter/X, email list or issue tracker).
+If you believe you have found a security vulnerability in the software or an implementation of the software, please report it [here](https://github.com/geonetwork/core-geonetwork/security/advisories/new) as described below. Do not publish the vulnerability in any public forums (such as Twitter/X, email list or issue tracker).
 
 ## Supported Versions
 
@@ -23,8 +23,8 @@ If your organisation is making use of a GeoNetwork version that is no longer in
 
 If you encounter a security vulnerability in GeoNetwork please take care to report in a responsible fashion:
 
-* Keep exploit details out of mailing list and issue tracker (send details to the Project Steering Committee via [email protected])
+* Keep exploit details out of mailing list and issue tracker (instead provide details to the Project Steering Committee via the GitHub [Report a vulnerability](https://github.com/geonetwork/core-geonetwork/security/advisories/new) option link at the top of this page or send an email to [email protected])
 * Be prepared to work with community members on a solution
-* Keep in mind community members are volunteers and an extensive fix may require fundraising / resources
+* Keep in mind that community members are volunteers and an extensive fix may require fundraising / resources
 
 For more information see [How to contribute](https://github.com/geonetwork/core-geonetwork/wiki/How-to-contribute).
diff --git a/common/pom.xml b/common/pom.xml
@@ -175,8 +175,6 @@
     <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>
-      <version>20140107</version>
-      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.bitbucket.cowwoc</groupId>

diff --git a/core/pom.xml b/core/pom.xml
@@ -340,10 +340,6 @@
       <groupId>org.apache.xmlgraphics</groupId>
       <artifactId>xmlgraphics-commons</artifactId>
     </dependency>
-    <dependency>
-      <groupId>commons-fileupload</groupId>
-      <artifactId>commons-fileupload</artifactId>
-    </dependency>
     <dependency>
       <groupId>commons-collections</groupId>
       <artifactId>commons-collections</artifactId>
@@ -583,9 +579,10 @@
       <artifactId>spring-test</artifactId>
       <scope>compile</scope>
     </dependency>
+
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
     </dependency>
 
     <dependency>
@@ -699,6 +696,7 @@
               <clusterName>test</clusterName>
               <transportPort>9300</transportPort>
               <httpPort>9200</httpPort>
+              <environmentVariables><ES_JAVA_OPTS>-Xmx2g</ES_JAVA_OPTS></environmentVariables>
             </configuration>
             <executions>
               <execution>

diff --git a/core/src/main/java/org/fao/geonet/constants/Geonet.java b/core/src/main/java/org/fao/geonet/constants/Geonet.java
@@ -630,7 +630,7 @@ public static final class Namespaces {
 
     public static class IndexFieldNames {
         public static final String HASXLINKS = "_hasxlinks";
-        public static final String XLINK = "_xlink";
+        public static final String XLINK = "xlink";
         public static final String ROOT = "_root";
         public static final String SCHEMA = "documentStandard";
         public static final String DATABASE_CREATE_DATE = "createDate";

diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java
@@ -3,7 +3,7 @@
 //=== DataManager
 //===
 //=============================================================================
-//===	Copyright (C) 2001-2007 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -143,11 +143,6 @@ public void init(ServiceContext context, Boolean force) throws Exception {
         }
     }
 
-    @Deprecated
-    public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext context) throws Exception {
-        metadataIndexer.rebuildIndexXLinkedMetadata(context);
-    }
-
     @Deprecated
     public synchronized void rebuildIndexForSelection(final ServiceContext context, String bucket, boolean clearXlink) throws Exception {
         metadataIndexer.rebuildIndexForSelection(context, bucket, clearXlink);

diff --git a/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java b/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java
@@ -23,13 +23,14 @@
 
 package org.fao.geonet.kernel;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.UserSession;
 import jeeves.server.context.ServiceContext;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.constants.Edit;
 import org.fao.geonet.constants.Geonet;
@@ -248,8 +249,9 @@ public void selectAll(String type, ServiceContext context, UserSession session)
                     EsSearchManager searchManager = context.getBean(EsSearchManager.class);
                     searchResponse = searchManager.query(request.get("query"), FIELDLIST_UUID, 0, maxhits);
                     List<String> uuidList = new ArrayList();
-                    for (SearchHit h : Arrays.asList(searchResponse.getHits().getHits())) {
-                        uuidList.add((String) h.getSourceAsMap().get(Geonet.IndexFieldNames.UUID));
+                    ObjectMapper objectMapper = new ObjectMapper();
+                    for (Hit h : (List<Hit>) searchResponse.hits().hits()) {
+                        uuidList.add((String) objectMapper.convertValue(h.source(), Map.class).get(Geonet.IndexFieldNames.UUID));
                     }
 
                     if (selection != null) {

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java
@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2011 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -24,11 +24,9 @@
 package org.fao.geonet.kernel.datamanager;
 
 import java.io.IOException;
-import java.util.Calendar;
 import java.util.List;
 
 import org.fao.geonet.domain.AbstractMetadata;
-import org.fao.geonet.kernel.search.ISearchManager;
 import org.fao.geonet.kernel.search.IndexingMode;
 import org.jdom.Element;
 import org.springframework.data.jpa.domain.Specification;
@@ -65,11 +63,6 @@ public interface IMetadataIndexer {
      */
     int batchDeleteMetadataAndUpdateIndex(Specification<? extends AbstractMetadata> specification) throws Exception;
 
-    /**
-     * Search for all records having XLinks (ie. indexed with _hasxlinks flag), clear the cache and reindex all records found.
-     */
-    void rebuildIndexXLinkedMetadata(ServiceContext context) throws Exception;
-
     /**
      * Reindex all records in current selection.
      */

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java
@@ -200,31 +200,6 @@ public int batchDeleteMetadataAndUpdateIndex(Specification<? extends AbstractMet
         return metadataToDelete.size();
     }
 
-    @Override
-    /**
-     * Search for all records having XLinks (ie. indexed with _hasxlinks flag),
-     * clear the cache and reindex all records found.
-     */
-    public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext context) throws Exception {
-
-        // get all metadata with XLinks
-        Set<Integer> toIndex = searchManager.getDocsWithXLinks();
-
-        if (Log.isDebugEnabled(Geonet.DATA_MANAGER))
-            Log.debug(Geonet.DATA_MANAGER, "Will index " + toIndex.size() + " records with XLinks");
-        if (toIndex.size() > 0) {
-            // clean XLink Cache so that cache and index remain in sync
-            Processor.clearCache();
-
-            ArrayList<String> stringIds = new ArrayList<String>();
-            for (Integer id : toIndex) {
-                stringIds.add(id.toString());
-            }
-            // execute indexing operation
-            batchIndexInThreadPool(context, stringIds);
-        }
-    }
-
     /**
      * Reindex all records in current selection.
      */
@@ -350,9 +325,9 @@ public void indexMetadata(final String metadataId,
                 List<Attribute> xlinks = Processor.getXLinks(md);
                 if (xlinks.size() > 0) {
                     fields.put(Geonet.IndexFieldNames.HASXLINKS, true);
-                    StringBuilder sb = new StringBuilder();
                     for (Attribute xlink : xlinks) {
                         fields.put(Geonet.IndexFieldNames.XLINK, xlink.getValue());
+                        fields.put(Geonet.IndexFieldNames.XLINK, xlink.getValue().replaceAll("local://srv/api/registries/entries/(.*)\\?.*", "$1"));
                     }
                     Processor.detachXLink(md, getServiceContext());
                 } else {

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
@@ -1342,8 +1342,8 @@ public boolean isValid(Integer id) {
     }
 
     boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata) throws Exception {
-        StringBuilder query = new StringBuilder(String.format("xlink:*%s*", metadata.getUuid()));
-        return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0;
+        StringBuilder query = new StringBuilder(String.format("xlink:\"%s\"", metadata.getUuid()));
+        return this.searchManager.query(query.toString(), null, 0, 0).hits().total().value() > 0;
     }
 
 }
diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataOperations.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataOperations.java
@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2011 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2024 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -167,7 +167,7 @@ public boolean setOperation(ServiceContext context, int mdId, int grpId, int opI
      */
     @Override
     public boolean forceSetOperation(ServiceContext context, int mdId, int grpId, int opId) throws Exception {
-        Optional<OperationAllowed> opAllowed = _getOperationAllowedToAdd(context, mdId, grpId, opId, false);
+        Optional<OperationAllowed> opAllowed = getOperationAllowedToAddInternal(context, mdId, grpId, opId, false);
 
         if (opAllowed.isPresent()) {
             Log.trace(Geonet.DATA_MANAGER, "Operation is allowed");
@@ -199,11 +199,11 @@ public boolean forceSetOperation(ServiceContext context, int mdId, int grpId, in
     @Override
     public Optional<OperationAllowed> getOperationAllowedToAdd(final ServiceContext context, final int mdId, final int grpId,
                                                                final int opId) {
-        return _getOperationAllowedToAdd(context, mdId, grpId, opId, true);
+        return getOperationAllowedToAddInternal(context, mdId, grpId, opId, true);
     }
 
-    private Optional<OperationAllowed> _getOperationAllowedToAdd(final ServiceContext context, final int mdId, final int grpId,
-                                                                 final int opId, boolean shouldCheckPermission) {
+    private Optional<OperationAllowed> getOperationAllowedToAddInternal(final ServiceContext context, final int mdId, final int grpId,
+                                                                        final int opId, boolean shouldCheckPermission) {
         Log.trace(Geonet.DATA_MANAGER, "_getOperationAllowedToAdd(" + mdId + ", "
             + grpId + ", " + opId + ", " + shouldCheckPermission + ")");
         final OperationAllowed operationAllowed = opAllowedRepo.findOneById_GroupIdAndId_MetadataIdAndId_OperationId(grpId, mdId, opId);
@@ -340,14 +340,12 @@ public void copyDefaultPrivForGroup(ServiceContext context, String id, String gr
 
         setOperation(context, id, groupId, ReservedOperation.view);
         setOperation(context, id, groupId, ReservedOperation.notify);
+        setOperation(context, id, groupId, ReservedOperation.download);
+        setOperation(context, id, groupId, ReservedOperation.dynamic);
         //
-        // Restrictive: new and inserted records should not be editable,
-        // their resources can't be downloaded and any interactive maps can't be
-        // displayed by users in the same group
+        // Restrictive: new and inserted records should not be editable by users in the same group,
         if (fullRightsForGroup) {
             setOperation(context, id, groupId, ReservedOperation.editing);
-            setOperation(context, id, groupId, ReservedOperation.download);
-            setOperation(context, id, groupId, ReservedOperation.dynamic);
         }
         // Ultimately this should be configurable elsewhere
     }