Merge remote-tracking branch 'upstream/master' into 52n-master

.clabot

@@ -77,6 +77,8 @@
     "ahmdthr",
     "fvicent",
     "RegisSinjari",
-    "Gpetrak"
+    "Gpetrak",
+    "cmotadev",
+    "kilichenko-pixida"
   ]
 }

geonode/assets/local.py

@@ -7,7 +7,7 @@
 from django.http import HttpResponse, StreamingHttpResponse
 from django.urls import reverse
 from django_downloadview import DownloadResponse
-from zipstream import ZipStream, walk
+from zipstream import ZipStream
 
 from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface
 from geonode.assets.models import LocalAsset
@@ -261,9 +261,7 @@ def create_response(
             match attachment:
                 case True:
                     logger.info(f"Zipping file '{localfile}' with name '{orig_base}'")
-                    zs = ZipStream(sized=True)
-                    for filepath in walk(LocalAssetHandler._get_managed_dir(asset)):
-                        zs.add_path(filepath, os.path.basename(filepath))
+                    zs = ZipStream(sized=True).from_path(LocalAssetHandler._get_managed_dir(asset), arcname="/")
                     # closing zip for all contents to be written
                     return StreamingHttpResponse(
                         zs,

geonode/assets/management/commands/migrate_file_to_assets.py

@@ -98,9 +98,19 @@ def handle(self, **options):
 
                 logger.info("Moving file to the asset folder")
 
-                dest = shutil.move(source, handler._create_asset_dir())
-
-                logger.info("Fixing perms")
+                if len(asset.location) == 1:
+                    # In older installations, all documents are stored in a single folder.
+                    # Instead of moving the entire folder, we can simply move the individual document.
+                    # This approach prevents the risk of breaking the other documents
+                    # that are stored in the same folder
+                    # oldpath = {MEDIA_ROOT}/documents/document/file.extension
+                    dest = shutil.move(asset.location[0], handler._create_asset_dir())
+                else:
+                    dest = shutil.move(source, handler._create_asset_dir())
+
+                logger.info(f"New destination path: {dest}")
+
+                logger.info("Fixing file/folder perms if required")
                 if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
                     os.chmod(os.path.dirname(dest), settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
 
@@ -113,7 +123,10 @@ def handle(self, **options):
 
             logger.info("Updating location field with new folder value")
 
-            asset.location = [x.replace(source, dest) for x in asset.location]
+            if len(asset.location) == 1:
+                asset.location = [dest]
+            else:
+                asset.location = [x.replace(source, dest) for x in asset.location]
             asset.save()
 
             logger.info("Checking if geoserver should be updated")

geonode/base/admin.py

@@ -247,7 +247,7 @@ def import_rdf(self, request):
         if request.method == "POST":
             try:
                 rdf_file = request.FILES["rdf_file"]
-                name = slugify(rdf_file.name)
+                name = slugify(rdf_file.name).removesuffix("-rdf")
                 call_command("load_thesaurus", file=rdf_file, name=name)
                 self.message_user(request, "Your RDF file has been imported", messages.SUCCESS)
                 return redirect("..")

geonode/base/api/tests.py

@@ -42,6 +42,7 @@
 from django.contrib.auth import get_user_model
 
 from owslib.etree import etree
+from avatar.templatetags.avatar_tags import avatar_url
 
 from rest_framework.test import APITestCase
 from rest_framework.renderers import JSONRenderer
@@ -1030,13 +1031,13 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -1111,7 +1112,7 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
@@ -1121,13 +1122,13 @@ def test_perms_resources(self):
                         "username": norman.username,
                         "first_name": norman.first_name,
                         "last_name": norman.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "edit",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -1160,13 +1161,13 @@ def test_perms_resources(self):
                     "username": bobby.username,
                     "first_name": bobby.first_name,
                     "last_name": bobby.last_name,
-                    "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                    "avatar": build_absolute_uri(avatar_url(bobby)),
                     "permissions": "owner",
                     "is_staff": False,
                     "is_superuser": False,
                 },
                 {
-                    "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                    "avatar": build_absolute_uri(avatar_url(bobby)),
                     "first_name": "admin",
                     "id": 1,
                     "last_name": "",
@@ -1223,13 +1224,13 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -2156,7 +2157,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_superuser": False,
                         "is_staff": False,
@@ -2166,7 +2167,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_superuser": True,
                         "is_staff": True,
@@ -2197,7 +2198,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_staff": False,
                         "is_superuser": False,
@@ -2207,7 +2208,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": True,
                         "is_superuser": True,
@@ -2236,7 +2237,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_staff": False,
                         "is_superuser": False,
@@ -2246,7 +2247,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": True,
                         "is_superuser": True,

geonode/base/api/views.py

@@ -1457,7 +1457,7 @@ def base_linked_resources(instance, user, params):
         return Response(data={"message": e.args[0], "success": False}, status=500, exception=True)
 
 
-def base_linked_resources_payload(instance, user, params={}):
+def base_linked_resources_instances(instance, user, params={}):
     resource_type = params.get("resource_type", None)
     link_type = params.get("link_type", None)
     type_list = resource_type.split(",") if resource_type else []
@@ -1498,7 +1498,7 @@ def base_linked_resources_payload(instance, user, params={}):
         linked_to_visib_ids = linked_to_visib.values_list("id", flat=True)
         linked_to = [lres for lres in linked_to_over_loopable if lres.target.id in linked_to_visib_ids]
 
-        ret["linked_to"] = LinkedResourceSerializer(linked_to, embed=True, many=True).data
+        ret["linked_to"] = linked_to
 
     if not link_type or link_type == "linked_by":
         linked_by_over = instance.get_linked_resources(as_target=True)
@@ -1517,11 +1517,25 @@ def base_linked_resources_payload(instance, user, params={}):
         linked_by_visib_ids = linked_by_visib.values_list("id", flat=True)
         linked_by = [lres for lres in linked_by_over_loopable if lres.source.id in linked_by_visib_ids]
 
-        ret["linked_by"] = LinkedResourceSerializer(
-            instance=linked_by, serialize_source=True, embed=True, many=True
-        ).data
+        ret["linked_by"] = linked_by
 
     if not ret["WARNINGS"]:
         ret.pop("WARNINGS")
 
     return ret
+
+
+def base_linked_resources_payload(instance, user, params={}):
+    lres = base_linked_resources_instances(instance, user, params)
+    ret = {}
+    if "linked_to" in lres:
+        ret["linked_to"] = LinkedResourceSerializer(lres["linked_to"], embed=True, many=True).data
+    if "linked_by" in lres:
+        ret["linked_by"] = LinkedResourceSerializer(
+            instance=lres["linked_by"], serialize_source=True, embed=True, many=True
+        ).data
+
+    if lres.get("WARNINGS", None):
+        ret["WARNINGS"] = lres["WARNINGS"]
+
+    return ret

geonode/base/forms.py

@@ -342,6 +342,7 @@ def _get_thesauro_keyword_label(item, lang):
 
     @staticmethod
     def _get_thesauro_title_label(item, lang):
+        lang = remove_country_from_languagecode(lang)
         tname = ThesaurusLabel.objects.values_list("label", flat=True).filter(thesaurus=item).filter(lang=lang)
         if not tname:
             return Thesaurus.objects.get(id=item.id).title

geonode/base/management/commands/load_thesaurus.py

@@ -114,7 +114,8 @@ def load_thesaurus(self, input_file, name, store):
                     thesaurus_label.save()
 
         for concept in g.subjects(RDF.type, SKOS.Concept):
-            pref = preferredLabel(g, concept, default_lang)[0][1]
+            prefs = preferredLabel(g, concept, default_lang)
+            pref = prefs[0][1] if prefs else "-"
             about = str(concept)
             alt_label = g.value(concept, SKOS.altLabel, object=None, default=None)
             if alt_label is not None:

geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py

@@ -1,4 +1,5 @@
 # Generated by Django 4.2.9 on 2024-03-12 11:55
+import itertools
 import logging
 import os
 
@@ -10,7 +11,7 @@
 
 from geonode.base.models import Link
 from geonode.assets.models import LocalAsset
-from geonode.utils import build_absolute_uri
+from geonode.utils import build_absolute_uri, get_supported_datasets_file_types
 
 logger = logging.getLogger(__name__)
 
@@ -24,9 +25,9 @@ def get_ext(filename):
             logger.warning(f"Could not find extension for Resource '{res_hm.title}, file '{filename}': {e}")
             return None
 
-    ResourceBase_hm = apps.get_model('base', 'ResourceBase')
-    Dataset_hm = apps.get_model('layers', 'Dataset')
-    Document_hm = apps.get_model('documents', 'Document')
+    ResourceBase_hm = apps.get_model("base", "ResourceBase")
+    Dataset_hm = apps.get_model("layers", "Dataset")
+    Document_hm = apps.get_model("documents", "Document")
 
     if hasattr(ResourceBase_hm, "files"):
         # looping on available resources with files to generate the LocalAssets
@@ -37,12 +38,7 @@ def get_ext(filename):
 
             files = res_hm.files
             # creating the local asset object
-            asset = LocalAsset(
-                title="Files",
-                description="Original uploaded files",
-                owner=owner,
-                location=files
-            )
+            asset = LocalAsset(title="Files", description="Original uploaded files", owner=owner, location=files)
             asset.save()
 
             ### creating the association between asset and Link
@@ -60,10 +56,14 @@ def get_ext(filename):
                 ext = get_ext(files[0])
             else:
                 ext = None
+                supported_file_types = get_supported_datasets_file_types()
                 for file in files:
-                    for filetype in settings.SUPPORTED_DATASET_FILE_TYPES:
+                    for filetype in supported_file_types:
                         file_ext = get_ext(file)
-                        if file_ext in filetype["ext"]:
+                        _ext = list(
+                            itertools.chain.from_iterable(y for y in [x["required_ext"] for x in filetype["formats"]])
+                        )
+                        if file_ext in _ext:
                             ext = filetype["id"]
                             break
                     if ext:
@@ -75,14 +75,13 @@ def get_ext(filename):
                 link_type="uploaded",
                 name="Original upload",
                 extension=ext or "unknown",
-                url=url
+                url=url,
             )
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-
         ("base", "0091_create_link_asset_alter_link_type"),
     ]
 

geonode/base/urls.py

@@ -26,7 +26,6 @@
     OwnerRightsRequestView,
     ResourceBaseAutocomplete,
     HierarchicalKeywordAutocomplete,
-    ThesaurusKeywordLabelAutocomplete,
     LinkedResourcesAutocomplete,
 )
 
@@ -57,11 +56,6 @@
         ThesaurusAvailable.as_view(),
         name="thesaurus_available",
     ),
-    re_path(
-        r"^thesaurus_autocomplete/$",
-        ThesaurusKeywordLabelAutocomplete.as_view(),
-        name="thesaurus_autocomplete",
-    ),
     re_path(
         r"^datasets_autocomplete/$",
         DatasetsAutocomplete.as_view(),