Merge remote-tracking branch 'origin/hotfix/2.6.8' into develop

Author: petrjasek

dev-requirements.txt

@@ -14,7 +14,7 @@ pytest
 pytest-env
 python3-saml>=1.9,<1.10
 typing_extensions>=3.7.4
-moto[sqs]
+moto[sqs]<5.0
 
 -e .
 -e git+https://github.com/superdesk/[email protected]#egg=superdesk-planning

superdesk/eve_backend.py

@@ -26,6 +26,7 @@
 from superdesk.errors import SuperdeskApiError
 from superdesk.notification import push_notification as _push_notification
 from superdesk.cache import cache
+from superdesk.utils import get_list_chunks
 
 
 SYSTEM_KEYS = set(
@@ -391,7 +392,8 @@ def delete_docs(self, endpoint_name, docs):
                 except Exception:
                     logger.exception("item can not be removed from elastic _id=%s" % (doc[config.ID_FIELD],))
         if len(removed_ids):
-            backend.remove(endpoint_name, {config.ID_FIELD: {"$in": removed_ids}})
+            for chunk in get_list_chunks(removed_ids):
+                backend.remove(endpoint_name, {config.ID_FIELD: {"$in": chunk}})
             logger.info("Removed %d documents from %s.", len(removed_ids), endpoint_name)
             for doc in docs:
                 self._push_resource_notification("deleted", endpoint_name, _id=str(doc["_id"]))

superdesk/text_checkers/ai/__init__.py

@@ -33,11 +33,12 @@ class AIResource(Resource):
             "type": "dict",
             "required": True,
             "schema": {
-                "guid": {"type": "string", "required": True},
+                "guid": {"type": "string", "required": False},
                 "abstract": {"type": "string", "required": False},
-                "language": {"type": "string", "required": True},
-                "headline": {"type": "string", "nullable": True},
+                "language": {"type": "string", "required": False},
                 "body_html": {"type": "string", "required": True},
+                "headline": {"type": "string", "required": False},
+                "slugline": {"type": "string", "required": False},
             },
         },
         "tags": {

superdesk/utils.py

@@ -363,3 +363,7 @@ def abort(status: int, message: str) -> None:
     for key, val in get_cors_headers():
         response.headers[key] = val
     flask.abort(response)
+
+
+def get_list_chunks(items, chunk_size=100):
+    return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]

tests/datalayer_tests.py

@@ -10,6 +10,7 @@
 
 
 import superdesk
+
 from bson import ObjectId
 from superdesk.tests import TestCase
 from superdesk.datalayer import SuperdeskJSONEncoder
@@ -65,3 +66,12 @@ def test_get_all_batch(self):
             assert item["_id"] == "test-{:04d}".format(counter)
             counter += 1
         assert counter == SIZE
+
+    def test_delete_chunks(self):
+        items = []
+        for i in range(5000):  # must be larger than 1k
+            items.append({"_id": ObjectId()})
+        service = superdesk.get_resource_service("audit")
+        service.create(items)
+        service.delete({})
+        assert 0 == service.find({}).count()

tests/utils_test.py

@@ -34,3 +34,14 @@ def test_allowed_container(self):
         self.assertNotIn("bar", container)
         allowed = [x for x in container]
         self.assertEqual(["foo"], allowed)
+
+    def test_list_chunks(self):
+        items = [1, 2, 3, 4, 5]
+        chunks = utils.get_list_chunks(items, 1)
+        assert [[1], [2], [3], [4], [5]] == chunks
+        chunks = utils.get_list_chunks(items, 2)
+        assert [[1, 2], [3, 4], [5]] == chunks
+        chunks = utils.get_list_chunks(items, 5)
+        assert [[1, 2, 3, 4, 5]] == chunks
+        chunks = utils.get_list_chunks(items, 10)
+        assert [[1, 2, 3, 4, 5]] == chunks