[SDESK-7497] Small adjustments for superdesk-planning behave tests (#2853)

* Small adjustments for superdesk-planning behave tests

SDESK-7497

* Allow empty async resources

SDESK-7497

* Add missing context block

* Rollback workaround as it won't be needed

 Some newer implementation of the publishing engine will most likely solve the problem

* Fix mypy error and remove comment

Author: eos87

apps/publish/content/common.py

@@ -242,9 +242,11 @@ def update(self, id, updates, original):
                 # race condition.
                 published_service = get_resource_service(PUBLISHED)
                 assert published_service is not None
+
                 published_service.patch(id, {QUEUE_STATE: PUBLISH_STATE.PUSHED})
                 from apps.publish.enqueue import push_publish
 
+                # TODO-ASYNC: update this service so task below completes eagearly and tests pass
                 push_publish.apply_async(str(id))
 
             push_notification(

superdesk/core/resources/resource_rest_endpoints.py

@@ -9,6 +9,9 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import math
+import logging
+import traceback
+
 from typing import Optional, cast, Any, Literal
 from copy import deepcopy
 
@@ -44,6 +47,9 @@
 from .utils import combine_projection_args
 
 
+logger = logging.getLogger("superdesk")
+
+
 @dataclass
 class RestParentLink:
     #: Name of the resource this parent link belongs to
@@ -368,7 +374,7 @@ async def create_item(self, request: Request) -> Response:
         issues = []
         return_code = 201
         for value in deepcopy(payload):
-            # Validate the provided item,
+            # Validate the provided item
             try:
                 for parent_link in self.endpoint_config.parent_links or []:
                     parent_item = parent_items.get(parent_link.resource_name)
@@ -385,6 +391,8 @@ async def create_item(self, request: Request) -> Response:
                         ISSUES: get_field_errors_from_pydantic_validation_error(validation_error),
                     }
                 )
+                self._log_traceback(validation_error, "Validation error while creating item")
+
             except SuperdeskError as superdesk_error:
                 return_code = superdesk_error.status_code
                 # Let the Superdesk exception populate the issue dictionary
@@ -402,6 +410,7 @@ async def create_item(self, request: Request) -> Response:
                         ISSUES: {"exception": str(exception)},
                     }
                 )
+                self._log_traceback(exception, "Unexpected exception while creating item")
 
         if self.endpoint_config.exclude_fields_in_response:
             # If projection is enabled, we fetch all newly created items with projection applied
@@ -413,7 +422,14 @@ async def create_item(self, request: Request) -> Response:
         results: list[dict]
         results = [
             {
-                **self._populate_item_hateoas(request, model_instance.to_dict()),
+                **self._populate_item_hateoas(
+                    request,
+                    # make sure to include unset and default values
+                    model_instance.to_dict(
+                        exclude_unset=False,
+                        exclude_defaults=False,
+                    ),
+                ),
                 STATUS: STATUS_OK,
             }
             for model_instance in model_instances
@@ -499,6 +515,8 @@ async def update_item(
         except ValidationError as validation_error:
             return_code = 400
             issues = get_field_errors_from_pydantic_validation_error(validation_error)
+            self._log_traceback(validation_error, "Validation error while updating item")
+
         except SuperdeskError as superdesk_error:
             return_code = superdesk_error.status_code
             # Let the Superdesk exception populate the issue dictionary
@@ -730,3 +748,8 @@ async def on_fetched_item(self, request: Request, doc: dict) -> None:
 
     async def on_fetched(self, request: Request, doc: RestGetResponse) -> None:
         pass
+
+    def _log_traceback(self, exception: Exception, message: str = "Exception occurred") -> None:
+        """Logs the exception message and the full traceback."""
+        logger.warn(f"{message}: {exception}")
+        logger.warn(traceback.format_exc())

superdesk/tests/__init__.py

@@ -403,7 +403,9 @@ async def setup(context=None, config=None, app_factory=get_app, reset=False, aut
 
     if context:
         context.app = app
+        app.test_client_class = TestClient
         context.client = app.test_client()
+
         if not hasattr(context, "BEHAVE") and not hasattr(context, "test_context"):
             context.test_context = app.test_request_context("/")
             await context.test_context.push()
@@ -434,6 +436,10 @@ def add_user_info_to_context(context: Any, token: str, user: User, auth_id=None)
     will be converted back to string (internal) by quart/werkzeug Request.
     """
     basic_token_header = token_to_basic_auth_header(token)
+
+    # remove any existing authorization header. Updates the list (in-place)
+    # to preserve any references to context.headers elsewhere
+    context.headers[:] = [h for h in context.headers if h[0] != "Authorization"]
     context.headers.append(basic_token_header)
 
     if getattr(context, "user", None):
@@ -607,6 +613,15 @@ def assertDictContains(self, source: dict, contains: dict):
 
 
 class TestClient(QuartClient):
+    async def open(self, *args, **kwargs) -> Response:
+        """
+        Appends the request path to the response object for later debugging
+        """
+
+        response = await super().open(*args, **kwargs)
+        response.request_path = kwargs.get("path", args[0] if args else "/")  # type: ignore[attr-defined]
+        return response
+
     def model_instance_to_json(self, model_instance: ResourceModel):
         return model_instance.to_dict(mode="json")
 

superdesk/tests/publish_steps.py

@@ -28,8 +28,9 @@
 
 
 @when("we enqueue published")
-def step_impl_when_auth(context):
-    enqueue_published.apply_async()
+@async_run_until_complete
+async def step_impl_when_auth(context):
+    await enqueue_published.apply_async()
 
 
 @then("we get formatted item")

superdesk/tests/steps.py

@@ -10,7 +10,9 @@
 
 
 import os
+import io
 import time
+from typing import Any
 import arrow
 import celery
 import shutil
@@ -20,8 +22,9 @@
 from unittest import mock
 from copy import deepcopy
 from base64 import b64encode
-from datetime import datetime, timedelta, date
 from os.path import basename
+from datetime import datetime, timedelta, date
+from quart.datastructures import FileStorage
 from re import findall
 from inspect import isawaitable
 from urllib.parse import urlparse
@@ -82,10 +85,11 @@ async def expect_status_in(response, codes):
 
     assert response.status_code in [
         int(code) for code in codes
-    ], "expected on of {expected}, got {code}, reason={reason}".format(
+    ], "expected one of {expected}, got {code}, reason={reason}, url='{requested_path}'".format(
         code=response.status_code,
         expected=codes,
         reason=(await response.get_data()).decode("utf-8"),
+        requested_path=response.request_path,
     )
 
 
@@ -491,12 +495,25 @@ def format_items(items):
     return ",\n".join(output)
 
 
+async def delete_entries_for(context, resource: str) -> None:
+    """
+    Attempts to remove all items from the resources MongoDB and/or Elastic.
+    First tries with async, otherwise it falls back to sync resources.
+    """
+
+    async with context.app.test_request_context(context.app.config["URL_PREFIX"]):
+        try:
+            async_app = context.app.async_app
+            await async_app.resources.get_resource_service(resource).delete_many({})
+        except KeyError:
+            get_resource_service(resource).delete_action()
+
+
 @given('empty "{resource}"')
 @async_run_until_complete
 async def step_impl_given_empty(context, resource):
     if not is_user_resource(resource):
-        async with context.app.test_request_context(context.app.config["URL_PREFIX"]):
-            get_resource_service(resource).delete_action()
+        await delete_entries_for(context, resource)
 
 
 @given('"{resource}"')
@@ -1230,15 +1247,28 @@ async def when_upload_patch_dictionary(context):
 async def upload_file(context, dest, filename, file_field, extra_data=None, method="post", user_headers=None):
     if user_headers is None:
         user_headers = []
+
     with open(get_fixture_path(context, filename), "rb") as f:
-        data = {file_field: f}
-        if extra_data:
-            data.update(extra_data)
+        file_content = f.read()
+        files = {
+            file_field: FileStorage(
+                io.BytesIO(file_content),
+                filename=os.path.basename(filename),
+                name=file_field,
+            )
+        }
+
         headers = [("Content-Type", "multipart/form-data")]
         headers.extend(user_headers)
         headers = unique_headers(headers, context.headers)
         url = get_prefixed_url(context.app, dest)
-        context.response = await getattr(context.client, method)(url, data=data, headers=headers)
+
+        context.response = await getattr(context.client, method)(
+            url,
+            files=files,
+            form=extra_data,
+            headers=headers,
+        )
         await assert_ok(context.response)
         await store_placeholder(context, url)
 
@@ -1298,7 +1328,6 @@ async def _step_impl_then_get_error(context, code):
 async def step_impl_then_get_error(context, code):
     await expect_status(context.response, int(code))
     if context.text:
-        print("got", (await context.response.get_data()).decode("utf-8"))
         await test_json(context)
 
 
@@ -1469,7 +1498,6 @@ async def step_impl_then_get_existing_resource(context):
 
 async def step_impl_then_get_existing(context):
     await assert_200(context.response)
-    print("got", get_response_readable(await context.response.get_data()))
     await test_json(context)