Merge pull request #331 from MerginMaps/gh#2613-add-push-finish-signal

Introduce push_finished signal

server/mergin/sync/public_api_controller.py

@@ -86,7 +86,8 @@
 from .errors import StorageLimitHit
 from ..utils import format_time_delta
 
-push_triggered = signal("push_triggered")
+push_finished = signal("push_finished")
+# TODO: Move to database events to handle all commits to project versions
 project_version_created = signal("project_version_created")
 
 
@@ -732,7 +733,6 @@ def project_push(namespace, project_name):
     if not ws:
         abort(404)
 
-    push_triggered.send(project)
     # fixme use get_latest
     pv = ProjectVersion.query.filter_by(
         project_id=project.id, name=project.latest_version
@@ -874,6 +874,7 @@ def project_push(namespace, project_name):
                 f"Transaction id: {upload.id}. No upload."
             )
             project_version_created.send(pv)
+            push_finished.send(pv)
             return jsonify(ProjectSchema().dump(project)), 200
         except IntegrityError as err:
             db.session.rollback()
@@ -1084,6 +1085,7 @@ def push_finish(transaction_id):
             f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}."
         )
         project_version_created.send(pv)
+        push_finished.send(pv)
     except (psycopg2.Error, FileNotFoundError, DataSyncError, IntegrityError) as err:
         db.session.rollback()
         logging.exception(

server/mergin/tests/test_private_project_api.py

@@ -6,15 +6,19 @@
 import json
 import os
 
-import pytest
 from flask import url_for
 
 from ..app import db
 from ..sync.models import AccessRequest, Project, ProjectRole, RequestStatus
 from ..auth.models import User
 from ..config import Configuration
 from . import json_headers
-from .utils import add_user, login, create_project, create_workspace
+from .utils import (
+    add_user,
+    login,
+    create_project,
+    create_workspace,
+)
 
 
 def test_project_unsubscribe(client, diff_project):

server/mergin/tests/test_project_controller.py

@@ -18,6 +18,7 @@
 import re
 
 from flask_login import current_user
+from unittest.mock import patch
 from pygeodiff import GeoDiff
 from flask import url_for, current_app
 import tempfile
@@ -58,6 +59,7 @@
     login,
     file_info,
     login_as_admin,
+    upload_file_to_project,
 )
 from ..config import Configuration
 from ..sync.config import Configuration as SyncConfiguration
@@ -1791,6 +1793,8 @@ def test_clone_project(client, data, username, expected):
         # cleanup
         shutil.rmtree(project.storage.project_dir)
 
+    Configuration.GLOBAL_STORAGE = 104857600
+
 
 def test_optimize_storage(app, client, diff_project):
     """Test optimize storage for geopackages which could be restored from diffs
@@ -2461,3 +2465,14 @@ def test_delete_diff_file(client):
         change=PushChangeType.DELETE.value,
     ).first()
     assert fh.path == "base.gpkg" and fh.diff is None
+
+
+def test_signals(client):
+    workspace = create_workspace()
+    user = User.query.filter(User.username == "mergin").first()
+    project = create_project("test-project", workspace, user)
+    with patch(
+        "mergin.sync.public_api_controller.push_finished.send"
+    ) as push_finished_mock:
+        upload_file_to_project(project, "test.txt", client)
+        push_finished_mock.assert_called_once()

server/mergin/tests/utils.py

@@ -4,6 +4,7 @@
 
 import json
 import shutil
+from typing import Tuple
 import pysqlite3
 import uuid
 import math
@@ -213,10 +214,7 @@ def file_info(project_dir, path, chunk_size=1024):
     }
 
 
-def upload_file_to_project(project, filename, client):
-    """Add test file to project - start, upload and finish push process"""
-    file = os.path.join(test_project_dir, filename)
-    assert os.path.exists(file)
+def mock_changes_data(project, filename) -> dict:
     changes = {
         "added": [file_info(test_project_dir, filename)],
         "updated": [],
@@ -226,13 +224,33 @@ def upload_file_to_project(project, filename, client):
         "version": ProjectVersion.to_v_name(project.latest_version),
         "changes": changes,
     }
+    return data
+
+
+def push_file_start(
+    project: Project, filename: str, client, mocked_changes_data=None
+) -> dict:
+    """
+    Initiate the process of pushing a file to a project by calling /push endpoint.
+    """
+    file = os.path.join(test_project_dir, filename)
+    assert os.path.exists(file)
+    data = mocked_changes_data or mock_changes_data(project, filename)
     resp = client.post(
         f"/v1/project/push/{project.workspace.name}/{project.name}",
         data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"),
         headers=json_headers,
     )
+    return resp
+
+
+def upload_file_to_project(project: Project, filename: str, client) -> dict:
+    """Add test file to project - start, upload and finish push process"""
+    file = os.path.join(test_project_dir, filename)
+    data = mock_changes_data(project, filename)
+    changes = data.get("changes")
+    resp = push_file_start(project, filename, client, data)
     upload_id = resp.json["transaction"]
-    changes = data["changes"]
     file_meta = changes["added"][0]
     for chunk_id in file_meta["chunks"]:
         url = f"/v1/project/push/chunk/{upload_id}/{chunk_id}"
@@ -241,7 +259,10 @@ def upload_file_to_project(project, filename, client):
         client.post(
             url, data=f_data, headers={"Content-Type": "application/octet-stream"}
         )
-    assert client.post(f"/v1/project/push/finish/{upload_id}").status_code == 200
+    push_finish = client.post(f"/v1/project/push/finish/{upload_id}")
+    assert resp.status_code == 200
+    assert push_finish.status_code == 200
+    return push_finish
 
 
 def gpkgs_are_equal(file1, file2):