From 2b8c382f900ede3923103fc7d567668fca6808ce Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 14 Nov 2024 15:30:24 +0100 Subject: [PATCH 1/8] Introduce push_finished signal --- server/mergin/sync/public_api_controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 2cf62ee0..ef15d07c 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -86,7 +86,8 @@ from .errors import StorageLimitHit from ..utils import format_time_delta -push_triggered = signal("push_triggered") +push_finished = signal("push_triggered") +# TODO: Move to database events to handle all commits to project versions project_version_created = signal("project_version_created") @@ -732,7 +733,6 @@ def project_push(namespace, project_name): if not ws: abort(404) - push_triggered.send(project) # fixme use get_latest pv = ProjectVersion.query.filter_by( project_id=project.id, name=project.latest_version @@ -874,6 +874,7 @@ def project_push(namespace, project_name): f"Transaction id: {upload.id}. No upload." ) project_version_created.send(pv) + push_finished.send(pv) return jsonify(ProjectSchema().dump(project)), 200 except IntegrityError as err: db.session.rollback() @@ -1084,6 +1085,7 @@ def push_finish(transaction_id): f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}." ) project_version_created.send(pv) + push_finished.send(pv) except (psycopg2.Error, FileNotFoundError, DataSyncError, IntegrityError) as err: db.session.rollback() logging.exception( From 5a8bf131c17e8a4effdcc524f1ef9409f2a7a00e Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 14 Nov 2024 16:28:20 +0100 Subject: [PATCH 2/8] Introduce push_file utils method for tests to ge response objects --- server/mergin/tests/utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index 8ba33953..ab52c574 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -4,6 +4,7 @@ import json import shutil +from typing import Tuple import pysqlite3 import uuid import math @@ -213,8 +214,7 @@ def file_info(project_dir, path, chunk_size=1024): } -def upload_file_to_project(project, filename, client): - """Add test file to project - start, upload and finish push process""" +def push_file(project, filename, client) -> Tuple: file = os.path.join(test_project_dir, filename) assert os.path.exists(file) changes = { @@ -231,6 +231,8 @@ def upload_file_to_project(project, filename, client): data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"), headers=json_headers, ) + if resp.status_code != 200: + return (resp, None) upload_id = resp.json["transaction"] changes = data["changes"] file_meta = changes["added"][0] @@ -241,7 +243,14 @@ def upload_file_to_project(project, filename, client): client.post( url, data=f_data, headers={"Content-Type": "application/octet-stream"} ) - assert client.post(f"/v1/project/push/finish/{upload_id}").status_code == 200 + return (resp, client.post(f"/v1/project/push/finish/{upload_id}")) + + +def upload_file_to_project(project, filename, client): + """Add test file to project - start, upload and finish push process""" + resp_push_start, resp_push_finish = push_file(project, filename, client) + assert resp_push_finish.status_code == 200 + assert resp_push_start.status_code == 200 def gpkgs_are_equal(file1, file2): From 81e312bb365e4e075b4843892e39246130e7c9bd Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 15 Nov 2024 09:17:58 +0100 Subject: [PATCH 3/8] Rename signal --- server/mergin/sync/public_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index ef15d07c..b9e07552 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -86,7 +86,7 @@ from .errors import StorageLimitHit from ..utils import format_time_delta -push_finished = signal("push_triggered") +push_finished = signal("push_finished") # TODO: Move to database events to handle all commits to project versions project_version_created = signal("project_version_created") From 62adb6665b10c314a054c81808c8a6ad6b83463a Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 18 Nov 2024 13:11:44 +0100 Subject: [PATCH 4/8] Added push_file_start --- server/mergin/tests/utils.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index ab52c574..47ce6112 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -214,9 +214,7 @@ def file_info(project_dir, path, chunk_size=1024): } -def push_file(project, filename, client) -> Tuple: - file = os.path.join(test_project_dir, filename) - assert os.path.exists(file) +def mock_changes(project, filename) -> dict: changes = { "added": [file_info(test_project_dir, filename)], "updated": [], @@ -226,15 +224,28 @@ def push_file(project, filename, client) -> Tuple: "version": ProjectVersion.to_v_name(project.latest_version), "changes": changes, } + return data + + +def push_file_start(project, filename, client, mocked_changes=None) -> Tuple: + file = os.path.join(test_project_dir, filename) + assert os.path.exists(file) + data = mocked_changes or mock_changes(project, filename) resp = client.post( f"/v1/project/push/{project.workspace.name}/{project.name}", data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"), headers=json_headers, ) - if resp.status_code != 200: - return (resp, None) + return resp + + +def upload_file_to_project(project, filename, client): + """Add test file to project - start, upload and finish push process""" + file = os.path.join(test_project_dir, filename) + changes = mock_changes(project, filename)["changes"] + resp = push_file_start(project, filename, client, changes) + print(resp.json) upload_id = resp.json["transaction"] - changes = data["changes"] file_meta = changes["added"][0] for chunk_id in file_meta["chunks"]: url = f"/v1/project/push/chunk/{upload_id}/{chunk_id}" @@ -243,14 +254,9 @@ def push_file(project, filename, client) -> Tuple: client.post( url, data=f_data, headers={"Content-Type": "application/octet-stream"} ) - return (resp, client.post(f"/v1/project/push/finish/{upload_id}")) - - -def upload_file_to_project(project, filename, client): - """Add test file to project - start, upload and finish push process""" - resp_push_start, resp_push_finish = push_file(project, filename, client) - assert resp_push_finish.status_code == 200 - assert resp_push_start.status_code == 200 + push_finish = client.post(f"/v1/project/push/finish/{upload_id}") + assert resp.status_code == 200 + assert push_finish.status_code == 200 def gpkgs_are_equal(file1, file2): From 9f2f1d3f3f5ca6ca570eefa36de3e3fb7b38d7f2 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 18 Nov 2024 13:42:37 +0100 Subject: [PATCH 5/8] Introduce push_file_start methods for testing utils --- server/mergin/tests/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index 47ce6112..c171532e 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -214,7 +214,7 @@ def file_info(project_dir, path, chunk_size=1024): } -def mock_changes(project, filename) -> dict: +def mock_changes_data(project, filename) -> dict: changes = { "added": [file_info(test_project_dir, filename)], "updated": [], @@ -227,10 +227,10 @@ def mock_changes(project, filename) -> dict: return data -def push_file_start(project, filename, client, mocked_changes=None) -> Tuple: +def push_file_start(project, filename, client, mocked_changes_data=None) -> Tuple: file = os.path.join(test_project_dir, filename) assert os.path.exists(file) - data = mocked_changes or mock_changes(project, filename) + data = mocked_changes_data or mock_changes_data(project, filename) resp = client.post( f"/v1/project/push/{project.workspace.name}/{project.name}", data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"), @@ -242,8 +242,9 @@ def push_file_start(project, filename, client, mocked_changes=None) -> Tuple: def upload_file_to_project(project, filename, client): """Add test file to project - start, upload and finish push process""" file = os.path.join(test_project_dir, filename) - changes = mock_changes(project, filename)["changes"] - resp = push_file_start(project, filename, client, changes) + data = mock_changes_data(project, filename) + changes = data.get("changes") + resp = push_file_start(project, filename, client, data) print(resp.json) upload_id = resp.json["transaction"] file_meta = changes["added"][0] From 21a8c5904cab55d7ae20d01b0acf2cec98d3fad7 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 18 Nov 2024 14:55:56 +0100 Subject: [PATCH 6/8] test signal --- server/mergin/tests/test_private_project_api.py | 8 ++++++-- server/mergin/tests/test_project_controller.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/mergin/tests/test_private_project_api.py b/server/mergin/tests/test_private_project_api.py index d16219a2..d8c3145c 100644 --- a/server/mergin/tests/test_private_project_api.py +++ b/server/mergin/tests/test_private_project_api.py @@ -6,7 +6,6 @@ import json import os -import pytest from flask import url_for from ..app import db @@ -14,7 +13,12 @@ from ..auth.models import User from ..config import Configuration from . import json_headers -from .utils import add_user, login, create_project, create_workspace +from .utils import ( + add_user, + login, + create_project, + create_workspace, +) def test_project_unsubscribe(client, diff_project): diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 0a7b1c92..2c9bcb5d 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -18,6 +18,7 @@ import re from flask_login import current_user +from unittest.mock import patch from pygeodiff import GeoDiff from flask import url_for, current_app import tempfile @@ -58,6 +59,7 @@ login, file_info, login_as_admin, + upload_file_to_project, ) from ..config import Configuration from ..sync.config import Configuration as SyncConfiguration @@ -2461,3 +2463,14 @@ def test_delete_diff_file(client): change=PushChangeType.DELETE.value, ).first() assert fh.path == "base.gpkg" and fh.diff is None + + +def test_signals(client): + workspace = create_workspace() + user = User.query.filter(User.username == "mergin").first() + project = create_project("test-project", workspace, user) + with patch( + "mergin.sync.public_api_controller.push_finished.send" + ) as push_finished_mock: + upload_file_to_project(project, "test.txt", client) + push_finished_mock.assert_called_once() From 4a7f381cfda5112c378be7d15c1dd095cb48aa87 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 18 Nov 2024 15:31:50 +0100 Subject: [PATCH 7/8] Addressign comments --- server/mergin/tests/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index c171532e..debc096e 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -227,7 +227,12 @@ def mock_changes_data(project, filename) -> dict: return data -def push_file_start(project, filename, client, mocked_changes_data=None) -> Tuple: +def push_file_start( + project: Project, filename: str, client, mocked_changes_data=None +) -> dict: + """ + Initiate the process of pushing a file to a project by calling /push endpoint. + """ file = os.path.join(test_project_dir, filename) assert os.path.exists(file) data = mocked_changes_data or mock_changes_data(project, filename) @@ -239,13 +244,12 @@ def push_file_start(project, filename, client, mocked_changes_data=None) -> Tupl return resp -def upload_file_to_project(project, filename, client): +def upload_file_to_project(project: Project, filename: str, client) -> dict: """Add test file to project - start, upload and finish push process""" file = os.path.join(test_project_dir, filename) data = mock_changes_data(project, filename) changes = data.get("changes") resp = push_file_start(project, filename, client, data) - print(resp.json) upload_id = resp.json["transaction"] file_meta = changes["added"][0] for chunk_id in file_meta["chunks"]: @@ -258,6 +262,7 @@ def upload_file_to_project(project, filename, client): push_finish = client.post(f"/v1/project/push/finish/{upload_id}") assert resp.status_code == 200 assert push_finish.status_code == 200 + return push_finish def gpkgs_are_equal(file1, file2): From ecbdb3bf61a771413f0555055480742456acafb6 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 18 Nov 2024 16:48:52 +0100 Subject: [PATCH 8/8] Reset storage in clone project --- server/mergin/tests/test_project_controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 2c9bcb5d..c2db1742 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -1793,6 +1793,8 @@ def test_clone_project(client, data, username, expected): # cleanup shutil.rmtree(project.storage.project_dir) + Configuration.GLOBAL_STORAGE = 104857600 + def test_optimize_storage(app, client, diff_project): """Test optimize storage for geopackages which could be restored from diffs