From 4d68cfbe806c6a66cb4878f5bdc61a6868cced97 Mon Sep 17 00:00:00 2001 From: Addison Schiller Date: Wed, 4 Oct 2017 11:40:08 -0400 Subject: [PATCH 1/3] Support to log metadata GET requests Removed superfluous return in remote_logging Metadata requests are currently not logged as they need an OSF addition to work correctly. --- waterbutler/core/remote_logging.py | 6 +----- waterbutler/server/api/v1/provider/__init__.py | 8 +++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/waterbutler/core/remote_logging.py b/waterbutler/core/remote_logging.py index 0b70b65a4..5bf0132e8 100644 --- a/waterbutler/core/remote_logging.py +++ b/waterbutler/core/remote_logging.py @@ -19,7 +19,7 @@ @utils.async_retry(retries=5, backoff=5) async def log_to_callback(action, source=None, destination=None, start_time=None, errors=[]): """PUT a logging payload back to the callback given by the auth provider.""" - if action in ('download_file', 'download_zip'): + if action in ('download_file', 'download_zip', 'metadata'): logger.debug('Not logging for {} action'.format(action)) return @@ -41,10 +41,6 @@ async def log_to_callback(action, source=None, destination=None, start_time=None log_payload['metadata'] = source.serialize() log_payload['provider'] = log_payload['metadata']['provider'] - if action in ('download_file', 'download_zip'): - logger.info('Not logging for {} action'.format(action)) - return - resp = await utils.send_signed_request('PUT', auth['callback_url'], log_payload) resp_data = await resp.read() diff --git a/waterbutler/server/api/v1/provider/__init__.py b/waterbutler/server/api/v1/provider/__init__.py index 56f8df292..8dbb4f7b2 100644 --- a/waterbutler/server/api/v1/provider/__init__.py +++ b/waterbutler/server/api/v1/provider/__init__.py @@ -145,12 +145,10 @@ def on_finish(self): if any((method in ('HEAD', 'OPTIONS'), status == 202, status > 302, status < 200)): return - if method == 'GET' and 'meta' in self.request.query_arguments: - return - # Done here just because method is defined action = { - 'GET': lambda: 'download_file' if self.path.is_file else 'download_zip', + 'GET': lambda: 'metadata' if 'meta' in self.request.query_arguments else ( + 'download_file' if self.path.is_file else 'download_zip'), 'PUT': lambda: ('create' if self.target_path.is_file else 'create_folder') if status == 201 else 'update', 'POST': lambda: 'move' if self.json['action'] == 'rename' else self.json['action'], 'DELETE': lambda: 'delete' @@ -175,7 +173,7 @@ def _send_hook(self, action): ) elif action in ('create', 'create_folder', 'update'): source = LogPayload(self.resource, self.provider, metadata=self.metadata) - elif action in ('delete', 'download_file', 'download_zip'): + elif action in ('delete', 'download_file', 'download_zip', 'metadata'): source = LogPayload(self.resource, self.provider, path=self.path) else: return From 307b9e856e17aa289801146e3f964de2ffc7ded8 Mon Sep 17 00:00:00 2001 From: John Tordoff Date: Mon, 18 Dec 2017 10:27:58 -0500 Subject: [PATCH 2/3] Changes logging behavior for different file actions and adds unit tests to track that behavior. --- tests/core/fixtures.py | 210 +++++++++++++++++++++++++ tests/core/test_remote_logging.py | 97 ++++++++++++ tests/providers/osfstorage/fixtures.py | 1 + tests/server/api/v1/fixtures.py | 122 ++++++++++++++ tests/server/api/v1/test_handler.py | 95 +++++++++++ waterbutler/core/log_payload.py | 4 + 6 files changed, 529 insertions(+) create mode 100644 tests/core/fixtures.py create mode 100644 tests/server/api/v1/fixtures.py create mode 100644 tests/server/api/v1/test_handler.py diff --git a/tests/core/fixtures.py b/tests/core/fixtures.py new file mode 100644 index 000000000..83332e36b --- /dev/null +++ b/tests/core/fixtures.py @@ -0,0 +1,210 @@ +import time +from unittest import mock + +import pytest + +from tests.utils import MockCoroutine +from waterbutler.core.path import WaterButlerPath +from waterbutler.core.log_payload import LogPayload +from tests.providers.osfstorage.fixtures import ( + file_metadata_object, + file_path, + file_metadata, + file_lineage, + provider, + auth +) + + +@pytest.fixture +def log_payload(file_metadata_object, file_path, provider): + return LogPayload('guid0', provider, file_metadata_object, file_path) + + +@pytest.fixture +def callback_log_payload_move(): + return { + 'auth': { + 'callback_url': 'fakecallback.com', + 'id': 'cat', + 'name': 'cat', + 'email': 'cat@cat.com' + }, + 'time': 70, + 'action': 'move', + 'source': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + }, + 'errors': [], + 'destination': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + } + } + + +@pytest.fixture +def callback_log_payload_copy(): + return { + 'auth': { + 'callback_url': 'fakecallback.com', + 'id': 'cat', + 'name': 'cat', + 'email': 'cat@cat.com' + }, + 'time': 70, + 'action': 'copy', + 'source': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + }, + 'errors': [], + 'destination': { + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'path': '/59a9b628b7d1c903ab5a8f52', + 'kind': 'file', + 'extra': { + 'checkout': None, + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf' + }, + 'version': 1 + }, + 'nid': 'guid0', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'contentType': None, 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'name': 'doc.rst', + 'size': 5596, + 'resource': 'guid0' + } + } + + +@pytest.fixture +def callback_log_payload_upload(): + return { + 'auth': { + 'id': 'cat', + 'email': 'cat@cat.com', + 'name': 'cat', + 'callback_url': 'fakecallback.com' + }, + 'errors': [], + 'time': 70, + 'action': 'upload', + 'provider': 'osfstorage', + 'metadata': { + 'kind': 'file', + 'name': 'doc.rst', + 'resource': 'guid0', + 'modified_utc': '2017-09-01T19:34:00.175741+00:00', + 'created_utc': '2017-09-01T19:34:00.175741+00:00', + 'provider': 'osfstorage', + 'modified': '2017-09-01T19:34:00.175741+00:00', + 'size': 5596, + 'path': '/59a9b628b7d1c903ab5a8f52', + 'etag': 'eccd2270585257f4b48d8493bed863c01cf0b6dc0bb590101407c9b5e10b8e08', + 'materialized': WaterButlerPath('/doc.rst', prepend=None), + 'extra': { + 'downloads': 0, + 'guid': None, + 'hashes': { + 'sha256': '043be9ff919762f0dc36fff0222cd90c753ce28b39feb52112be9360c476ef88', + 'md5': 'eb3f7cc15ba7b6effb2186284185c5cf'}, + 'checkout': None, + 'version': 1 + }, + 'contentType': None, + 'nid': 'guid0'} + } + + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + + +class MockResponse(): + status = 200 + read = MockCoroutine(return_value=b'{"status": "success"}') + + +class MockBadResponse(): + status = 500 + read = MockCoroutine(return_value=b'{"status": "failure"}') + + +@pytest.fixture +def mock_signed_request(): + return MockCoroutine(return_value=MockResponse()) \ No newline at end of file diff --git a/tests/core/test_remote_logging.py b/tests/core/test_remote_logging.py index 88ca8bd42..24f028c9f 100644 --- a/tests/core/test_remote_logging.py +++ b/tests/core/test_remote_logging.py @@ -1,6 +1,30 @@ +from unittest import mock + import pytest from waterbutler.core import remote_logging +from waterbutler.core.log_payload import LogPayload +from waterbutler.core.remote_logging import log_to_callback + +from tests.providers.osfstorage.fixtures import ( + file_metadata_object, + file_path, + file_metadata, + file_lineage, + provider, + auth, + credentials, + settings +) +from tests.core.fixtures import ( + log_payload, + MockBadResponse, + mock_time, + callback_log_payload_move, + callback_log_payload_copy, + callback_log_payload_upload, + mock_signed_request +) class TestScrubPayloadForKeen: @@ -74,3 +98,76 @@ def test_max_iteration(self): 'key-test': 'value2', 'key-test-1': 'value3' } + + +class TestLogPayLoad: + + def test_log_payload(self, log_payload, file_metadata_object, file_path, provider): + assert log_payload.resource == 'guid0' + assert log_payload.provider == provider + assert log_payload.metadata == file_metadata_object + assert log_payload.path == file_path + + with pytest.raises(Exception) as exc: + LogPayload('guid0', 'osfstorage') + assert exc.message == 'Log payload needs either a path or metadata.' + + +class TestLogToCallback: + + @pytest.mark.asyncio + async def test_log_to_callback_no_logging(self): + assert (await log_to_callback('download_file')) is None + assert (await log_to_callback('download_zip')) is None + assert (await log_to_callback('metadata')) is None + + @pytest.mark.asyncio + async def test_log_to_callback_move(self, + log_payload, + callback_log_payload_move, + mock_signed_request, + mock_time): + + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('move', log_payload, log_payload) + mock_signed_request.assert_called_with('PUT', + log_payload.auth['callback_url'], + callback_log_payload_move) + + @pytest.mark.asyncio + async def test_log_to_callback_copy(self, + log_payload, + callback_log_payload_copy, + mock_signed_request, + mock_time): + + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('copy', log_payload, log_payload) + mock_signed_request.assert_called_with('PUT', + log_payload.auth['callback_url'], + callback_log_payload_copy) + + @pytest.mark.asyncio + async def test_log_to_callback_upload(self, + log_payload, + callback_log_payload_upload, + mock_signed_request, + mock_time): + + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + await log_to_callback('upload', log_payload, log_payload) + mock_signed_request.assert_called_with('PUT', + log_payload.auth['callback_url'], + callback_log_payload_upload) + + @pytest.mark.skipif(reason="This test takes too much time because it has 5 retries before " + "throwing the desired exception, it should take around 50-60 " + "seconds") + @pytest.mark.asyncio + async def test_log_to_callback_throws_exception(self, log_payload, mock_signed_request): + + with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): + with pytest.raises(Exception) as exc: + await log_to_callback('upload', log_payload, log_payload) + assert exc.message == 'Callback for upload request failed with {},' \ + ' got {"status": "failure"}'.format(MockBadResponse()) diff --git a/tests/providers/osfstorage/fixtures.py b/tests/providers/osfstorage/fixtures.py index bc3d9243d..354857987 100644 --- a/tests/providers/osfstorage/fixtures.py +++ b/tests/providers/osfstorage/fixtures.py @@ -22,6 +22,7 @@ def auth(): 'id': 'cat', 'name': 'cat', 'email': 'cat@cat.com', + 'callback_url': 'fakecallback.com', } diff --git a/tests/server/api/v1/fixtures.py b/tests/server/api/v1/fixtures.py new file mode 100644 index 000000000..5a75df4c4 --- /dev/null +++ b/tests/server/api/v1/fixtures.py @@ -0,0 +1,122 @@ +import time +import asyncio +from unittest import mock + +import pytest +import tornado + +from waterbutler.server.app import make_app +from waterbutler.server.api.v1.provider import ProviderHandler +from waterbutler.core.path import WaterButlerPath +from waterbutler.core.log_payload import LogPayload + +from tests.providers.osfstorage.fixtures import ( + file_metadata_object, + file_metadata, + file_path, + file_lineage, + provider, + auth +) + +from tests.utils import ( + MockProvider, + MockFileMetadata +) + +@pytest.yield_fixture +def event_loop(): + """Create an instance of the default event loop for each test case.""" + policy = asyncio.get_event_loop_policy() + res = policy.new_event_loop() + asyncio.set_event_loop(res) + res._close = res.close + res.close = lambda: None + + yield res + + res._close() + +@pytest.fixture +def http_request(): + http_request = tornado.httputil.HTTPServerRequest( + uri='/v1/resources/test/providers/test/path/mock', + method='GET') + http_request.headers['User-Agent'] = 'test' + http_request.connection = tornado.http1connection.HTTP1ConnectionParameters() + http_request.connection.set_close_callback = mock.Mock() + http_request.request_time = mock.Mock(return_value=10) + + return http_request + +@pytest.fixture +def log_payload(): + return LogPayload('test', MockProvider(), path=WaterButlerPath('/test_path')) + +@pytest.fixture +def mock_time(monkeypatch): + mock_time = mock.Mock() + mock_time.return_value = 10 + monkeypatch.setattr(time, 'time', mock_time) + +@pytest.fixture +def handler(http_request): + handler = ProviderHandler(make_app(True), http_request) + handler.path = WaterButlerPath('/test_path') + + handler.provider = MockProvider() + handler.resource = 'test_source_resource' + handler.metadata = MockFileMetadata() + + handler.dest_provider = MockProvider() + handler.dest_resource = 'test_dest_resource' + handler.dest_meta = MockFileMetadata() + + return handler + + +@pytest.fixture +def source_payload(handler): + return LogPayload(handler.resource, + handler.provider, + path=handler.path) + + +@pytest.fixture +def destination_payload(handler): + return LogPayload(handler.dest_resource, + handler.provider, + metadata=handler.dest_meta) + + +@pytest.fixture +def payload_path(handler): + return LogPayload(handler.resource, + handler.provider, + path=handler.path) + + +@pytest.fixture +def payload_metadata(handler): + return LogPayload(handler.resource, + handler.provider, + metadata=handler.metadata) + + +@pytest.fixture +def serialzied_request(handler): + return { + 'request': { + 'url': 'http://127.0.0.1/v1/resources/test/providers/test/path/mock', + 'method': 'GET', + 'headers': {}, + 'time': 10 + }, + 'tech': { + 'ua': 'test', + 'ip': None + }, + 'referrer': { + 'url': None + } + } \ No newline at end of file diff --git a/tests/server/api/v1/test_handler.py b/tests/server/api/v1/test_handler.py new file mode 100644 index 000000000..97fbc5857 --- /dev/null +++ b/tests/server/api/v1/test_handler.py @@ -0,0 +1,95 @@ +from unittest import mock + +import pytest + +from tests.utils import ( + HandlerTestCase, + MockProvider, + MockFileMetadata, + MockCoroutine, +) +from tests.server.api.v1.fixtures import ( + mock_time, + handler, + http_request, + log_payload, + source_payload, + destination_payload, + payload_metadata, + payload_path, + serialzied_request +) + +class TestSendHook: + + @pytest.mark.parametrize('action', ['move', 'copy']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_cant_intra_move_copy(self, mocked_log_file_action, handler, action): + + assert handler._send_hook(action) is None + mocked_log_file_action.assert_not_called() + + @pytest.mark.parametrize('action', ['move', 'copy']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_can_intra_move_copy(self, + mocked_log_file_action, + handler, + source_payload, + destination_payload, + serialzied_request, + event_loop, + action): + + setattr(handler.provider, 'can_intra_' + action, mock.Mock(return_value=True)) + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with(action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=destination_payload, + request=serialzied_request) + + @pytest.mark.parametrize("action", ['create', 'create_folder', 'update']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_always_send_metadata(self, + mocked_log_file_action, + handler, + payload_metadata, + serialzied_request, + event_loop, + action): + + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with(action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=payload_metadata, + destination=None, + request=serialzied_request) + + @pytest.mark.parametrize("action", ['delete', 'download_file', 'download_zip', 'metadata']) + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_always_send_path(self, + mocked_log_file_action, + handler, + source_payload, + serialzied_request, + event_loop, + action): + + assert handler._send_hook(action) is None + mocked_log_file_action.assert_called_once_with(action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=None, + request=serialzied_request) + + @mock.patch('waterbutler.core.remote_logging.log_file_action') + def test_send_hook_invalid_action(self, mocked_log_file_action, handler, event_loop): + + assert handler._send_hook('invalid_action') is None + mocked_log_file_action.assert_not_called() diff --git a/waterbutler/core/log_payload.py b/waterbutler/core/log_payload.py index b95e0a267..a9946feef 100644 --- a/waterbutler/core/log_payload.py +++ b/waterbutler/core/log_payload.py @@ -57,6 +57,10 @@ def serialize(self): return payload + def __eq__(self, other: object) -> bool: + # This is allows for easy comparisons via unit tests. + return isinstance(other, LogPayload) and self.serialize() == other.serialize() + @property def auth(self): """The auth object for the entity. Contains the callback_url.""" From da471b101c35a46a5d688b9e2d8628ffb76e2c83 Mon Sep 17 00:00:00 2001 From: longze chen Date: Wed, 3 Jan 2018 14:38:40 -0500 Subject: [PATCH 3/3] Fix minor bugs and improve style. --- tests/core/fixtures.py | 32 ++++--- tests/core/test_remote_logging.py | 114 +++++++++++++------------ tests/server/api/v1/fixtures.py | 85 ++++++++----------- tests/server/api/v1/test_handler.py | 126 ++++++++++++++-------------- 4 files changed, 168 insertions(+), 189 deletions(-) diff --git a/tests/core/fixtures.py b/tests/core/fixtures.py index 83332e36b..acc276069 100644 --- a/tests/core/fixtures.py +++ b/tests/core/fixtures.py @@ -1,19 +1,15 @@ import time +from http import HTTPStatus from unittest import mock import pytest -from tests.utils import MockCoroutine from waterbutler.core.path import WaterButlerPath from waterbutler.core.log_payload import LogPayload -from tests.providers.osfstorage.fixtures import ( - file_metadata_object, - file_path, - file_metadata, - file_lineage, - provider, - auth -) + +from tests.utils import MockCoroutine +from tests.providers.osfstorage.fixtures import (auth, file_path, file_lineage, provider, + file_metadata_object, file_metadata) @pytest.fixture @@ -195,16 +191,16 @@ def mock_time(monkeypatch): monkeypatch.setattr(time, 'time', mock_time) -class MockResponse(): - status = 200 - read = MockCoroutine(return_value=b'{"status": "success"}') +@pytest.fixture +def mock_signed_request(): + return MockCoroutine(return_value=MockResponse()) -class MockBadResponse(): - status = 500 - read = MockCoroutine(return_value=b'{"status": "failure"}') +class MockResponse: + status = HTTPStatus.OK + read = MockCoroutine(return_value=b'{"status": "success"}') -@pytest.fixture -def mock_signed_request(): - return MockCoroutine(return_value=MockResponse()) \ No newline at end of file +class MockBadResponse: + status = HTTPStatus.INTERNAL_SERVER_ERROR + read = MockCoroutine(return_value=b'{"status": "failure"}') diff --git a/tests/core/test_remote_logging.py b/tests/core/test_remote_logging.py index 24f028c9f..4ad891a77 100644 --- a/tests/core/test_remote_logging.py +++ b/tests/core/test_remote_logging.py @@ -6,25 +6,12 @@ from waterbutler.core.log_payload import LogPayload from waterbutler.core.remote_logging import log_to_callback -from tests.providers.osfstorage.fixtures import ( - file_metadata_object, - file_path, - file_metadata, - file_lineage, - provider, - auth, - credentials, - settings -) -from tests.core.fixtures import ( - log_payload, - MockBadResponse, - mock_time, - callback_log_payload_move, - callback_log_payload_copy, - callback_log_payload_upload, - mock_signed_request -) +from tests.core.fixtures import (MockBadResponse, log_payload, mock_time, + mock_signed_request, callback_log_payload_upload, + callback_log_payload_move, callback_log_payload_copy) +from tests.providers.osfstorage.fixtures import (auth, credentials, provider, + settings, file_path, file_lineage, + file_metadata, file_metadata_object) class TestScrubPayloadForKeen: @@ -122,52 +109,63 @@ async def test_log_to_callback_no_logging(self): assert (await log_to_callback('metadata')) is None @pytest.mark.asyncio - async def test_log_to_callback_move(self, - log_payload, - callback_log_payload_move, - mock_signed_request, - mock_time): - + async def test_log_to_callback_move( + self, + log_payload, + callback_log_payload_move, + mock_signed_request, + mock_time + ): with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): - await log_to_callback('move', log_payload, log_payload) - mock_signed_request.assert_called_with('PUT', - log_payload.auth['callback_url'], - callback_log_payload_move) + await log_to_callback('move', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_move + ) @pytest.mark.asyncio - async def test_log_to_callback_copy(self, - log_payload, - callback_log_payload_copy, - mock_signed_request, - mock_time): - + async def test_log_to_callback_copy( + self, + log_payload, + callback_log_payload_copy, + mock_signed_request, + mock_time + ): with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): - await log_to_callback('copy', log_payload, log_payload) - mock_signed_request.assert_called_with('PUT', - log_payload.auth['callback_url'], - callback_log_payload_copy) + await log_to_callback('copy', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_copy + ) @pytest.mark.asyncio - async def test_log_to_callback_upload(self, - log_payload, - callback_log_payload_upload, - mock_signed_request, - mock_time): - + async def test_log_to_callback_upload( + self, + log_payload, + callback_log_payload_upload, + mock_signed_request, + mock_time + ): with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): - await log_to_callback('upload', log_payload, log_payload) - mock_signed_request.assert_called_with('PUT', - log_payload.auth['callback_url'], - callback_log_payload_upload) - - @pytest.mark.skipif(reason="This test takes too much time because it has 5 retries before " - "throwing the desired exception, it should take around 50-60 " - "seconds") + await log_to_callback('upload', source=log_payload, destination=log_payload) + mock_signed_request.assert_called_with( + 'PUT', + log_payload.auth['callback_url'], + callback_log_payload_upload + ) + + # TODO: should we fix or skip this? This test never passes for me locally but always takes a long time. + @pytest.mark.skipif( + reason="This test takes too much time because it has 5 retries before " + "throwing the desired exception, it should take around 50-60 seconds" + ) @pytest.mark.asyncio - async def test_log_to_callback_throws_exception(self, log_payload, mock_signed_request): - + async def test_log_to_callback_throws_exception(self, mock_signed_request): with mock.patch('waterbutler.core.utils.send_signed_request', mock_signed_request): with pytest.raises(Exception) as exc: - await log_to_callback('upload', log_payload, log_payload) - assert exc.message == 'Callback for upload request failed with {},' \ - ' got {"status": "failure"}'.format(MockBadResponse()) + await log_to_callback('upload') + expected_message = 'Callback for upload request failed with {},' \ + ' got {{"status": "failure"}}'.format(MockBadResponse()) + assert exc.message == expected_message diff --git a/tests/server/api/v1/fixtures.py b/tests/server/api/v1/fixtures.py index 5a75df4c4..c10b55dc4 100644 --- a/tests/server/api/v1/fixtures.py +++ b/tests/server/api/v1/fixtures.py @@ -3,26 +3,18 @@ from unittest import mock import pytest -import tornado +from tornado.httputil import HTTPServerRequest +from tornado.http1connection import HTTP1ConnectionParameters from waterbutler.server.app import make_app -from waterbutler.server.api.v1.provider import ProviderHandler from waterbutler.core.path import WaterButlerPath from waterbutler.core.log_payload import LogPayload +from waterbutler.server.api.v1.provider import ProviderHandler -from tests.providers.osfstorage.fixtures import ( - file_metadata_object, - file_metadata, - file_path, - file_lineage, - provider, - auth -) +from tests.utils import MockProvider, MockFileMetadata +from tests.providers.osfstorage.fixtures import (auth, provider, file_metadata_object, + file_metadata, file_path, file_lineage) -from tests.utils import ( - MockProvider, - MockFileMetadata -) @pytest.yield_fixture def event_loop(): @@ -37,74 +29,71 @@ def event_loop(): res._close() + @pytest.fixture def http_request(): - http_request = tornado.httputil.HTTPServerRequest( + mocked_http_request = HTTPServerRequest( uri='/v1/resources/test/providers/test/path/mock', - method='GET') - http_request.headers['User-Agent'] = 'test' - http_request.connection = tornado.http1connection.HTTP1ConnectionParameters() - http_request.connection.set_close_callback = mock.Mock() - http_request.request_time = mock.Mock(return_value=10) + method='GET' + ) + mocked_http_request.headers['User-Agent'] = 'test' + mocked_http_request.connection = HTTP1ConnectionParameters() + mocked_http_request.connection.set_close_callback = mock.Mock() + mocked_http_request.request_time = mock.Mock(return_value=10) + + return mocked_http_request - return http_request @pytest.fixture def log_payload(): return LogPayload('test', MockProvider(), path=WaterButlerPath('/test_path')) + @pytest.fixture def mock_time(monkeypatch): - mock_time = mock.Mock() - mock_time.return_value = 10 - monkeypatch.setattr(time, 'time', mock_time) + mocked_time = mock.Mock() + mocked_time.return_value = 10 + monkeypatch.setattr(time, 'time', mocked_time) + @pytest.fixture def handler(http_request): - handler = ProviderHandler(make_app(True), http_request) - handler.path = WaterButlerPath('/test_path') + mocked_handler = ProviderHandler(make_app(True), http_request) + mocked_handler.path = WaterButlerPath('/test_path') - handler.provider = MockProvider() - handler.resource = 'test_source_resource' - handler.metadata = MockFileMetadata() + mocked_handler.provider = MockProvider() + mocked_handler.resource = 'test_source_resource' + mocked_handler.metadata = MockFileMetadata() - handler.dest_provider = MockProvider() - handler.dest_resource = 'test_dest_resource' - handler.dest_meta = MockFileMetadata() + mocked_handler.dest_provider = MockProvider() + mocked_handler.dest_resource = 'test_dest_resource' + mocked_handler.dest_meta = MockFileMetadata() - return handler + return mocked_handler @pytest.fixture def source_payload(handler): - return LogPayload(handler.resource, - handler.provider, - path=handler.path) + return LogPayload(handler.resource, handler.provider, path=handler.path) @pytest.fixture def destination_payload(handler): - return LogPayload(handler.dest_resource, - handler.provider, - metadata=handler.dest_meta) + return LogPayload(handler.dest_resource, handler.provider, metadata=handler.dest_meta) @pytest.fixture def payload_path(handler): - return LogPayload(handler.resource, - handler.provider, - path=handler.path) + return LogPayload(handler.resource, handler.provider, path=handler.path) @pytest.fixture def payload_metadata(handler): - return LogPayload(handler.resource, - handler.provider, - metadata=handler.metadata) + return LogPayload(handler.resource, handler.provider, metadata=handler.metadata) @pytest.fixture -def serialzied_request(handler): +def serialized_request(): return { 'request': { 'url': 'http://127.0.0.1/v1/resources/test/providers/test/path/mock', @@ -118,5 +107,5 @@ def serialzied_request(handler): }, 'referrer': { 'url': None - } - } \ No newline at end of file + }, + } diff --git a/tests/server/api/v1/test_handler.py b/tests/server/api/v1/test_handler.py index 97fbc5857..fdead56ce 100644 --- a/tests/server/api/v1/test_handler.py +++ b/tests/server/api/v1/test_handler.py @@ -2,94 +2,90 @@ import pytest -from tests.utils import ( - HandlerTestCase, - MockProvider, - MockFileMetadata, - MockCoroutine, -) -from tests.server.api.v1.fixtures import ( - mock_time, - handler, - http_request, - log_payload, - source_payload, - destination_payload, - payload_metadata, - payload_path, - serialzied_request -) +from tests.utils import HandlerTestCase, MockProvider, MockFileMetadata, MockCoroutine +from tests.server.api.v1.fixtures import (handler, mock_time, http_request, + source_payload, destination_payload, + log_payload, payload_metadata, + payload_path, serialized_request) + class TestSendHook: @pytest.mark.parametrize('action', ['move', 'copy']) @mock.patch('waterbutler.core.remote_logging.log_file_action') def test_send_hook_cant_intra_move_copy(self, mocked_log_file_action, handler, action): - assert handler._send_hook(action) is None mocked_log_file_action.assert_not_called() @pytest.mark.parametrize('action', ['move', 'copy']) @mock.patch('waterbutler.core.remote_logging.log_file_action') - def test_send_hook_can_intra_move_copy(self, - mocked_log_file_action, - handler, - source_payload, - destination_payload, - serialzied_request, - event_loop, - action): - + def test_send_hook_can_intra_move_copy( + self, + mocked_log_file_action, + handler, + action, + source_payload, + destination_payload, + serialized_request, + event_loop + ): setattr(handler.provider, 'can_intra_' + action, mock.Mock(return_value=True)) assert handler._send_hook(action) is None - mocked_log_file_action.assert_called_once_with(action, - api_version='v1', - bytes_downloaded=0, - bytes_uploaded=0, - source=source_payload, - destination=destination_payload, - request=serialzied_request) + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=destination_payload, + request=serialized_request + ) @pytest.mark.parametrize("action", ['create', 'create_folder', 'update']) @mock.patch('waterbutler.core.remote_logging.log_file_action') - def test_send_hook_always_send_metadata(self, - mocked_log_file_action, - handler, - payload_metadata, - serialzied_request, - event_loop, - action): - + def test_send_hook_always_send_metadata( + self, + mocked_log_file_action, + handler, + action, + payload_metadata, + serialized_request, + event_loop + ): assert handler._send_hook(action) is None - mocked_log_file_action.assert_called_once_with(action, - api_version='v1', - bytes_downloaded=0, - bytes_uploaded=0, - source=payload_metadata, - destination=None, - request=serialzied_request) + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=payload_metadata, + destination=None, + request=serialized_request + ) @pytest.mark.parametrize("action", ['delete', 'download_file', 'download_zip', 'metadata']) @mock.patch('waterbutler.core.remote_logging.log_file_action') - def test_send_hook_always_send_path(self, - mocked_log_file_action, - handler, - source_payload, - serialzied_request, - event_loop, - action): - + def test_send_hook_always_send_path( + self, + mocked_log_file_action, + handler, + action, + source_payload, + serialized_request, + event_loop + ): assert handler._send_hook(action) is None - mocked_log_file_action.assert_called_once_with(action, - api_version='v1', - bytes_downloaded=0, - bytes_uploaded=0, - source=source_payload, - destination=None, - request=serialzied_request) + mocked_log_file_action.assert_called_once_with( + action, + api_version='v1', + bytes_downloaded=0, + bytes_uploaded=0, + source=source_payload, + destination=None, + request=serialized_request + ) @mock.patch('waterbutler.core.remote_logging.log_file_action') def test_send_hook_invalid_action(self, mocked_log_file_action, handler, event_loop): - assert handler._send_hook('invalid_action') is None mocked_log_file_action.assert_not_called()