-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
tests: add test for S3 orphan check and stabilize code
- Loading branch information
1 parent
24d95d0
commit 254b9b9
Showing
3 changed files
with
137 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,53 +1,97 @@ | ||
| import pathlib | ||
| from unittest import mock | ||
|
|
||
| import ckan | ||
| import ckan.common | ||
| import ckan.model | ||
| import ckan.tests.factories as factories | ||
| import dcor_shared | ||
| import ckan.logic | ||
| from ckan.tests.helpers import call_action | ||
| import ckanext.dcor_schemas.plugin | ||
|
|
||
| from dcor_control import inspect | ||
| import dcor_shared | ||
| from dcor_shared import s3, s3cc | ||
| from dcor_shared.testing import make_dataset, synchronous_enqueue_job | ||
|
|
||
| from dcor_shared.testing import make_dataset | ||
| import pytest | ||
|
|
||
|
|
||
| data_path = pathlib.Path(__file__).parent / "data" | ||
|
|
||
|
|
||
| def test_check_orphaned_files(create_with_upload, monkeypatch, ckan_config, | ||
| tmpdir): | ||
| def test_check_orphaned_files_temp(create_with_upload, monkeypatch, | ||
| ckan_config, tmpdir): | ||
| """Make sure .rtdc~ files are removed for existing resources""" | ||
| monkeypatch.setitem(ckan_config, 'ckan.storage_path', str(tmpdir)) | ||
| monkeypatch.setattr(ckan.lib.uploader, | ||
| 'get_storage_path', | ||
| lambda: str(tmpdir)) | ||
|
|
||
| user = factories.User() | ||
|
|
||
| user_obj = ckan.model.User.by_name(user["name"]) | ||
| monkeypatch.setattr(ckan.common, | ||
| 'current_user', | ||
| user_obj) | ||
|
|
||
| owner_org = factories.Organization(users=[{ | ||
| 'name': user['id'], | ||
| 'capacity': 'admin' | ||
| }]) | ||
| # Note: `call_action` bypasses authorization! | ||
| # create 1st dataset | ||
| create_context1 = {'ignore_auth': False, | ||
| 'user': user['name'], 'api_version': 3} | ||
|
|
||
| _, res = make_dataset( | ||
| create_context1, owner_org, | ||
| _, res_dict = make_dataset( | ||
| create_with_upload=create_with_upload, | ||
| resource_path=data_path / "calibration_beads_47.rtdc", | ||
| activate=True, | ||
| authors="Peter Pan") | ||
|
|
||
| path = dcor_shared.get_resource_path(res["id"]) | ||
| path = dcor_shared.get_resource_path(res_dict["id"]) | ||
| path_to_delete = path.with_name(path.stem + "_peter.rtdc~") | ||
| path_to_delete.touch() | ||
| assert path_to_delete.exists() | ||
| inspect.check_orphaned_files(assume_yes=True) | ||
| assert not path_to_delete.exists() | ||
|
|
||
|
|
||
| @pytest.mark.usefixtures('clean_db', 'with_request_context') | ||
| @mock.patch('ckan.plugins.toolkit.enqueue_job', | ||
| side_effect=synchronous_enqueue_job) | ||
| def test_check_orphaned_s3_artifacts(enqueue_job_mock, create_with_upload, | ||
| monkeypatch, ckan_config, tmpdir): | ||
| monkeypatch.setitem(ckan_config, 'ckan.storage_path', str(tmpdir)) | ||
| monkeypatch.setattr(ckan.lib.uploader, | ||
| 'get_storage_path', | ||
| lambda: str(tmpdir)) | ||
| monkeypatch.setattr( | ||
| ckanext.dcor_schemas.plugin, | ||
| 'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS', | ||
| True) | ||
|
|
||
| ds_dict, res_dict = make_dataset( | ||
| create_with_upload=create_with_upload, | ||
| resource_path=data_path / "calibration_beads_47.rtdc", | ||
| activate=True, | ||
| private=False, | ||
| authors="Peter Pan") | ||
|
|
||
| rid = res_dict["id"] | ||
|
|
||
| bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid) | ||
|
|
||
| # Check whether the S3 resource exists | ||
| assert s3.object_exists(bucket_name, object_name) | ||
| # Check that the organization exists | ||
| org_list = ckan.logic.get_action("organization_list")() | ||
| assert ds_dict["organization"]["name"] in org_list | ||
|
|
||
| # Attempt to remove objects from S3, the object should still be there | ||
| # afterward. | ||
| inspect.check_orphaned_s3_artifacts(assume_yes=True, | ||
| older_than_days=0) | ||
| assert s3.object_exists(bucket_name, object_name) | ||
|
|
||
| # Delete the entire dataset | ||
| call_action(action_name="package_delete", | ||
| context={'ignore_auth': True, 'user': 'default'}, | ||
| id=ds_dict["id"] | ||
| ) | ||
| call_action(action_name="dataset_purge", | ||
| context={'ignore_auth': True, 'user': 'default'}, | ||
| id=ds_dict["id"] | ||
| ) | ||
|
|
||
| # Make sure that the S3 object is still there | ||
| assert s3.object_exists(bucket_name, object_name) | ||
|
|
||
| # Perform the actual cleanup | ||
| inspect.check_orphaned_s3_artifacts(assume_yes=True, | ||
| older_than_days=0) | ||
| assert not s3.object_exists(bucket_name, object_name) |