From c4526d8ab03706d2a0682227b0eb27f21a72ba42 Mon Sep 17 00:00:00 2001 From: MaxGelbakhiani Date: Tue, 19 Mar 2024 10:21:35 +0100 Subject: [PATCH] Tests for replace_objects option Signed-off-by: MaxGelbakhiani --- .github/workflows/tests.yml | 69 +++++++++++++++++++++++++++++++++++++ tests/conftest.py | 11 ++++++ tests/test_downloads.py | 7 ++-- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53b1544..51803a8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,6 +76,8 @@ jobs: PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }} NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} URL_PREFIX: ${{ env.URL_PREFIX }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -96,6 +98,8 @@ jobs: LIFETIME: ${{ vars.LIFETIME }} PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }} URL_PREFIX: ${{ env.URL_PREFIX }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -136,6 +140,8 @@ jobs: LIFETIME: ${{ vars.LIFETIME }} PATH_TO_FILES_DIR: ${{ env.PREFIX_DIR }} NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False - name: Run tests env: @@ -144,3 +150,66 @@ jobs: run: | source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" working-directory: ./tests + + - name: Prepare directory name for tests with object by object replacement + shell: bash + env: + TIMESTAMP: ${{ steps.date.outputs.timestamp }} + run: | + echo "REPL_DATA_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-obj-by-obj-replacement" >> $GITHUB_ENV + + - name: Create a directory tree for tests with objects replacement + shell: bash + run: | + mkdir "$REPL_DATA_DIR" + + - name: Move files to the directory tree for tests with url prefix + shell: bash + env: + SOURCE_DIR: ${{ env.TESTS_DATA_DIR }} + DEST_DIR: ${{ env.REPL_DATA_DIR }} + run: | + rsync -av "$SOURCE_DIR" "$DEST_DIR" + + - name: Run gh-push-to-neofs with pre objects replacement + id: gh_push_to_neofs_with_pre_replace_objects + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }} + REPLACE_OBJECTS: False + REPLACE_CONTAINER_CONTENTS: False + + - name: Modify test data + shell: bash + env: + DATA_DIR: ${{ env.REPL_DATA_DIR }} + run: | + find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_replaced_obj_by_obj/' {} + + + - name: Run gh-push-to-neofs with objects replacement + id: gh_push_to_neofs_with_replace_objects + uses: ./ + with: + NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }} + NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }} + NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }} + NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }} + STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }} + LIFETIME: ${{ vars.LIFETIME }} + PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }} + REPLACE_OBJECTS: True + REPLACE_CONTAINER_CONTENTS: False + + - name: Run tests + env: + OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_replace_objects.outputs.OUTPUT_CONTAINER_URL }} + REPORT_DIR: ${{ env.REPL_DATA_DIR }} + run: | + source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR + working-directory: ./tests diff --git a/tests/conftest.py b/tests/conftest.py index c1c8209..f1a3d98 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,12 @@ def pytest_addoption(parser): default=None, help="Directory combine report and zip files", ) + parser.addoption( + "--data_dir_prefix", + action="store", + default=None, + help="Prefix dir to add to modified test data", + ) @pytest.fixture @@ -21,3 +27,8 @@ def base_url(request): @pytest.fixture def report_dir(request): return request.config.getoption("--report_dir") + + +@pytest.fixture +def data_dir_prefix(request): + return request.config.getoption("--data_dir_prefix") diff --git a/tests/test_downloads.py b/tests/test_downloads.py index c84d1e9..c0c3bda 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -20,9 +20,10 @@ def download_file(url: str) -> str: "data/dir/subdir/subdir_2/5.txt", ], ) -def test_file_content(base_url, report_dir, path): +def test_file_content(base_url, report_dir, data_dir_prefix, path): if base_url is None: pytest.fail("base_url is not provided. Provide it using --base_url option.") + full_path = os.path.join(data_dir_prefix or "", path) if not base_url.endswith("/"): base_url += "/" @@ -35,9 +36,9 @@ def test_file_content(base_url, report_dir, path): print(f"full_url: {full_url}") remote_content = download_file(full_url) - with open(path, "r") as local_file: + with open(full_path, "r") as local_file: local_content = local_file.read() assert ( remote_content == local_content - ), f"Contents of {full_url} and {path} do not match." + ), f"Contents of {full_url} and {full_path} do not match."