Tests for replace_objects option

Signed-off-by: MaxGelbakhiani <[email protected]>
nspcc-dev · Mar 19, 2024 · c4526d8 · c4526d8
1 parent 3173757
commit c4526d8
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 3 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -76,6 +76,8 @@ jobs:
           PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }}
           NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }}
           URL_PREFIX: ${{ env.URL_PREFIX }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -96,6 +98,8 @@ jobs:
           LIFETIME: ${{ vars.LIFETIME }}
           PATH_TO_FILES_DIR: ${{ env.TESTS_DATA_DIR }}
           URL_PREFIX: ${{ env.URL_PREFIX }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -136,6 +140,8 @@ jobs:
           LIFETIME: ${{ vars.LIFETIME }}
           PATH_TO_FILES_DIR: ${{ env.PREFIX_DIR }}
           NEOFS_ATTRIBUTES: ${{ env.NEOFS_ATTRIBUTES }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
 
       - name: Run tests
         env:
@@ -144,3 +150,66 @@ jobs:
         run: |
           source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR"
         working-directory: ./tests
+
+      - name: Prepare directory name for tests with object by object replacement
+        shell: bash
+        env:
+          TIMESTAMP: ${{ steps.date.outputs.timestamp }}
+        run: |
+          echo "REPL_DATA_DIR=${{ github.run_number }}-$TIMESTAMP-$(uuidgen)-obj-by-obj-replacement" >> $GITHUB_ENV
+
+      - name: Create a directory tree for tests with objects replacement
+        shell: bash
+        run: |
+          mkdir "$REPL_DATA_DIR"
+
+      - name: Move files to the directory tree for tests with url prefix
+        shell: bash
+        env:
+          SOURCE_DIR: ${{ env.TESTS_DATA_DIR }}
+          DEST_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          rsync -av "$SOURCE_DIR" "$DEST_DIR"
+
+      - name: Run gh-push-to-neofs with pre objects replacement
+        id: gh_push_to_neofs_with_pre_replace_objects
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }}
+          REPLACE_OBJECTS: False
+          REPLACE_CONTAINER_CONTENTS: False
+
+      - name: Modify test data
+        shell: bash
+        env:
+          DATA_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          find $DATA_DIR -type f -name '*.txt' -exec sed -i '$ s/$/_replaced_obj_by_obj/' {} +
+
+      - name: Run gh-push-to-neofs with objects replacement
+        id: gh_push_to_neofs_with_replace_objects
+        uses: ./
+        with:
+          NEOFS_WALLET: ${{ secrets.NEOFS_WALLET }}
+          NEOFS_WALLET_PASSWORD: ${{ secrets.NEOFS_WALLET_PASSWORD }}
+          NEOFS_NETWORK_DOMAIN: ${{ vars.NEOFS_NETWORK_DOMAIN }}
+          NEOFS_HTTP_GATE: ${{ vars.NEOFS_HTTP_GATE }}
+          STORE_OBJECTS_CID: ${{ vars.STORE_OBJECTS_CID }}
+          LIFETIME: ${{ vars.LIFETIME }}
+          PATH_TO_FILES_DIR: ${{ env.REPL_DATA_DIR }}
+          REPLACE_OBJECTS: True
+          REPLACE_CONTAINER_CONTENTS: False
+
+      - name: Run tests
+        env:
+          OUTPUT_CONTAINER_URL: ${{ steps.gh_push_to_neofs_with_replace_objects.outputs.OUTPUT_CONTAINER_URL }}
+          REPORT_DIR: ${{ env.REPL_DATA_DIR }}
+        run: |
+          source venv/bin/activate && pytest test_downloads.py --base_url="$OUTPUT_CONTAINER_URL" --report_dir="$REPORT_DIR" --data_dir_prefix=../$REPORT_DIR
+        working-directory: ./tests
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -11,6 +11,12 @@ def pytest_addoption(parser):
         default=None,
         help="Directory combine report and zip files",
     )
+    parser.addoption(
+        "--data_dir_prefix",
+        action="store",
+        default=None,
+        help="Prefix dir to add to modified test data",
+    )
 
 
 @pytest.fixture
@@ -21,3 +27,8 @@ def base_url(request):
 @pytest.fixture
 def report_dir(request):
     return request.config.getoption("--report_dir")
+
+
+@pytest.fixture
+def data_dir_prefix(request):
+    return request.config.getoption("--data_dir_prefix")
diff --git a/tests/test_downloads.py b/tests/test_downloads.py
@@ -20,9 +20,10 @@ def download_file(url: str) -> str:
         "data/dir/subdir/subdir_2/5.txt",
     ],
 )
-def test_file_content(base_url, report_dir, path):
+def test_file_content(base_url, report_dir, data_dir_prefix, path):
     if base_url is None:
         pytest.fail("base_url is not provided. Provide it using --base_url option.")
+    full_path = os.path.join(data_dir_prefix or "", path)
 
     if not base_url.endswith("/"):
         base_url += "/"
@@ -35,9 +36,9 @@ def test_file_content(base_url, report_dir, path):
     print(f"full_url: {full_url}")
 
     remote_content = download_file(full_url)
-    with open(path, "r") as local_file:
+    with open(full_path, "r") as local_file:
         local_content = local_file.read()
 
     assert (
         remote_content == local_content
-    ), f"Contents of {full_url} and {path} do not match."
+    ), f"Contents of {full_url} and {full_path} do not match."