Fix "No space left" (#572)

In this PR I fix #571 and reorganize the tests a bit.
Now after each test module the temporary files
and objects are cleared, the logs for each test are available in the
report, the test marks are updated.

pytest_tests/helpers/file_helper.py

@@ -5,7 +5,7 @@
 from typing import Any, Optional
 
 import allure
-from common import ASSETS_DIR
+from common import ASSETS_DIR, TEST_FILES_DIR
 
 logger = logging.getLogger("NeoLogger")
 
@@ -19,7 +19,7 @@ def generate_file(size: int) -> str:
     Returns:
         The path to the generated file.
     """
-    file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+    file_path = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, str(uuid.uuid4()))
     with open(file_path, "wb") as file:
         file.write(os.urandom(size))
     logger.info(f"File with size {size} bytes has been generated: {file_path}")
@@ -49,7 +49,7 @@ def generate_file_with_content(
         mode = "wb"
 
     if not file_path:
-        file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+        file_path = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, str(uuid.uuid4()))
     else:
         if not os.path.exists(os.path.dirname(file_path)):
             os.makedirs(os.path.dirname(file_path))
@@ -99,7 +99,7 @@ def concat_files(file_paths: list, resulting_file_path: Optional[str] = None) ->
         Path to the resulting file.
     """
     if not resulting_file_path:
-        resulting_file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+        resulting_file_path = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, str(uuid.uuid4()))
     with open(resulting_file_path, "wb") as f:
         for file in file_paths:
             with open(file, "rb") as part_file:

pytest_tests/steps/session_token.py

@@ -9,7 +9,7 @@
 
 import allure
 import json_transformers
-from common import ASSETS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
+from common import ASSETS_DIR, TEST_FILES_DIR, TEST_OBJECTS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG
 from data_formatters import get_wallet_public_key
 from json_transformers import encode_for_json
 from neofs_testlib.cli import NeofsCli
@@ -248,7 +248,7 @@ def create_session_token(
     Returns:
         The path to the generated session token file.
     """
-    session_token = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+    session_token = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, str(uuid.uuid4()))
     neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC)
     neofscli.session.create(
         rpc_endpoint=rpc_endpoint,
@@ -273,7 +273,7 @@ def sign_session_token(shell: Shell, session_token_file: str, wlt: WalletFile) -
     Returns:
         The path to the signed token.
     """
-    signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
+    signed_token_file = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, str(uuid.uuid4()))
     neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG)
     neofscli.util.sign_session_token(
         wallet=wlt.path, from_file=session_token_file, to_file=signed_token_file

pytest_tests/testsuites/acl/conftest.py

@@ -68,7 +68,7 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
 
 
 @pytest.fixture(scope="module")
-def file_path(simple_object_size):
+def file_path(simple_object_size, artifacts_directory):
     yield generate_file(simple_object_size)
 
 

pytest_tests/testsuites/acl/storage_group/test_storagegroup.py

@@ -6,7 +6,7 @@
 import allure
 import pytest
 from cluster_test_base import ClusterTestBase
-from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
+from common import ASSETS_DIR, TEST_FILES_DIR, FREE_STORAGE, WALLET_PASS
 from file_helper import generate_file
 from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
 from neofs_testlib.utils.wallet import init_wallet
@@ -46,7 +46,7 @@ class TestStorageGroup(ClusterTestBase):
     @pytest.fixture(autouse=True)
     def prepare_two_wallets(self, default_wallet):
         self.main_wallet = default_wallet
-        self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
+        self.other_wallet = os.path.join(os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, f"{str(uuid.uuid4())}.json")
         init_wallet(self.other_wallet, WALLET_PASS)
         if not FREE_STORAGE:
             main_chain = self.cluster.main_chain_nodes[0]

pytest_tests/testsuites/conftest.py

@@ -3,6 +3,7 @@
 import re
 import shutil
 import uuid
+import hashlib
 from datetime import datetime
 
 import allure
@@ -12,6 +13,8 @@
 from cluster import Cluster
 from common import (
     ASSETS_DIR,
+    TEST_FILES_DIR,
+    TEST_OBJECTS_DIR,
     COMPLEX_OBJECT_CHUNKS_COUNT,
     COMPLEX_OBJECT_TAIL_SIZE,
     FREE_STORAGE,
@@ -134,35 +137,63 @@ def check_binary_versions(request, hosting: Hosting, client_shell: Shell):
 
 @pytest.fixture(scope="session")
 @allure.title("Prepare tmp directory")
-def temp_directory():
+def temp_directory() -> str:
     with allure.step("Prepare tmp directory"):
         full_path = os.path.join(os.getcwd(), ASSETS_DIR)
-        shutil.rmtree(full_path, ignore_errors=True)
-        os.mkdir(full_path)
+        create_dir(full_path)
 
     yield full_path
 
     with allure.step("Remove tmp directory"):
-        shutil.rmtree(full_path)
+        remove_dir(full_path)
+
+
+@pytest.fixture(scope="module", autouse=True)
+@allure.title(f'Prepare test files directories')
+def artifacts_directory(temp_directory: str) -> None:
+    dirs = [TEST_FILES_DIR, TEST_OBJECTS_DIR]
+    for dir_name in dirs:
+        with allure.step(f"Prepare {dir_name} directory"):
+            full_path = os.path.join(temp_directory, dir_name)
+            create_dir(full_path)
+
+    yield
+
+    for dir_name in dirs:
+        with allure.step(f"Remove {dir_name} directory"):
+            remove_dir(full_path)
 
 
 @pytest.fixture(scope="session", autouse=True)
-@allure.title("Collect logs")
-def collect_logs(temp_directory, hosting: Hosting):
+@allure.title("Collect full logs")
+def collect_full_tests_logs(temp_directory, hosting: Hosting):
+    test_name = 'full_logs'
     start_time = datetime.utcnow()
     yield
     end_time = datetime.utcnow()
-
-    # Dump logs to temp directory (because they might be too large to keep in RAM)
     logs_dir = os.path.join(temp_directory, "logs")
-    dump_logs(hosting, logs_dir, start_time, end_time)
-    attach_logs(logs_dir)
+    store_logs(hosting, logs_dir, test_name, start_time, end_time)
     check_logs(logs_dir)
 
 
+@pytest.fixture(scope="function", autouse=True)
+@allure.title("Collect test logs")
+def collect_test_logs(request, temp_directory, hosting: Hosting):
+    test_name = request.node.nodeid.translate(str.maketrans(":[]/", "____"))
+    hash_suffix = hashlib.md5(test_name.encode()).hexdigest()
+    file_name = (test_name[:200] + '_' + hash_suffix)  # limit total length to 255
+    logs_dir = os.path.join(temp_directory, "logs")
+    with allure.step(f'Start collecting logs for {file_name}'):
+        start_time = datetime.utcnow()
+    yield
+    with allure.step(f'Stop collecting logs for {file_name}, logs path: {logs_dir} '):
+        end_time = datetime.utcnow()
+        store_logs(hosting, logs_dir, file_name, start_time, end_time)
+
+
 @pytest.fixture(scope="session", autouse=True)
 @allure.title("Run health check for all storage nodes")
-def run_health_check(collect_logs, cluster: Cluster):
+def run_health_check(collect_full_tests_logs, cluster: Cluster):
     failed_nodes = []
     for node in cluster.storage_nodes:
         health_check = storage_node_healthcheck(node)
@@ -288,20 +319,37 @@ def check_logs(logs_dir: str):
         raise pytest.fail(f"System logs {', '.join(logs_with_problem)} contain critical errors")
 
 
+def store_logs(hosting: Hosting, logs_dir: str, file_name: str, start_time: datetime, end_time: datetime) -> None:
+    os.makedirs(logs_dir, exist_ok=True)
+    dump_logs(hosting, logs_dir, start_time, end_time)
+    attach_logs(logs_dir, os.path.join(os.getcwd(), ASSETS_DIR, file_name))
+
+
 def dump_logs(hosting: Hosting, logs_dir: str, since: datetime, until: datetime) -> None:
     # Dump logs to temp directory (because they might be too large to keep in RAM)
-    os.makedirs(logs_dir)
+    os.makedirs(logs_dir, exist_ok=True)
 
     for host in hosting.hosts:
-        with allure.step(f"Dump logs from host {host.config.address}"):
+        with allure.step(f"Dump logs from host {host.config.address} to {logs_dir}"):
             try:
                 host.dump_logs(logs_dir, since=since, until=until)
             except Exception as ex:
                 logger.warning(f"Exception during logs collection: {ex}")
 
 
-def attach_logs(logs_dir: str) -> None:
+def attach_logs(logs_dir: str, test_name: str) -> None:
     # Zip all files and attach to Allure because it is more convenient to download a single
     # zip with all logs rather than mess with individual logs files per service or node
-    logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
-    allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")
+    logs_zip_file_path = shutil.make_archive(test_name, "zip", logs_dir)
+    allure.attach.file(logs_zip_file_path, name=f'{test_name}.zip', extension="zip")
+
+
+def create_dir(dir_path: str) -> None:
+    with allure.step("Create directory"):
+        remove_dir(dir_path)
+        os.mkdir(dir_path)
+
+
+def remove_dir(dir_path: str) -> None:
+    with allure.step("Remove directory"):
+        shutil.rmtree(dir_path, ignore_errors=True)

pytest_tests/testsuites/object/test_object_api.py

@@ -30,6 +30,7 @@
 from helpers.storage_object_info import StorageObjectInfo
 from steps.cluster_test_base import ClusterTestBase
 from steps.storage_object import delete_objects
+from test_control import expect_not_raises
 
 logger = logging.getLogger("NeoLogger")
 
@@ -93,7 +94,7 @@ def generate_ranges(
     params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
     ids=["simple object", "complex object"],
     # Scope session to upload/delete each files set only once
-    scope="module",
+    scope="function",
 )
 def storage_objects(
     default_wallet: str, client_shell: Shell, cluster: Cluster, request: FixtureRequest
@@ -131,7 +132,8 @@ def storage_objects(
     yield storage_objects
 
     # Teardown after all tests done with current param
-    delete_objects(storage_objects, client_shell, cluster)
+    with expect_not_raises():
+        delete_objects(storage_objects, client_shell, cluster)
 
 
 @pytest.mark.grpc_api

pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py

@@ -17,7 +17,6 @@ def pytest_generate_tests(metafunc):
 class TestS3GateACL(TestS3GateBase):
     @pytest.mark.sanity
     @allure.title("Test S3: Object ACL")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/784")
     def test_s3_object_ACL(self, bucket, simple_object_size):
         file_path = generate_file(simple_object_size)
         file_name = object_key_from_file_path(file_path)

pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py

@@ -24,7 +24,6 @@ def pytest_generate_tests(metafunc):
 class TestS3GateBucket(TestS3GateBase):
     @pytest.mark.sanity
     @allure.title("Test S3: Create Bucket with different ACL")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-s3-gw/issues/784")
     def test_s3_create_bucket_with_ACL(self):
 
         with allure.step("Create bucket with ACL private"):

pytest_tests/testsuites/services/s3_gate/test_s3_locking.py

@@ -73,8 +73,6 @@ def test_s3_object_locking(self, version_id, simple_object_size):
                 s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name, version_id)
 
     @allure.title("Test S3: Checking the impossibility to change the retention mode COMPLIANCE")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-testcases/issues/558")
-    @pytest.mark.nspcc_dev__neofs_testcases__issue_558
     def test_s3_mode_compliance(self, version_id, simple_object_size):
         file_path = generate_file(simple_object_size)
         file_name = object_key_from_file_path(file_path)
@@ -116,8 +114,6 @@ def test_s3_mode_compliance(self, version_id, simple_object_size):
                 )
 
     @allure.title("Test S3: Checking the ability to change retention mode GOVERNANCE")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-testcases/issues/558")
-    @pytest.mark.nspcc_dev__neofs_testcases__issue_558
     def test_s3_mode_governance(self, version_id, simple_object_size):
         file_path = generate_file(simple_object_size)
         file_name = object_key_from_file_path(file_path)
@@ -186,8 +182,6 @@ def test_s3_mode_governance(self, version_id, simple_object_size):
             )
 
     @allure.title("Test S3: Checking if an Object Cannot Be Locked")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-testcases/issues/558")
-    @pytest.mark.nspcc_dev__neofs_testcases__issue_558
     def test_s3_legal_hold(self, version_id, simple_object_size):
         file_path = generate_file(simple_object_size)
         file_name = object_key_from_file_path(file_path)

pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py

@@ -62,8 +62,6 @@ def test_s3_object_multipart(self):
             assert get_file_hash(got_object) == get_file_hash(file_name_large)
 
     @allure.title("Test S3 Multipart abord")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-testcases/issues/558")
-    @pytest.mark.nspcc_dev__neofs_testcases__issue_558
     def test_s3_abort_multipart(self):
         bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
         set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@@ -91,8 +89,6 @@ def test_s3_abort_multipart(self):
             assert not uploads, f"Expected there is no uploads in bucket {bucket}"
 
     @allure.title("Test S3 Upload Part Copy")
-    @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-testcases/issues/558")
-    @pytest.mark.nspcc_dev__neofs_testcases__issue_558
     def test_s3_multipart_copy(self):
         bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
         set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)