From 0563cfd3c60a91284fbf03b3eebe9d727417e473 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Mon, 15 Jan 2024 20:35:26 -0500 Subject: [PATCH] Adapt http tests to new dynamic env Signed-off-by: Evgeniy Zayats --- .../lib/helpers/storage_object_info.py | 25 + .../lib/http_gw/http_utils.py | 78 ++ dynamic_env_pytest_tests/lib/s3/s3_utils.py | 41 + dynamic_env_pytest_tests/pytest.ini | 4 +- dynamic_env_pytest_tests/tests/conftest.py | 2 +- .../services/http_gate/test_http_bearer.py | 140 +++ .../services/http_gate/test_http_gate.py | 553 +++++++++++ .../services/http_gate/test_http_headers.py | 228 +++++ .../services/http_gate/test_http_object.py | 138 +++ .../services/http_gate/test_http_streaming.py | 70 ++ .../http_gate/test_http_system_header.py | 417 ++++++++ .../tests/services/s3_gate/test_s3_object.py | 895 ++++++++++++++++++ .../tests/services/s3_gate/test_s3_policy.py | 155 +++ requirements.txt | 2 +- robot/resources/files/container_policy.json | 4 +- .../lib/python_keywords/neofs_verbs.py | 22 +- venv/no-dev-env-pytest/environment.sh | 1 + 17 files changed, 2764 insertions(+), 11 deletions(-) create mode 100644 dynamic_env_pytest_tests/lib/helpers/storage_object_info.py create mode 100644 dynamic_env_pytest_tests/lib/http_gw/http_utils.py create mode 100644 dynamic_env_pytest_tests/lib/s3/s3_utils.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py create mode 100644 dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py create mode 100644 dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_object.py create mode 100644 dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_policy.py diff --git a/dynamic_env_pytest_tests/lib/helpers/storage_object_info.py b/dynamic_env_pytest_tests/lib/helpers/storage_object_info.py new file mode 100644 index 000000000..dd46740e1 --- /dev/null +++ b/dynamic_env_pytest_tests/lib/helpers/storage_object_info.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class ObjectRef: + cid: str + oid: str + + +@dataclass +class LockObjectInfo(ObjectRef): + lifetime: Optional[int] = None + expire_at: Optional[int] = None + + +@dataclass +class StorageObjectInfo(ObjectRef): + size: Optional[int] = None + wallet_file_path: Optional[str] = None + file_path: Optional[str] = None + file_hash: Optional[str] = None + attributes: Optional[list[dict[str, str]]] = None + tombstone: Optional[str] = None + locks: Optional[list[LockObjectInfo]] = None diff --git a/dynamic_env_pytest_tests/lib/http_gw/http_utils.py b/dynamic_env_pytest_tests/lib/http_gw/http_utils.py new file mode 100644 index 000000000..afdd4a017 --- /dev/null +++ b/dynamic_env_pytest_tests/lib/http_gw/http_utils.py @@ -0,0 +1,78 @@ +import random + +import allure +import neofs_verbs +from grpc_responses import OBJECT_NOT_FOUND, error_matches_status +from neofs_testlib.env.env import StorageNode +from neofs_testlib.shell import Shell +from python_keywords.http_gate import assert_hashes_are_equal, get_via_http_gate +from python_keywords.neofs_verbs import get_object + + +def get_object_and_verify_hashes( + oid: str, + file_name: str, + wallet: str, + cid: str, + shell: Shell, + nodes: list[StorageNode], + endpoint: str, + object_getter=None, +) -> None: + nodes_list = get_nodes_without_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + nodes=nodes, + ) + # for some reason we can face with case when nodes_list is empty due to object resides in all nodes + if nodes_list: + random_node = random.choice(nodes_list) + else: + random_node = random.choice(nodes) + + object_getter = object_getter or get_via_http_gate + + got_file_path = get_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + endpoint=random_node.endpoint, + ) + got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint) + + assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) + + +@allure.step("Get Nodes Without Object") +def get_nodes_without_object( + wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode] +) -> list[StorageNode]: + """ + The function returns list of nodes which do not store + the given object. + Args: + wallet (str): the path to the wallet on whose behalf + we request the nodes + cid (str): ID of the container which store the object + oid (str): object ID + shell: executor for cli command + Returns: + (list): nodes which do not store the object + """ + nodes_list = [] + for node in nodes: + try: + res = neofs_verbs.head_object( + wallet, cid, oid, shell=shell, endpoint=node.endpoint, is_direct=True + ) + if res is None: + nodes_list.append(node) + except Exception as err: + if error_matches_status(err, OBJECT_NOT_FOUND): + nodes_list.append(node) + else: + raise Exception(f"Got error {err} on head object command") from err + return nodes_list diff --git a/dynamic_env_pytest_tests/lib/s3/s3_utils.py b/dynamic_env_pytest_tests/lib/s3/s3_utils.py new file mode 100644 index 000000000..270b9c75e --- /dev/null +++ b/dynamic_env_pytest_tests/lib/s3/s3_utils.py @@ -0,0 +1,41 @@ +import logging + +import allure +import neofs_verbs +from neofs_testlib.env.env import StorageNode +from neofs_testlib.shell import Shell + +logger = logging.getLogger("NeoLogger") + + +@allure.step("Get Simple Object Copies") +def get_simple_object_copies( + wallet: str, cid: str, oid: str, shell: Shell, nodes: list[StorageNode] +) -> int: + """ + To figure out the number of a simple object copies, only direct + HEAD requests should be made to the every node of the container. + We consider non-empty HEAD response as a stored object copy. + Args: + wallet (str): the path to the wallet on whose behalf the + copies are got + cid (str): ID of the container + oid (str): ID of the Object + shell: executor for cli command + nodes: nodes to search on + Returns: + (int): the number of object copies in the container + """ + copies = 0 + for node in nodes: + try: + response = neofs_verbs.head_object( + wallet, cid, oid, shell=shell, endpoint=node.endpoint, is_direct=True + ) + if response: + logger.info(f"Found object {oid} on node {node}") + copies += 1 + except Exception: + logger.info(f"No {oid} object copy found on {node}, continue") + continue + return copies diff --git a/dynamic_env_pytest_tests/pytest.ini b/dynamic_env_pytest_tests/pytest.ini index b6276362a..b0bfed1b3 100644 --- a/dynamic_env_pytest_tests/pytest.ini +++ b/dynamic_env_pytest_tests/pytest.ini @@ -1,8 +1,8 @@ [pytest] log_cli = 1 log_cli_level = debug -log_cli_format = %(asctime)s [%(levelname)4s] %(message)s -log_format = %(asctime)s [%(levelname)4s] %(message)s +log_cli_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s +log_format = [%(threadName)s] %(asctime)s [%(levelname)4s] %(message)s [%(threadName)s] log_cli_date_format = %Y-%m-%d %H:%M:%S log_date_format = %H:%M:%S markers = diff --git a/dynamic_env_pytest_tests/tests/conftest.py b/dynamic_env_pytest_tests/tests/conftest.py index 0c47e94f7..038caed9a 100644 --- a/dynamic_env_pytest_tests/tests/conftest.py +++ b/dynamic_env_pytest_tests/tests/conftest.py @@ -36,7 +36,7 @@ def neofs_env(request): neofs_env.neofs_adm().morph.set_config( rpc_endpoint=f"http://{neofs_env.morph_rpc}", alphabet_wallets=neofs_env.alphabet_wallets_dir, - post_data=f"ContainerFee=0 ContainerAliasFee=0", + post_data=f"ContainerFee=0 ContainerAliasFee=0 MaxObjectSize=1024", ) yield neofs_env diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py new file mode 100644 index 000000000..32a9ca5b8 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_bearer.py @@ -0,0 +1,140 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import upload_via_http_gate_curl +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.acl import ( + EACLAccess, + EACLOperation, + EACLRole, + EACLRule, + bearer_token_base64_from_file, + create_eacl, + form_bearertoken_file, + set_eacl, + sign_bearer, + wait_for_cache_expired, +) +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_bearer(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_bearer.wallet = default_wallet + + @pytest.fixture(scope="class") + def user_container(self) -> str: + return create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + @pytest.fixture(scope="class") + def eacl_deny_for_others(self, user_container: str) -> None: + with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"): + eacl = EACLRule( + access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT + ) + set_eacl( + self.wallet.path, + user_container, + create_eacl(user_container, eacl, shell=self.shell), + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + wait_for_cache_expired() + + @pytest.fixture(scope="class") + def bearer_token_no_limit_for_others(self, user_container: str) -> str: + with allure.step(f"Create bearer token for {EACLRole.OTHERS} with all operations allowed"): + bearer = form_bearertoken_file( + self.wallet.path, + user_container, + [ + EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) + for op in EACLOperation + ], + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + sign=False, + ) + bearer_signed = f"{bearer}_signed" + sign_bearer( + shell=self.shell, + wallet_path=self.wallet.path, + eacl_rules_file_from=bearer, + eacl_rules_file_to=bearer_signed, + json=False, + ) + return bearer_token_base64_from_file(bearer_signed) + + @allure.title(f"[negative] Put object without bearer token for {EACLRole.OTHERS}") + def test_unable_put_without_bearer_token( + self, simple_object_size: int, user_container: str, eacl_deny_for_others + ): + eacl_deny_for_others + upload_via_http_gate_curl( + cid=user_container, + filepath=generate_file(simple_object_size), + endpoint=f"http://{self.neofs_env.http_gw.address}", + error_pattern="access to object operation denied", + ) + + @pytest.mark.parametrize("bearer_type", ("header", "cookie")) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_put_with_bearer_when_eacl_restrict( + self, + object_size: int, + bearer_type: str, + user_container: str, + eacl_deny_for_others, + bearer_token_no_limit_for_others: str, + ): + eacl_deny_for_others + bearer = bearer_token_no_limit_for_others + file_path = generate_file(object_size) + with allure.step( + f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes" + ): + headers = None + cookies = None + if bearer_type == "header": + headers = [f" -H 'Authorization: Bearer {bearer}'"] + if bearer_type == "cookie": + cookies = {"Bearer": bearer} + + oid = upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + cookies=cookies, + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=user_container, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py new file mode 100644 index 000000000..1bb188b17 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_gate.py @@ -0,0 +1,553 @@ +import json +import logging +import os + +import allure +import neofs_env.neofs_epoch as neofs_epoch +import pytest +from file_helper import generate_file, generate_file_with_content, get_file_hash +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.container import create_container +from python_keywords.http_gate import ( + attr_into_header, + get_object_by_attr_and_verify_hashes, + get_via_http_curl, + get_via_http_gate, + get_via_zip_http_gate, + try_to_get_object_and_expect_error, + upload_via_http_gate, + upload_via_http_gate_curl, +) +from python_keywords.neofs_verbs import put_object_to_random_node +from utility import wait_for_gc_pass_on_storage_nodes +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") +OBJECT_NOT_FOUND_ERROR = "not found" + + +@allure.link( + "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway" +) +@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") +@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") +@pytest.mark.sanity +@pytest.mark.http_gate +class TestHttpGate(NeofsEnvTestBase): + PLACEMENT_RULE_1 = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" + PLACEMENT_RULE_2 = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + TestHttpGate.wallet = default_wallet + + @allure.title("Test Put over gRPC, Get over HTTP") + def test_put_grpc_get_http(self, complex_object_size, simple_object_size): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create simple and large objects. + 2. Put objects using gRPC (neofs-cli). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Get objects using gRPC (neofs-cli). + 5. Compare hashes for got objects. + 6. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_1, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using gRPC"): + oid_simple = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path_simple, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + ) + oid_large = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path_large, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Verify Content-Disposition header") + def test_put_http_get_http_content_disposition(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Verify Content-Disposition"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + content_disposition_type, filename = resp.headers["Content-Disposition"].split(";") + assert content_disposition_type.strip() == "inline" + assert filename.strip().split("=")[1] == file_path.split("/")[-1] + + with allure.step("Verify Content-Disposition with download=true"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + download=True, + ) + content_disposition_type, filename = resp.headers["Content-Disposition"].split(";") + assert content_disposition_type.strip() == "attachment" + assert filename.strip().split("=")[1] == file_path.split("/")[-1] + + @allure.title("Verify Content-Type if uploaded without any Content-Type specified") + def test_put_http_get_http_without_content_type(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload binary object"): + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "application/octet-stream" + + with allure.step("Upload text object"): + file_path = generate_file_with_content(simple_object_size, content="123") + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "text/plain; charset=utf-8" + + @allure.title("Verify Content-Type if uploaded with X-Attribute-Content-Type") + def test_put_http_get_http_with_x_atribute_content_type(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload object with X-Attribute-Content-Type"): + file_path = generate_file(simple_object_size) + + headers = {"X-Attribute-Content-Type": "CoolContentType"} + oid = upload_via_http_gate( + cid=cid, + path=file_path, + headers=headers, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "CoolContentType" + + @allure.title("Verify Content-Type if uploaded with multipart Content-Type") + def test_put_http_get_http_with_multipart_content_type(self): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + with allure.step("Upload object with multipart content type"): + file_path = generate_file_with_content(0, content="123") + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + file_content_type="application/json", + ) + + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + assert resp.headers["Content-Type"] == "application/json" + + @allure.title("Verify special HTTP headers") + def test_put_http_get_http_special_attributes(self, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + file_path = generate_file(simple_object_size) + + oid = upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + resp = get_via_http_gate( + cid=cid, + oid=oid, + endpoint=f"http://{self.neofs_env.http_gw.address}", + return_response=True, + ) + with open(self.neofs_env.http_gw.wallet.path) as wallet_file: + wallet_json = json.load(wallet_file) + + assert resp.headers["X-Owner-Id"] == wallet_json["accounts"][0]["address"] + assert resp.headers["X-Object-Id"] == oid + assert resp.headers["X-Container-Id"] == cid + + @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") + @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") + @allure.title("Test Put over HTTP, Get over HTTP") + @pytest.mark.smoke + def test_put_http_get_http(self, complex_object_size, simple_object_size): + """ + Test that object can be put and get using HTTP interface. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using HTTP"): + oid_simple = upload_via_http_gate( + cid=cid, path=file_path_simple, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + oid_large = upload_via_http_gate( + cid=cid, path=file_path_large, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.link( + "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes" + ) + @allure.title("Test Put over HTTP, Get over HTTP with headers") + @pytest.mark.parametrize( + "attributes", + [ + {"fileName": "simple_obj_filename"}, + {"file-Name": "simple obj filename"}, + {"cat%jpeg": "cat%jpeg"}, + ], + ids=["simple", "hyphen", "percent"], + ) + def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size): + """ + Test that object can be downloaded using different attributes in HTTP header. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP with particular attributes in the header. + 3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(simple_object_size) + + with allure.step("Put objects using HTTP with attribute"): + headers = attr_into_header(attributes) + oid = upload_via_http_gate( + cid=cid, + path=file_path, + headers=headers, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attributes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Test Expiration-Epoch in HTTP header") + def test_expiration_epoch_in_http(self, simple_object_size): + endpoint = self.neofs_env.sn_rpc + http_endpoint = f"http://{self.neofs_env.http_gw.address}" + + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=endpoint, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(simple_object_size) + oids = [] + + curr_epoch = neofs_epoch.get_epoch(self.neofs_env) + epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100) + + for epoch in epochs: + headers = {"X-Attribute-Neofs-Expiration-Epoch": str(epoch)} + + with allure.step("Put objects using HTTP with attribute Expiration-Epoch"): + oids.append( + upload_via_http_gate( + cid=cid, path=file_path, headers=headers, endpoint=http_endpoint + ) + ) + + assert len(oids) == len(epochs), "Expected all objects have been put successfully" + + with allure.step("All objects can be get"): + for oid in oids: + get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) + + for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]: + self.tick_epochs_and_wait(1) + + # Wait for GC, because object with expiration is counted as alive until GC removes it + wait_for_gc_pass_on_storage_nodes() + + for oid in expired_objects: + try_to_get_object_and_expect_error( + cid=cid, + oid=oid, + error_pattern=OBJECT_NOT_FOUND_ERROR, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step("Other objects can be get"): + for oid in not_expired_objects: + get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) + + @allure.title("Test Zip in HTTP header") + def test_zip_in_http(self, complex_object_size, simple_object_size): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + common_prefix = "my_files" + + headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"} + headers2 = {"X-Attribute-FilePath": f"{common_prefix}/file2"} + + upload_via_http_gate( + cid=cid, + path=file_path_simple, + headers=headers1, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + upload_via_http_gate( + cid=cid, + path=file_path_large, + headers=headers2, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + dir_path = get_via_zip_http_gate( + cid=cid, prefix=common_prefix, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + + with allure.step("Verify hashes"): + assert get_file_hash(f"{dir_path}/file1") == get_file_hash(file_path_simple) + assert get_file_hash(f"{dir_path}/file2") == get_file_hash(file_path_large) + + @pytest.mark.long + @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object") + def test_put_http_get_http_large_file(self, complex_object_size): + """ + This test checks upload and download using curl with 'large' object. + Large is object with size up to 20Mb. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + + obj_size = int(os.getenv("BIG_OBJ_SIZE", complex_object_size)) + file_path = generate_file(obj_size) + + with allure.step("Put objects using HTTP"): + oid_gate = upload_via_http_gate( + cid=cid, path=file_path, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + oid_curl = upload_via_http_gate_curl( + cid=cid, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + get_object_and_verify_hashes( + oid=oid_gate, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + get_object_and_verify_hashes( + oid=oid_curl, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + object_getter=get_via_http_curl, + ) + + @allure.title("Test Put/Get over HTTP using Curl utility") + def test_put_http_get_http_curl(self, complex_object_size, simple_object_size): + """ + Test checks upload and download over HTTP using curl utility. + """ + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE_2, + basic_acl=PUBLIC_ACL, + ) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) + + with allure.step("Put objects using curl utility"): + oid_simple = upload_via_http_gate_curl( + cid=cid, + filepath=file_path_simple, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + oid_large = upload_via_http_gate_curl( + cid=cid, + filepath=file_path_large, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + object_getter=get_via_http_curl, + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py new file mode 100644 index 000000000..b9d6778bb --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_headers.py @@ -0,0 +1,228 @@ +import logging +import os + +import allure +import pytest +from container import ( + create_container, + delete_container, + list_containers, + wait_for_container_deletion, +) +from file_helper import generate_file +from http_gate import ( + attr_into_str_header_curl, + get_object_by_attr_and_verify_hashes, + try_to_get_object_and_expect_error, + try_to_get_object_via_passed_request_and_expect_error, + upload_via_http_gate_curl, +) +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from pytest import FixtureRequest +from python_keywords.neofs_verbs import delete_object +from wellknown_acl import PUBLIC_ACL + +from helpers.storage_object_info import StorageObjectInfo + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_headers(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + obj1_keys = ["Writer", "Chapter1", "Chapter2"] + obj2_keys = ["Writer", "Ch@pter1", "chapter2"] + values = ["Leo Tolstoy", "peace", "w@r"] + OBJECT_ATTRIBUTES = [ + {obj1_keys[0]: values[0], obj1_keys[1]: values[1], obj1_keys[2]: values[2]}, + {obj2_keys[0]: values[0], obj2_keys[1]: values[1], obj2_keys[2]: values[2]}, + ] + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_headers.wallet = default_wallet + + @pytest.fixture( + params=[ + pytest.lazy_fixture("simple_object_size"), + pytest.lazy_fixture("complex_object_size"), + ], + ids=["simple object", "complex object"], + scope="class", + ) + def storage_objects_with_attributes(self, request: FixtureRequest) -> list[StorageObjectInfo]: + storage_objects = [] + wallet = self.wallet.path + cid = create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(request.param) + for attributes in self.OBJECT_ATTRIBUTES: + storage_object_id = upload_via_http_gate_curl( + cid=cid, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=attr_into_str_header_curl(attributes), + ) + storage_object = StorageObjectInfo(cid, storage_object_id) + storage_object.size = os.path.getsize(file_path) + storage_object.wallet_file_path = wallet + storage_object.file_path = file_path + storage_object.attributes = attributes + + storage_objects.append(storage_object) + + yield storage_objects + + @allure.title("Get object1 by attribute") + def test_object1_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object#1 by attribute and comapre hashes + + Steps: + 1. Download object#1 with attributes [Chapter2=w@r] and compare hashes + """ + + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + f'Download object#1 via wget with attributes Chapter2: {storage_object_1.attributes["Chapter2"]} and compare hashes' + ): + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs={"Chapter2": storage_object_1.attributes["Chapter2"]}, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("Test get object2 with different attributes, then delete object2 and get object1") + def test_object2_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1, + and when obj2 is deleted you can get obj1 by 1st attribute + + Steps: + 1. Download object#2 with attributes [chapter2=w@r] and compare hashes + 2. Download object#2 with attributes [Ch@pter1=peace] and compare hashes + 3. Delete object#2 + 4. Download object#1 with attributes [Writer=Leo Tolstoy] and compare hashes + """ + storage_object_1 = storage_objects_with_attributes[0] + storage_object_2 = storage_objects_with_attributes[1] + + with allure.step( + f'Download object#2 via wget with attributes [chapter2={storage_object_2.attributes["chapter2"]}] / [Ch@pter1={storage_object_2.attributes["Ch@pter1"]}] and compare hashes' + ): + selected_attributes_object2 = [ + {"chapter2": storage_object_2.attributes["chapter2"]}, + {"Ch@pter1": storage_object_2.attributes["Ch@pter1"]}, + ] + for attributes in selected_attributes_object2: + get_object_by_attr_and_verify_hashes( + oid=storage_object_2.oid, + file_name=storage_object_2.file_path, + cid=storage_object_2.cid, + attrs=attributes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("Delete object#2 and verify is the container deleted"): + delete_object( + wallet=self.wallet.path, + cid=storage_object_2.cid, + oid=storage_object_2.oid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + error_pattern = "404 Not Found" + try_to_get_object_and_expect_error( + cid=storage_object_2.cid, + oid=storage_object_2.oid, + error_pattern=error_pattern, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + storage_objects_with_attributes.remove(storage_object_2) + + with allure.step( + f'Download object#1 with attributes [Writer={storage_object_1.attributes["Writer"]}] and compare hashes' + ): + key_value_pair = {"Writer": storage_object_1.attributes["Writer"]} + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs=key_value_pair, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + @allure.title("[Negative] Try to put object and get right after container is deleted") + def test_negative_put_and_get_object3( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to attempt to put object and try to download it right after the container has been deleted + + Steps: + 1. [Negative] Allocate and attempt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace] + Expected: "Error duplication of attributes detected" + 2. Delete container + 3. [Negative] Try to download object with attributes [peace=peace] + Expected: "HTTP request sent, awaiting response... 404 Not Found" + """ + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]" + ): + file_path_3 = generate_file(storage_object_1.size) + attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"} + headers = attr_into_str_header_curl(attrs_obj3) + headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"}))) + error_pattern = f"key duplication error: X-Attribute-Writer" + upload_via_http_gate_curl( + cid=storage_object_1.cid, + filepath=file_path_3, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=error_pattern, + ) + with allure.step("Delete container and verify container deletion"): + delete_container( + wallet=self.wallet.path, + cid=storage_object_1.cid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + self.tick_epochs_and_wait(1) + wait_for_container_deletion( + self.wallet.path, + storage_object_1.cid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + assert storage_object_1.cid not in list_containers( + self.wallet.path, shell=self.shell, endpoint=self.neofs_env.sn_rpc + ) + with allure.step( + "[Negative] Try to download (wget) object via wget with attributes [peace=peace]" + ): + request = f"/get/{storage_object_1.cid}/peace/peace" + error_pattern = "404 Not Found" + try_to_get_object_via_passed_request_and_expect_error( + cid=storage_object_1.cid, + oid="", + error_pattern=error_pattern, + attrs=attrs_obj3, + http_request_path=request, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py new file mode 100644 index 000000000..2b544bc0e --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_object.py @@ -0,0 +1,138 @@ +import logging +import os + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import ( + get_object_by_attr_and_verify_hashes, + try_to_get_object_via_passed_request_and_expect_error, +) +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.neofs_verbs import put_object_to_random_node +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_object(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_object.wallet = default_wallet + + @allure.title("Test Put over gRPC, Get over HTTP") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_object_put_get_attributes(self, object_size: int): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create object; + 2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war]; + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + 5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace]; + 6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace]; + 7. Compare hashes between original and downloaded object; + 8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID]; + + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container"): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + # Generate file + file_path = generate_file(object_size) + + # List of Key=Value attributes + obj_key1 = "chapter1" + obj_value1 = "peace" + obj_key2 = "chapter2" + obj_value2 = "war" + + # Prepare for grpc PUT request + key_value1 = obj_key1 + "=" + obj_value1 + key_value2 = obj_key2 + "=" + obj_value2 + + with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"): + oid = put_object_to_random_node( + wallet=self.wallet.path, + path=file_path, + cid=cid, + shell=self.shell, + neofs_env=self.neofs_env, + attributes=f"{key_value1},{key_value2}", + ) + with allure.step("Get object and verify hashes [ get/$CID/$OID ]"): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"): + attrs = {obj_key1: obj_value1, obj_key2: obj_value2} + request = f"/get/{cid}/{obj_key1}/{obj_value1}" + expected_err_msg = "Failed to get object via HTTP gate:" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + attrs=attrs, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step( + "Download the object with attribute [get_by_attribute/$CID/chapter1/peace]" + ): + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attrs, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"): + request = f"/get_by_attribute/{cid}/{oid}" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + + with allure.step( + "[Negative] Try to get object with invalid attribute [get_by_attribute/$CID/chapter1/war]" + ): + with pytest.raises(Exception, match=".*object not found.*"): + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs={obj_key1: obj_value2}, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py new file mode 100644 index 000000000..82c8fef75 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_streaming.py @@ -0,0 +1,70 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import upload_via_http_gate_curl +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_streaming(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_streaming.wallet = default_wallet + + @allure.title("Test Put via pipe (steaming), Get over HTTP and verify hashes") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("complex_object_size")], + ids=["complex object"], + ) + def test_object_can_be_put_get_by_streaming(self, object_size: int): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create big object; + 2. Put object using curl with pipe (streaming); + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container and verify container creation"): + cid = create_container( + self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + with allure.step("Allocate big object"): + # Generate file + file_path = generate_file(object_size) + + with allure.step( + "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]" + ): + oid = upload_via_http_gate_curl( + cid=cid, filepath=file_path, endpoint=f"http://{self.neofs_env.http_gw.address}" + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=cid, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) diff --git a/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py new file mode 100644 index 000000000..5491f51b4 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/http_gate/test_http_system_header.py @@ -0,0 +1,417 @@ +import calendar +import datetime +import logging +from typing import Optional + +import allure +import neofs_env.neofs_epoch as neofs_epoch +import pytest +from container import create_container +from file_helper import generate_file +from grpc_responses import OBJECT_NOT_FOUND +from http_gate import ( + attr_into_str_header_curl, + try_to_get_object_and_expect_error, + upload_via_http_gate_curl, +) +from http_gw.http_utils import get_object_and_verify_hashes +from neofs_env.neofs_env_test_base import NeofsEnvTestBase +from python_keywords.neofs_verbs import get_netmap_netinfo, get_object_from_random_node, head_object +from wellknown_acl import PUBLIC_ACL + +logger = logging.getLogger("NeoLogger") +EXPIRATION_TIMESTAMP_HEADER = "__NEOFS__EXPIRATION_TIMESTAMP" +EXPIRATION_EPOCH_HEADER = "__NEOFS__EXPIRATION_EPOCH" +EXPIRATION_DURATION_HEADER = "__NEOFS__EXPIRATION_DURATION" +EXPIRATION_EXPIRATION_RFC = "__NEOFS__EXPIRATION_RFC3339" +NEOFS_EXPIRATION_EPOCH = "Neofs-Expiration-Epoch" +NEOFS_EXPIRATION_DURATION = "Neofs-Expiration-Duration" +NEOFS_EXPIRATION_TIMESTAMP = "Neofs-Expiration-Timestamp" +NEOFS_EXIPRATION_RFC3339 = "Neofs-Expiration-RFC3339" + + +@pytest.mark.http_gate +class Test_http_system_header(NeofsEnvTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_system_header.wallet = default_wallet + + @pytest.fixture(scope="class") + @allure.title("Create container") + def user_container(self): + return create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + @pytest.fixture(scope="class") + @allure.title("epoch_duration in seconds") + def epoch_duration(self) -> int: + net_info = get_netmap_netinfo( + wallet=self.wallet.path, + endpoint=self.neofs_env.sn_rpc, + shell=self.shell, + ) + epoch_duration_in_blocks = net_info["epoch_duration"] + time_per_block = net_info["time_per_block"] + return int(epoch_duration_in_blocks * time_per_block) + + @allure.title("Return N-epoch count in minutes") + def epoch_count_into_mins(self, epoch_duration: int, epoch: int) -> str: + mins = epoch_duration * epoch / 60 + return f"{mins}m" + + @allure.title("Return future timestamp after N epochs are passed") + def epoch_count_into_timestamp( + self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False + ) -> str: + current_datetime = datetime.datetime.utcnow() + epoch_count_in_seconds = epoch_duration * epoch + future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds) + if rfc3339: + return future_datetime.isoformat("T") + "Z" + else: + return str(calendar.timegm(future_datetime.timetuple())) + + @allure.title("Check is (header_output) Key=Value exists and equal in passed (header_to_find)") + def check_key_value_presented_header(self, header_output: dict, header_to_find: dict) -> bool: + header_att = header_output["header"]["attributes"] + for key_to_check, val_to_check in header_to_find.items(): + if key_to_check not in header_att or val_to_check != header_att[key_to_check]: + logger.info(f"Unable to find {key_to_check}: '{val_to_check}' in {header_att}") + return False + return True + + @allure.title( + f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent" + ) + def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None: + # check that __NEOFS__EXPIRATION_EPOCH attribute has corresponding epoch + assert self.check_key_value_presented_header( + head_info, {EXPIRATION_EPOCH_HEADER: str(expected_epoch)} + ), f'Expected to find {EXPIRATION_EPOCH_HEADER}: {expected_epoch} in: {head_info["header"]["attributes"]}' + # check that {EXPIRATION_EPOCH_HEADER} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_DURATION_HEADER: ""}) + ), f"Only {EXPIRATION_EPOCH_HEADER} can be displayed in header attributes" + # check that {EXPIRATION_TIMESTAMP_HEADER} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_TIMESTAMP_HEADER: ""}) + ), f"Only {EXPIRATION_TIMESTAMP_HEADER} can be displayed in header attributes" + # check that {EXPIRATION_EXPIRATION_RFC} absents in header output + assert not ( + self.check_key_value_presented_header(head_info, {EXPIRATION_EXPIRATION_RFC: ""}) + ), f"Only {EXPIRATION_EXPIRATION_RFC} can be displayed in header attributes" + + @allure.title("Put / get / verify object and return head command result to invoker") + def oid_header_info_for_object(self, file_path: str, attributes: dict, user_container: str): + oid = upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=attr_into_str_header_curl(attributes), + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet.path, + cid=user_container, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + head = head_object( + wallet=self.wallet.path, + cid=user_container, + oid=oid, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + ) + return oid, head + + @allure.title("[negative] attempt to put object with expired epoch") + def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl( + {"Neofs-Expiration-Epoch": str(neofs_epoch.get_epoch(self.neofs_env) - 1)} + ) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Expiration-Epoch where epoch is expired" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern="object has expired", + ) + + @allure.title("[negative] attempt to put object with negative Neofs-Expiration-Duration") + def test_unable_put_negative_duration(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-Duration": "-1h"}) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Neofs-Expiration-Duration where duration is negative" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_DURATION_HEADER} must be positive", + ) + + @allure.title( + "[negative] attempt to put object with Neofs-Expiration-Timestamp value in the past" + ) + def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-Timestamp": "1635075727"}) + file_path = generate_file(simple_object_size) + with allure.step( + "Put object using HTTP with attribute Neofs-Expiration-Timestamp where duration is in the past" + ): + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_TIMESTAMP_HEADER} must be in the future", + ) + + @allure.title( + "[negative] Put object using HTTP with attribute Neofs-Expiration-RFC3339 where duration is in the past" + ) + def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: int): + headers = attr_into_str_header_curl({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) + file_path = generate_file(simple_object_size) + upload_via_http_gate_curl( + cid=user_container, + filepath=file_path, + endpoint=f"http://{self.neofs_env.http_gw.address}", + headers=headers, + error_pattern=f"{EXPIRATION_EXPIRATION_RFC} must be in the future", + ) + + @pytest.mark.sanity + @allure.title("priority of attributes epoch>duration") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_epoch_duration( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 1 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = {NEOFS_EXPIRATION_EPOCH: expected_epoch, NEOFS_EXPIRATION_DURATION: "1m"} + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title( + f"priority of attributes duration>timestamp, duration time has higher priority and should be converted {EXPIRATION_EPOCH_HEADER}" + ) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_dur_timestamp( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXPIRATION_DURATION: self.epoch_count_into_mins( + epoch_duration=epoch_duration, epoch=2 + ), + NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=1 + ), + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title( + f"priority of attributes timestamp>Expiration-RFC, timestamp has higher priority and should be converted {EXPIRATION_EPOCH_HEADER}" + ) + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_attr_priority_timestamp_rfc( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=2 + ), + NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=1, rfc3339=True + ), + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, attributes=attributes, user_container=user_container + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) + + @allure.title("Test that object is automatically delete when expiration passed") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_http_rfc_object_unavailable_after_expir( + self, user_container: str, object_size: int, epoch_duration: int + ): + self.tick_epochs_and_wait(1) + epoch_count = 2 + expected_epoch = neofs_epoch.get_epoch(self.neofs_env) + epoch_count + logger.info( + f"epoch duration={epoch_duration}, current_epoch= {neofs_epoch.get_epoch(self.neofs_env)} expected_epoch {expected_epoch}" + ) + attributes = { + NEOFS_EXIPRATION_RFC3339: self.epoch_count_into_timestamp( + epoch_duration=epoch_duration, epoch=2, rfc3339=True + ) + } + file_path = generate_file(object_size) + with allure.step( + f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" + ): + oid, head_info = self.oid_header_info_for_object( + file_path=file_path, + attributes=attributes, + user_container=user_container, + ) + self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) + with allure.step("Check that object becomes unavailable when epoch is expired"): + self.tick_epochs_and_wait(epoch_count + 1) + # check that {EXPIRATION_EXPIRATION_RFC} absents in header output + assert ( + neofs_epoch.get_epoch(self.neofs_env) == expected_epoch + 1 + ), f"Epochs should be equal: {neofs_epoch.get_epoch(self.neofs_env)} != {expected_epoch + 1}" + + with allure.step("Check object deleted because it expires-on epoch"): + neofs_epoch.wait_for_epochs_align(self.neofs_env) + try_to_get_object_and_expect_error( + cid=user_container, + oid=oid, + error_pattern="404 Not Found", + endpoint=f"http://{self.neofs_env.http_gw.address}", + ) + # check that object is not available via grpc + with pytest.raises(Exception, match=OBJECT_NOT_FOUND): + get_object_from_random_node( + self.wallet.path, + user_container, + oid, + self.shell, + neofs_env=self.neofs_env, + ) diff --git a/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_object.py b/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_object.py new file mode 100644 index 000000000..09890fe5b --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_object.py @@ -0,0 +1,895 @@ +import os +import string +import uuid +from datetime import datetime, timedelta +from random import choices, sample + +import allure +import pytest +from common import ASSETS_DIR, WALLET_PASS +from data_formatters import get_wallet_public_key +from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash +from neofs_testlib.env.env import NodeWallet +from neofs_testlib.utils.wallet import init_wallet +from s3.s3_gate_base import TestNeofsS3GateBase +from s3_helper import ( + assert_object_lock_mode, + assert_object_s3_acl, + check_objects_in_bucket, + parametrize_clients, + set_bucket_versioning, +) + +from pytest_tests.steps import s3_gate_bucket, s3_gate_object + + +def pytest_generate_tests(metafunc): + parametrize_clients(metafunc) + + +@pytest.mark.s3_gate +@pytest.mark.s3_gate_object +class TestS3GateObject(TestNeofsS3GateBase): + @staticmethod + def object_key_from_file_path(full_path: str) -> str: + return os.path.basename(full_path) + + @pytest.mark.sanity + @allure.title("Test S3: Copy object") + def test_s3_copy_object(self, two_buckets, simple_object_size): + file_path = generate_file(simple_object_size) + file_name = self.object_key_from_file_path(file_path) + bucket_1_objects = [file_name] + + bucket_1, bucket_2 = two_buckets + + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket_1) + assert not objects_list, f"Expected empty bucket, got {objects_list}" + + with allure.step("Put object into one bucket"): + s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_path) + + with allure.step("Copy one object into the same bucket"): + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket_1, file_name) + bucket_1_objects.append(copy_obj_path) + check_objects_in_bucket(self.s3_client, bucket_1, bucket_1_objects) + + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket_2) + assert not objects_list, f"Expected empty bucket, got {objects_list}" + + with allure.step("Copy object from first bucket into second"): + copy_obj_path_b2 = s3_gate_object.copy_object_s3( + self.s3_client, bucket_1, file_name, bucket_dst=bucket_2 + ) + check_objects_in_bucket(self.s3_client, bucket_1, expected_objects=bucket_1_objects) + check_objects_in_bucket(self.s3_client, bucket_2, expected_objects=[copy_obj_path_b2]) + + with allure.step("Check copied object has the same content"): + got_copied_file_b2 = s3_gate_object.get_object_s3( + self.s3_client, bucket_2, copy_obj_path_b2 + ) + assert get_file_hash(file_path) == get_file_hash( + got_copied_file_b2 + ), "Hashes must be the same" + + with allure.step("Delete one object from first bucket"): + s3_gate_object.delete_object_s3(self.s3_client, bucket_1, file_name) + bucket_1_objects.remove(file_name) + check_objects_in_bucket(self.s3_client, bucket_1, expected_objects=bucket_1_objects) + check_objects_in_bucket(self.s3_client, bucket_2, expected_objects=[copy_obj_path_b2]) + + with allure.step("Copy one object into the same bucket"): + with pytest.raises(Exception): + s3_gate_object.copy_object_s3(self.s3_client, bucket_1, file_name) + + @allure.title("Test S3: Copy version of object") + def test_s3_copy_version_object(self, two_buckets, simple_object_size): + version_1_content = "Version 1" + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) + obj_key = os.path.basename(file_name_simple) + + bucket_1, bucket_2 = two_buckets + set_bucket_versioning(self.s3_client, bucket_1, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put object into bucket"): + s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_name_simple) + bucket_1_objects = [obj_key] + check_objects_in_bucket(self.s3_client, bucket_1, [obj_key]) + + with allure.step("Copy one object into the same bucket"): + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key) + bucket_1_objects.append(copy_obj_path) + check_objects_in_bucket(self.s3_client, bucket_1, bucket_1_objects) + + set_bucket_versioning(self.s3_client, bucket_2, s3_gate_bucket.VersioningStatus.ENABLED) + with allure.step("Copy object from first bucket into second"): + copy_obj_path_b2 = s3_gate_object.copy_object_s3( + self.s3_client, bucket_1, obj_key, bucket_dst=bucket_2 + ) + check_objects_in_bucket(self.s3_client, bucket_1, expected_objects=bucket_1_objects) + check_objects_in_bucket(self.s3_client, bucket_2, expected_objects=[copy_obj_path_b2]) + + with allure.step("Delete one object from first bucket and check object in bucket"): + s3_gate_object.delete_object_s3(self.s3_client, bucket_1, obj_key) + bucket_1_objects.remove(obj_key) + check_objects_in_bucket(self.s3_client, bucket_1, expected_objects=bucket_1_objects) + + with allure.step("Copy one object into the same bucket"): + with pytest.raises(Exception): + s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key) + + @pytest.mark.acl + @allure.title("Test S3: Checking copy with acl") + def test_s3_copy_acl(self, bucket, simple_object_size): + version_1_content = "Version 1" + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) + obj_key = os.path.basename(file_name_simple) + + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + check_objects_in_bucket(self.s3_client, bucket, [obj_key]) + + with allure.step("Copy object and check acl attribute"): + acl = "private" + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket, obj_key, ACL=acl) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, copy_obj_path) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser", acl=acl) + + @allure.title("Test S3: Copy object with metadata") + def test_s3_copy_metadate(self, bucket, simple_object_size): + object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + file_path = generate_file(simple_object_size) + file_name = self.object_key_from_file_path(file_path) + bucket_1_objects = [file_name] + + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put object into bucket"): + s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path, Metadata=object_metadata + ) + bucket_1_objects = [file_name] + check_objects_in_bucket(self.s3_client, bucket, bucket_1_objects) + + with allure.step("Copy one object"): + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket, file_name) + bucket_1_objects.append(copy_obj_path) + check_objects_in_bucket(self.s3_client, bucket, bucket_1_objects) + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, copy_obj_path) + assert ( + obj_head.get("Metadata") == object_metadata + ), f"Metadata must be {object_metadata}" + + with allure.step("Copy one object with metadata"): + copy_obj_path = s3_gate_object.copy_object_s3( + self.s3_client, bucket, file_name, metadata_directive="COPY" + ) + bucket_1_objects.append(copy_obj_path) + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, copy_obj_path) + assert ( + obj_head.get("Metadata") == object_metadata + ), f"Metadata must be {object_metadata}" + + with allure.step("Copy one object with new metadata"): + object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + copy_obj_path = s3_gate_object.copy_object_s3( + self.s3_client, + bucket, + file_name, + metadata_directive="REPLACE", + metadata=object_metadata_1, + ) + bucket_1_objects.append(copy_obj_path) + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, copy_obj_path) + assert ( + obj_head.get("Metadata") == object_metadata_1 + ), f"Metadata must be {object_metadata_1}" + + @allure.title("Test S3: Copy object with tagging") + def test_s3_copy_tagging(self, bucket, simple_object_size): + object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")] + file_path = generate_file(simple_object_size) + file_name_simple = self.object_key_from_file_path(file_path) + bucket_1_objects = [file_name_simple] + + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put several versions of object into bucket"): + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) + version_id_1 = s3_gate_object.put_object_tagging( + self.s3_client, bucket, file_name_simple, tags=object_tagging + ) + bucket_1_objects = [file_name_simple] + check_objects_in_bucket(self.s3_client, bucket, bucket_1_objects) + + with allure.step("Copy one object without tag"): + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket, file_name_simple) + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, copy_obj_path) + assert got_tags, f"Expected tags, got {got_tags}" + expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging] + for tag in expected_tags: + assert tag in got_tags, f"Expected tag {tag} in {got_tags}" + + with allure.step("Copy one object with tag"): + copy_obj_path_1 = s3_gate_object.copy_object_s3( + self.s3_client, bucket, file_name_simple, tagging_directive="COPY" + ) + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, copy_obj_path_1) + assert got_tags, f"Expected tags, got {got_tags}" + expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging] + for tag in expected_tags: + assert tag in got_tags, f"Expected tag {tag} in {got_tags}" + + with allure.step("Copy one object with new tag"): + tag_key = "tag1" + tag_value = uuid.uuid4() + new_tag = f"{tag_key}={tag_value}" + copy_obj_path = s3_gate_object.copy_object_s3( + self.s3_client, + bucket, + file_name_simple, + tagging_directive="REPLACE", + tagging=new_tag, + ) + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, copy_obj_path) + assert got_tags, f"Expected tags, got {got_tags}" + expected_tags = [{"Key": tag_key, "Value": str(tag_value)}] + for tag in expected_tags: + assert tag in got_tags, f"Expected tag {tag} in {got_tags}" + + @allure.title("Test S3: Delete version of object") + def test_s3_delete_versioning(self, bucket, complex_object_size, simple_object_size): + version_1_content = "Version 1" + version_2_content = "Version 2" + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) + + obj_key = os.path.basename(file_name_simple) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + file_name_1 = generate_file_with_content( + simple_object_size, file_path=file_name_simple, content=version_2_content + ) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) + + with allure.step("Check bucket shows all versions"): + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = { + version.get("VersionId") for version in versions if version.get("Key") == obj_key + } + assert obj_versions == { + version_id_1, + version_id_2, + }, f"Expected object has versions: {version_id_1, version_id_2}" + + with allure.step("Delete 1 version of object"): + delete_obj = s3_gate_object.delete_object_s3( + self.s3_client, bucket, obj_key, version_id=version_id_1 + ) + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = { + version.get("VersionId") for version in versions if version.get("Key") == obj_key + } + assert obj_versions == {version_id_2}, f"Expected object has versions: {version_id_2}" + assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" + + with allure.step("Delete second version of object"): + delete_obj = s3_gate_object.delete_object_s3( + self.s3_client, bucket, obj_key, version_id=version_id_2 + ) + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = { + version.get("VersionId") for version in versions if version.get("Key") == obj_key + } + assert not obj_versions, "Expected object not found" + assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" + + with allure.step("Put new object into bucket"): + file_name_simple = generate_file(complex_object_size) + obj_key = os.path.basename(file_name_simple) + version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + + with allure.step("Delete last object"): + delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key) + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket, True) + assert versions.get("DeleteMarkers", None), f"Expected delete Marker" + assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" + + @allure.title("Test S3: bulk delete version of object") + def test_s3_bulk_delete_versioning(self, bucket, simple_object_size): + version_1_content = "Version 1" + version_2_content = "Version 2" + version_3_content = "Version 3" + version_4_content = "Version 4" + file_name_1 = generate_file_with_content(simple_object_size, content=version_1_content) + + obj_key = os.path.basename(file_name_1) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) + file_name_2 = generate_file_with_content( + simple_object_size, file_path=file_name_1, content=version_2_content + ) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_2) + file_name_3 = generate_file_with_content( + simple_object_size, file_path=file_name_1, content=version_3_content + ) + version_id_3 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_3) + file_name_4 = generate_file_with_content( + simple_object_size, file_path=file_name_1, content=version_4_content + ) + version_id_4 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_4) + version_ids = {version_id_1, version_id_2, version_id_3, version_id_4} + + with allure.step("Check bucket shows all versions"): + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = { + version.get("VersionId") for version in versions if version.get("Key") == obj_key + } + assert obj_versions == version_ids, f"Expected object has versions: {version_ids}" + + with allure.step("Delete two objects from bucket one by one"): + version_to_delete_b1 = sample( + [version_id_1, version_id_2, version_id_3, version_id_4], k=2 + ) + version_to_save = list(set(version_ids) - set(version_to_delete_b1)) + for ver in version_to_delete_b1: + s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key, ver) + + with allure.step("Check bucket shows all versions"): + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = [ + version.get("VersionId") for version in versions if version.get("Key") == obj_key + ] + assert ( + obj_versions.sort() == version_to_save.sort() + ), f"Expected object has versions: {version_to_save}" + + @allure.title("Test S3: Get versions of object") + def test_s3_get_versioning(self, bucket, simple_object_size): + version_1_content = "Version 1" + version_2_content = "Version 2" + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) + + obj_key = os.path.basename(file_name_simple) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + file_name_1 = generate_file_with_content( + simple_object_size, file_path=file_name_simple, content=version_2_content + ) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) + + with allure.step("Get first version of object"): + object_1 = s3_gate_object.get_object_s3( + self.s3_client, bucket, obj_key, version_id_1, full_output=True + ) + assert ( + object_1.get("VersionId") == version_id_1 + ), f"Get object with version {version_id_1}" + + with allure.step("Get second version of object"): + object_2 = s3_gate_object.get_object_s3( + self.s3_client, bucket, obj_key, version_id_2, full_output=True + ) + assert ( + object_2.get("VersionId") == version_id_2 + ), f"Get object with version {version_id_2}" + + with allure.step("Get object"): + object_3 = s3_gate_object.get_object_s3( + self.s3_client, bucket, obj_key, full_output=True + ) + assert ( + object_3.get("VersionId") == version_id_2 + ), f"Get object with version {version_id_2}" + + @allure.title("Test S3: Get range") + def test_s3_get_range(self, bucket, complex_object_size: int, simple_object_size: int): + file_path = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path) + file_hash = get_file_hash(file_path) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) + + with allure.step("Get first version of object"): + object_1_part_1 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_1, + range=[0, int(complex_object_size / 3)], + ) + object_1_part_2 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_1, + range=[int(complex_object_size / 3) + 1, 2 * int(complex_object_size / 3)], + ) + object_1_part_3 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_1, + range=[2 * int(complex_object_size / 3) + 1, complex_object_size], + ) + con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3]) + assert get_file_hash(con_file) == file_hash, "Hashes must be the same" + + with allure.step("Get second version of object"): + object_2_part_1 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_2, + range=[0, int(simple_object_size / 3)], + ) + object_2_part_2 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_2, + range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)], + ) + object_2_part_3 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + version_id_2, + range=[2 * int(simple_object_size / 3) + 1, simple_object_size], + ) + con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3]) + assert get_file_hash(con_file_1) == get_file_hash( + file_name_1 + ), "Hashes must be the same" + + with allure.step("Get object"): + object_3_part_1 = s3_gate_object.get_object_s3( + self.s3_client, bucket, file_name, range=[0, int(simple_object_size / 3)] + ) + object_3_part_2 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)], + ) + object_3_part_3 = s3_gate_object.get_object_s3( + self.s3_client, + bucket, + file_name, + range=[2 * int(simple_object_size / 3) + 1, simple_object_size], + ) + con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3]) + assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same" + + @allure.title("Test S3: Copy object with metadata") + @pytest.mark.smoke + def test_s3_head_object(self, bucket, complex_object_size, simple_object_size): + object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + file_path = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("Put several versions of object into bucket"): + version_id_1 = s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path, Metadata=object_metadata + ) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) + + with allure.step("Get head of first version of object"): + response = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) + assert "LastModified" in response, "Expected LastModified field" + assert "ETag" in response, "Expected ETag field" + assert response.get("Metadata") == {}, "Expected Metadata empty" + assert ( + response.get("VersionId") == version_id_2 + ), f"Expected VersionId is {version_id_2}" + assert response.get("ContentLength") != 0, "Expected ContentLength is not zero" + + with allure.step("Get head ob first version of object"): + response = s3_gate_object.head_object_s3( + self.s3_client, bucket, file_name, version_id=version_id_1 + ) + assert "LastModified" in response, "Expected LastModified field" + assert "ETag" in response, "Expected ETag field" + assert ( + response.get("Metadata") == object_metadata + ), f"Expected Metadata is {object_metadata}" + assert ( + response.get("VersionId") == version_id_1 + ), f"Expected VersionId is {version_id_1}" + assert response.get("ContentLength") != 0, "Expected ContentLength is not zero" + + @allure.title("Test S3: list of object with versions") + @pytest.mark.parametrize("list_type", ["v1", "v2"]) + def test_s3_list_object(self, list_type: str, bucket, complex_object_size): + file_path_1 = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path_1) + file_path_2 = generate_file(complex_object_size) + file_name_2 = self.object_key_from_file_path(file_path_2) + + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + with allure.step("Put several versions of object into bucket"): + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1) + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2) + + with allure.step("Get list of object"): + if list_type == "v1": + list_obj = s3_gate_object.list_objects_s3(self.s3_client, bucket) + elif list_type == "v2": + list_obj = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket) + assert len(list_obj) == 2, f"bucket have 2 objects" + assert ( + list_obj.sort() == [file_name, file_name_2].sort() + ), f"bucket have object key {file_name, file_name_2}" + + with allure.step("Delete object"): + delete_obj = s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name) + if list_type == "v1": + list_obj_1 = s3_gate_object.list_objects_s3( + self.s3_client, bucket, full_output=True + ) + elif list_type == "v2": + list_obj_1 = s3_gate_object.list_objects_s3_v2( + self.s3_client, bucket, full_output=True + ) + contents = list_obj_1.get("Contents", []) + assert len(contents) == 1, f"bucket have only 1 object" + assert contents[0].get("Key") == file_name_2, f"bucket has object key {file_name_2}" + assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" + + @allure.title("Test S3: put object") + def test_s3_put_object(self, bucket, complex_object_size, simple_object_size): + file_path_1 = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path_1) + object_1_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + tag_key_1 = "tag1" + tag_value_1 = uuid.uuid4() + tag_1 = f"{tag_key_1}={tag_value_1}" + object_2_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + tag_key_2 = "tag2" + tag_value_2 = uuid.uuid4() + tag_2 = f"{tag_key_2}={tag_value_2}" + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.SUSPENDED) + + with allure.step("Put first object into bucket"): + s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path_1, Metadata=object_1_metadata, Tagging=tag_1 + ) + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) + assert obj_head.get("Metadata") == object_1_metadata, "Matadata must be the same" + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name) + assert got_tags, f"Expected tags, got {got_tags}" + assert got_tags == [ + {"Key": tag_key_1, "Value": str(tag_value_1)} + ], "Tags must be the same" + + with allure.step("Rewrite file into bucket"): + file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2 + ) + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) + assert obj_head.get("Metadata") == object_2_metadata, "Matadata must be the same" + got_tags_1 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name) + assert got_tags_1, f"Expected tags, got {got_tags_1}" + assert got_tags_1 == [ + {"Key": tag_key_2, "Value": str(tag_value_2)} + ], "Tags must be the same" + + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + file_path_3 = generate_file(complex_object_size) + file_hash = get_file_hash(file_path_3) + file_name_3 = self.object_key_from_file_path(file_path_3) + object_3_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + tag_key_3 = "tag3" + tag_value_3 = uuid.uuid4() + tag_3 = f"{tag_key_3}={tag_value_3}" + + with allure.step("Put third object into bucket"): + version_id_1 = s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path_3, Metadata=object_3_metadata, Tagging=tag_3 + ) + obj_head_3 = s3_gate_object.head_object_s3(self.s3_client, bucket, file_name_3) + assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same" + got_tags_3 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3) + assert got_tags_3, f"Expected tags, got {got_tags_3}" + assert got_tags_3 == [ + {"Key": tag_key_3, "Value": str(tag_value_3)} + ], "Tags must be the same" + + with allure.step("Put new version of file into bucket"): + file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_3) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4) + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = { + version.get("VersionId") + for version in versions + if version.get("Key") == file_name_3 + } + assert obj_versions == { + version_id_1, + version_id_2, + }, f"Expected object has versions: {version_id_1, version_id_2}" + got_tags_4 = s3_gate_object.get_object_tagging(self.s3_client, bucket, file_name_3) + assert not got_tags_4, f"No expected tags" + + with allure.step("Get object"): + object_3 = s3_gate_object.get_object_s3( + self.s3_client, bucket, file_name_3, full_output=True + ) + assert ( + object_3.get("VersionId") == version_id_2 + ), f"get object with version {version_id_2}" + object_3 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_3) + assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same" + + with allure.step("Get first version of object"): + object_4 = s3_gate_object.get_object_s3( + self.s3_client, bucket, file_name_3, version_id_1, full_output=True + ) + assert ( + object_4.get("VersionId") == version_id_1 + ), f"get object with version {version_id_1}" + object_4 = s3_gate_object.get_object_s3( + self.s3_client, bucket, file_name_3, version_id_1 + ) + assert file_hash == get_file_hash(object_4), "Hashes must be the same" + obj_head_3 = s3_gate_object.head_object_s3( + self.s3_client, bucket, file_name_3, version_id_1 + ) + assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same" + got_tags_3 = s3_gate_object.get_object_tagging( + self.s3_client, bucket, file_name_3, version_id_1 + ) + assert got_tags_3, f"Expected tags, got {got_tags_3}" + assert got_tags_3 == [ + {"Key": tag_key_3, "Value": str(tag_value_3)} + ], "Tags must be the same" + + @pytest.fixture + def prepare_two_wallets(self, default_wallet, client_shell): + self.main_wallet = default_wallet + self.main_public_key = get_wallet_public_key(self.main_wallet.path, WALLET_PASS) + other_wallet_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") + other_wallet_address = init_wallet(other_wallet_path, WALLET_PASS) + self.other_wallet = NodeWallet( + path=other_wallet_path, address=other_wallet_address, password=WALLET_PASS + ) + self.other_public_key = get_wallet_public_key(self.other_wallet.path, WALLET_PASS) + + @allure.title("Test S3: put object with ACL") + @pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"]) + def test_s3_put_object_acl( + self, + prepare_two_wallets, + bucket_versioning, + bucket, + complex_object_size, + simple_object_size, + ): + file_path_1 = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path_1) + if bucket_versioning == "ENABLED": + status = s3_gate_bucket.VersioningStatus.ENABLED + elif bucket_versioning == "SUSPENDED": + status = s3_gate_bucket.VersioningStatus.SUSPENDED + set_bucket_versioning(self.s3_client, bucket, status) + + with allure.step("Put object with acl private"): + acl = "private" + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1, ACL=acl) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser", acl=acl) + object_1 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) + assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same" + + with allure.step("Put object with acl public-read"): + acl = "public-read" + file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL=acl) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers", acl=acl) + object_2 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) + assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same" + + with allure.step("Put object with acl public-read-write"): + acl = "public-read-write" + file_path_3 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_3, ACL=acl) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers", acl=acl) + object_3 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) + assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same" + + with allure.step("Put object with acl authenticated-read"): + acl = "authenticated-read" + file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4, ACL=acl) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers", acl=acl) + object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) + assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same" + + file_path_5 = generate_file(complex_object_size) + file_name_5 = self.object_key_from_file_path(file_path_5) + + with allure.step("Put object with --grant-full-control id=mycanonicaluserid"): + file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5) + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_6, + GrantFullControl=f"id={self.other_public_key}", + ) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) + assert_object_s3_acl( + acl_grants=obj_acl, permitted_users="CanonicalUser", acl="grant-full-control" + ) + object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) + assert get_file_hash(file_path_5) == get_file_hash(object_4), "Hashes must be the same" + + with allure.step( + "Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers" + ): + file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5) + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_7, + GrantRead="uri=http://acs.amazonaws.com/groups/global/AllUsers", + ) + obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name_5) + assert_object_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers", acl="grant-read") + object_7 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name_5) + assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same" + + @allure.title("Test S3: put object with lock-mode") + def test_s3_put_object_lock_mode(self, complex_object_size, simple_object_size): + file_path_1 = generate_file(complex_object_size) + file_name = self.object_key_from_file_path(file_path_1) + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, True) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step( + "Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status" + ): + date_obj = datetime.utcnow() + timedelta(days=1) + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_1, + ObjectLockMode="GOVERNANCE", + ObjectLockRetainUntilDate=date_obj.strftime("%Y-%m-%dT%H:%M:%S"), + ObjectLockLegalHoldStatus="OFF", + ) + assert_object_lock_mode( + self.s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF" + ) + + with allure.step( + "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]" + ): + date_obj = datetime.utcnow() + timedelta(days=2) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_1, + ObjectLockMode="COMPLIANCE", + ObjectLockRetainUntilDate=date_obj, + ) + assert_object_lock_mode( + self.s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF" + ) + + with allure.step( + "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]" + ): + date_obj = datetime.utcnow() + timedelta(days=3) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_1, + ObjectLockMode="COMPLIANCE", + ObjectLockRetainUntilDate=date_obj, + ObjectLockLegalHoldStatus="ON", + ) + assert_object_lock_mode(self.s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON") + + with allure.step("Put object with lock-mode"): + with pytest.raises( + Exception, + match=r".*must both be supplied*", + ): + # x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied + s3_gate_object.put_object_s3( + self.s3_client, bucket, file_path_1, ObjectLockMode="COMPLIANCE" + ) + + with allure.step("Put object with lock-mode and past date"): + date_obj = datetime.utcnow() - timedelta(days=3) + with pytest.raises( + Exception, + match=r".*until date must be in the future*", + ): + # The retain until date must be in the future + s3_gate_object.put_object_s3( + self.s3_client, + bucket, + file_path_1, + ObjectLockMode="COMPLIANCE", + ObjectLockRetainUntilDate=date_obj, + ) + + @allure.title("Test S3 Sync directory") + @pytest.mark.parametrize("sync_type", ["sync", "cp"]) + @pytest.mark.aws_cli_only + def test_s3_sync_dir(self, sync_type, bucket, simple_object_size): + file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1") + file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2") + object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} + key_to_path = {"test_file_1": file_path_1, "test_file_2": file_path_2} + + generate_file_with_content(simple_object_size, file_path=file_path_1) + generate_file_with_content(simple_object_size, file_path=file_path_2) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + # TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed + if sync_type == "sync": + self.s3_client.sync( + bucket_name=bucket, + dir_path=os.path.dirname(file_path_1), + # ACL="public-read-write", + Metadata=object_metadata, + ) + elif sync_type == "cp": + self.s3_client.cp( + bucket_name=bucket, + dir_path=os.path.dirname(file_path_1), + # ACL="public-read-write", + Metadata=object_metadata, + ) + + with allure.step("Check objects are synced"): + objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert set(key_to_path.keys()) == set( + objects + ), f"Expected all abjects saved. Got {objects}" + + with allure.step("Check these are the same objects"): + for obj_key in objects: + got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key) + assert get_file_hash(got_object) == get_file_hash( + key_to_path.get(obj_key) + ), "Expected hashes are the same" + obj_head = s3_gate_object.head_object_s3(self.s3_client, bucket, obj_key) + assert ( + obj_head.get("Metadata") == object_metadata + ), f"Metadata of object is {object_metadata}" + # Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved + # obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, obj_key) + # assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers") + + @allure.title("Test S3 Put 10 nested level object") + def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size): + path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)]) + file_path_1 = os.path.join(temp_directory, path, "test_file_1") + generate_file_with_content(simple_object_size, file_path=file_path_1) + file_name = self.object_key_from_file_path(file_path_1) + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert not objects_list, f"Expected empty bucket, got {objects_list}" + + with allure.step("Put object"): + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_1) + check_objects_in_bucket(self.s3_client, bucket, [file_name]) diff --git a/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_policy.py b/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_policy.py new file mode 100644 index 000000000..9c875a7a5 --- /dev/null +++ b/dynamic_env_pytest_tests/tests/services/s3_gate/test_s3_policy.py @@ -0,0 +1,155 @@ +import os + +import allure +import pytest +from file_helper import generate_file +from python_keywords.container import search_container_by_name +from s3.s3_gate_base import TestNeofsS3GateBase +from s3.s3_utils import get_simple_object_copies +from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning + +from pytest_tests.steps import s3_gate_bucket, s3_gate_object + + +def pytest_generate_tests(metafunc): + policy = f"{os.getcwd()}/robot/resources/files/policy.json" + if "s3_client" in metafunc.fixturenames: + metafunc.parametrize( + "s3_client", + [("aws cli", policy), ("boto3", policy)], + indirect=True, + ids=["aws cli", "boto3"], + ) + + +@pytest.mark.s3_gate +class TestS3GatePolicy(TestNeofsS3GateBase): + @allure.title("Test S3: Verify bucket creation with retention policy applied") + def test_s3_bucket_location(self, simple_object_size): + file_path_1 = generate_file(simple_object_size) + file_name_1 = object_key_from_file_path(file_path_1) + file_path_2 = generate_file(simple_object_size) + file_name_2 = object_key_from_file_path(file_path_2) + + with allure.step("Create two buckets with different bucket configuration"): + bucket_1 = s3_gate_bucket.create_bucket_s3( + self.s3_client, bucket_configuration="complex" + ) + set_bucket_versioning(self.s3_client, bucket_1, s3_gate_bucket.VersioningStatus.ENABLED) + bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client, bucket_configuration="rep-3") + set_bucket_versioning(self.s3_client, bucket_2, s3_gate_bucket.VersioningStatus.ENABLED) + list_buckets = s3_gate_bucket.list_buckets_s3(self.s3_client) + assert ( + bucket_1 in list_buckets and bucket_2 in list_buckets + ), f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}" + + # with allure.step("Check head buckets"): + head_1 = s3_gate_bucket.head_bucket(self.s3_client, bucket_1) + head_2 = s3_gate_bucket.head_bucket(self.s3_client, bucket_2) + assert head_1 == {} or head_1.get("HEAD") == None, "Expected head is empty" + assert head_2 == {} or head_2.get("HEAD") == None, "Expected head is empty" + + with allure.step("Put objects into buckets"): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_path_1) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket_2, file_path_2) + check_objects_in_bucket(self.s3_client, bucket_1, [file_name_1]) + check_objects_in_bucket(self.s3_client, bucket_2, [file_name_2]) + + with allure.step("Check bucket location"): + bucket_loc_1 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_1) + bucket_loc_2 = s3_gate_bucket.get_bucket_location(self.s3_client, bucket_2) + assert bucket_loc_1 == "complex" + assert bucket_loc_2 == "rep-3" + + with allure.step("Check object policy"): + cid_1 = search_container_by_name( + self.wallet.path, bucket_1, shell=self.shell, endpoint=self.neofs_env.sn_rpc + ) + copies_1 = get_simple_object_copies( + wallet=self.wallet.path, + cid=cid_1, + oid=version_id_1, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + ) + assert copies_1 == 1 + cid_2 = search_container_by_name( + self.wallet.path, bucket_2, shell=self.shell, endpoint=self.neofs_env.sn_rpc + ) + copies_2 = get_simple_object_copies( + wallet=self.wallet.path, + cid=cid_2, + oid=version_id_2, + shell=self.shell, + nodes=self.neofs_env.storage_nodes, + ) + assert copies_2 == 3 + + @allure.title("Test S3: bucket policy ") + def test_s3_bucket_policy(self): + with allure.step("Create bucket with default policy"): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with allure.step("GetBucketPolicy"): + s3_gate_bucket.get_bucket_policy(self.s3_client, bucket) + + with allure.step("Put new policy"): + custom_policy = f"file://{os.getcwd()}/robot/resources/files/bucket_policy.json" + custom_policy = { + "Version": "2008-10-17", + "Id": "aaaa-bbbb-cccc-dddd", + "Statement": [ + { + "Sid": "AddPerm", + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetObject"], + "Resource": [f"arn:aws:s3:::{bucket}/*"], + } + ], + } + + s3_gate_bucket.put_bucket_policy(self.s3_client, bucket, custom_policy) + with allure.step("GetBucketPolicy"): + policy_1 = s3_gate_bucket.get_bucket_policy(self.s3_client, bucket) + print(policy_1) + + @allure.title("Test S3: bucket policy ") + def test_s3_cors(self): + with allure.step("Create bucket without cors"): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + + with pytest.raises(Exception): + bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket) + + with allure.step("Put bucket cors"): + cors = { + "CORSRules": [ + { + "AllowedOrigins": ["http://www.example.com"], + "AllowedHeaders": ["*"], + "AllowedMethods": ["PUT", "POST", "DELETE"], + "MaxAgeSeconds": 3000, + "ExposeHeaders": ["x-amz-server-side-encryption"], + }, + { + "AllowedOrigins": ["*"], + "AllowedHeaders": ["Authorization"], + "AllowedMethods": ["GET"], + "MaxAgeSeconds": 3000, + }, + ] + } + s3_gate_bucket.put_bucket_cors(self.s3_client, bucket, cors) + bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket) + assert bucket_cors == cors.get( + "CORSRules" + ), f"Expected corsrules must be {cors.get('CORSRules')}" + + with allure.step("delete bucket cors"): + s3_gate_bucket.delete_bucket_cors(self.s3_client, bucket) + + with pytest.raises(Exception): + bucket_cors = s3_gate_bucket.get_bucket_cors(self.s3_client, bucket) diff --git a/requirements.txt b/requirements.txt index 3caa37e22..576658da4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ mmh3==3.0.0 multidict==6.0.2 mypy==0.950 mypy-extensions==0.4.3 -neofs-testlib==1.1.14 +neofs-testlib==1.1.16 netaddr==0.8.0 packaging==21.3 paramiko==3.4.0 diff --git a/robot/resources/files/container_policy.json b/robot/resources/files/container_policy.json index eafdfe66f..dac24fa45 100644 --- a/robot/resources/files/container_policy.json +++ b/robot/resources/files/container_policy.json @@ -1,3 +1,5 @@ { - "rep-1": "REP 1" + "rep-1": "REP 1", + "rep-3": "REP 3", + "complex": "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" } diff --git a/robot/resources/lib/python_keywords/neofs_verbs.py b/robot/resources/lib/python_keywords/neofs_verbs.py index dde68dab7..fee4aa83d 100644 --- a/robot/resources/lib/python_keywords/neofs_verbs.py +++ b/robot/resources/lib/python_keywords/neofs_verbs.py @@ -1,6 +1,7 @@ import json import logging import os +import random import re import uuid from typing import Any, Optional @@ -8,8 +9,9 @@ import allure import json_transformers from cluster import Cluster -from common import ASSETS_DIR, TEST_OBJECTS_DIR, NEOFS_CLI_EXEC, WALLET_CONFIG +from common import ASSETS_DIR, NEOFS_CLI_EXEC, TEST_OBJECTS_DIR, WALLET_CONFIG from neofs_testlib.cli import NeofsCli +from neofs_testlib.env.env import NeoFSEnv from neofs_testlib.shell import Shell logger = logging.getLogger("NeoLogger") @@ -21,7 +23,8 @@ def get_object_from_random_node( cid: str, oid: str, shell: Shell, - cluster: Cluster, + cluster: Optional[Cluster] = None, + neofs_env: Optional[NeoFSEnv] = None, bearer: Optional[str] = None, write_object: Optional[str] = None, xhdr: Optional[dict] = None, @@ -47,7 +50,10 @@ def get_object_from_random_node( Returns: (str): path to downloaded file """ - endpoint = cluster.get_random_storage_rpc_endpoint() + if cluster: + endpoint = cluster.get_random_storage_rpc_endpoint() + if neofs_env: + endpoint = random.choice(neofs_env.storage_nodes).endpoint return get_object( wallet, cid, @@ -169,7 +175,8 @@ def put_object_to_random_node( path: str, cid: str, shell: Shell, - cluster: Cluster, + cluster: Optional[Cluster] = None, + neofs_env: Optional[NeoFSEnv] = None, bearer: Optional[str] = None, attributes: Optional[dict] = None, xhdr: Optional[dict] = None, @@ -188,9 +195,9 @@ def put_object_to_random_node( cid: ID of Container where we get the Object from shell: executor for cli command cluster: cluster under test + neofs_env: neofs env under test bearer: path to Bearer Token file, appends to `--bearer` key attributes: User attributes in form of Key1=Value1,Key2=Value2 - cluster: cluster under test wallet_config: path to the wallet config no_progress: do not show progress bar lifetime: Lock lifetime - relative to the current epoch. @@ -201,7 +208,10 @@ def put_object_to_random_node( ID of uploaded Object """ - endpoint = cluster.get_random_storage_rpc_endpoint() + if cluster: + endpoint = cluster.get_random_storage_rpc_endpoint() + if neofs_env: + endpoint = random.choice(neofs_env.storage_nodes).endpoint return put_object( wallet, path, diff --git a/venv/no-dev-env-pytest/environment.sh b/venv/no-dev-env-pytest/environment.sh index e71795f61..38b40c702 100644 --- a/venv/no-dev-env-pytest/environment.sh +++ b/venv/no-dev-env-pytest/environment.sh @@ -1,5 +1,6 @@ # DevEnv variables export NEOFS_MORPH_DISABLE_CACHE=true +export WALLET_PASS=password popd > /dev/null export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers:${VIRTUAL_ENV}/../pytest_tests/steps:${VIRTUAL_ENV}/../pytest_tests/resources:${VIRTUAL_ENV}/../dynamic_env_pytest_tests/lib