From 531c0058b722326131a6ad61158b768966beadae Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 21:22:05 -0400 Subject: [PATCH 1/7] lib: add missing lifetime, expire_at flags Signed-off-by: Evgeniy Zayats --- pytest_tests/steps/session_token.py | 4 ++++ robot/resources/lib/python_keywords/neofs_verbs.py | 8 +++++++- robot/resources/lib/python_keywords/storage_group.py | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pytest_tests/steps/session_token.py b/pytest_tests/steps/session_token.py index b15d4ec14..a0ecdcef6 100644 --- a/pytest_tests/steps/session_token.py +++ b/pytest_tests/steps/session_token.py @@ -236,6 +236,8 @@ def create_session_token( wallet_path: str, wallet_password: str, rpc_endpoint: str, + lifetime: Optional[int] = None, + expire_at: Optional[int] = None ) -> str: """ Create session token for an object. @@ -256,6 +258,8 @@ def create_session_token( wallet=wallet_path, wallet_password=wallet_password, out=session_token, + lifetime=lifetime, + expire_at=expire_at ) return session_token diff --git a/robot/resources/lib/python_keywords/neofs_verbs.py b/robot/resources/lib/python_keywords/neofs_verbs.py index b6fd5d2da..a27774ef2 100644 --- a/robot/resources/lib/python_keywords/neofs_verbs.py +++ b/robot/resources/lib/python_keywords/neofs_verbs.py @@ -174,6 +174,7 @@ def put_object_to_random_node( attributes: Optional[dict] = None, xhdr: Optional[dict] = None, wallet_config: Optional[str] = None, + lifetime: Optional[int] = None, expire_at: Optional[int] = None, no_progress: bool = True, session: Optional[str] = None, @@ -192,7 +193,8 @@ def put_object_to_random_node( cluster: cluster under test wallet_config: path to the wallet config no_progress: do not show progress bar - expire_at: Last epoch in the life of the object + lifetime: Lock lifetime - relative to the current epoch. + expire_at: Last epoch in the life of the object - absolute value. xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token Returns: @@ -213,6 +215,7 @@ def put_object_to_random_node( expire_at, no_progress, session, + lifetime ) @@ -230,6 +233,7 @@ def put_object( expire_at: Optional[int] = None, no_progress: bool = True, session: Optional[str] = None, + lifetime: Optional[int] = None, ): """ PUT of given file. @@ -247,6 +251,7 @@ def put_object( expire_at: Last epoch in the life of the object xhdr: Request X-Headers in form of Key=Value session: path to a JSON-encoded container session token + lifetime: Lock lifetime - relative to the current epoch. Returns: (str): ID of uploaded Object """ @@ -259,6 +264,7 @@ def put_object( cid=cid, attributes=attributes, bearer=bearer, + lifetime=lifetime, expire_at=expire_at, no_progress=no_progress, xhdr=xhdr, diff --git a/robot/resources/lib/python_keywords/storage_group.py b/robot/resources/lib/python_keywords/storage_group.py index 0309fecd3..ea359b462 100644 --- a/robot/resources/lib/python_keywords/storage_group.py +++ b/robot/resources/lib/python_keywords/storage_group.py @@ -26,6 +26,7 @@ def put_storagegroup( bearer: Optional[str] = None, wallet_config: str = WALLET_CONFIG, lifetime: int = 10, + expire_at: Optional[int] = None, ) -> str: """ Wrapper for `neofs-cli storagegroup put`. Before the SG is created, @@ -36,6 +37,7 @@ def put_storagegroup( wallet: Path to wallet on whose behalf the SG is created. cid: ID of Container to put SG to. lifetime: Storage group lifetime in epochs. + expire_at: The last active epoch of the storage group. objects: List of Object IDs to include into the SG. bearer: Path to Bearer token file. wallet_config: Path to neofs-cli config file. @@ -47,6 +49,7 @@ def put_storagegroup( wallet=wallet, cid=cid, lifetime=lifetime, + expire_at=expire_at, members=objects, bearer=bearer, rpc_endpoint=endpoint, From c02c88e2ac5ec536d2abc4e003231ac406cd6ee9 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 21:23:35 -0400 Subject: [PATCH 2/7] lib: add bearer create wrapper Signed-off-by: Evgeniy Zayats --- robot/resources/lib/python_keywords/acl.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/robot/resources/lib/python_keywords/acl.py b/robot/resources/lib/python_keywords/acl.py index dd18dc666..5236333eb 100644 --- a/robot/resources/lib/python_keywords/acl.py +++ b/robot/resources/lib/python_keywords/acl.py @@ -276,3 +276,30 @@ def bearer_token_base64_from_file( with open(bearer_path, "rb") as file: signed = file.read() return base64.b64encode(signed).decode("utf-8") + + +@allure.step("Create bearer token") +def create_bearer_token( + shell, + issued_at: int, + not_valid_before: int, + owner: str, + out: str, + rpc_endpoint: str, + json: Optional[bool] = False, + eacl: Optional[str] = None, + lifetime: Optional[int] = None, + expire_at: Optional[int] = None, +) -> str: + neofscli = NeofsCli(shell=shell, neofs_cli_exec_path=NEOFS_CLI_EXEC, config_file=WALLET_CONFIG) + neofscli.bearer.create( + issued_at=issued_at, + not_valid_before=not_valid_before, + owner=owner, + out=out, + rpc_endpoint=rpc_endpoint, + json=json, + eacl=eacl, + lifetime=lifetime, + expire_at=expire_at, + ) From 5c70015c78ea10c67002d10b47b48c7b5fd0e5dc Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 21:53:13 -0400 Subject: [PATCH 3/7] tests: add test_bearer_token_expiration Test to validate lifetime, expire_at params Signed-off-by: Evgeniy Zayats --- pytest_tests/testsuites/acl/test_bearer.py | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/pytest_tests/testsuites/acl/test_bearer.py b/pytest_tests/testsuites/acl/test_bearer.py index 09e87261b..08b82f763 100644 --- a/pytest_tests/testsuites/acl/test_bearer.py +++ b/pytest_tests/testsuites/acl/test_bearer.py @@ -1,14 +1,22 @@ +import os +import uuid + import allure import pytest from cluster_test_base import ClusterTestBase +from common import ASSETS_DIR, TEST_FILES_DIR, WALLET_PASS +from epoch import get_epoch +from neofs_testlib.utils.wallet import get_last_address_from_wallet from python_keywords.acl import ( EACLAccess, EACLOperation, EACLRole, EACLRule, + create_bearer_token, create_eacl, form_bearertoken_file, set_eacl, + sign_bearer, wait_for_cache_expired, ) from python_keywords.container_access import ( @@ -229,3 +237,76 @@ def test_bearer_token_compound_operations(self, wallets, eacl_container_with_obj shell=self.shell, cluster=self.cluster, ) + + @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-api/issues/273") + @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-node/issues/2538") + @pytest.mark.nspcc_dev__neofs_api__issue_273 + @pytest.mark.nspcc_dev__neofs_node__issue_2538 + @pytest.mark.parametrize("expiration_flag", ["lifetime", "expire_at"]) + def test_bearer_token_expiration(self, wallets, eacl_container_with_objects, expiration_flag): + current_epoch = get_epoch(self.shell, self.cluster) + self.tick_epochs(1) + cid, objects_oids, file_path = eacl_container_with_objects + user_wallet = wallets.get_wallet() + + with allure.step("Create and sign bearer token via cli"): + eacl = [ + EACLRule(access=EACLAccess.ALLOW, role=EACLRole.USER, operation=op) + for op in EACLOperation + ] + + path_to_bearer = os.path.join( + os.getcwd(), ASSETS_DIR, TEST_FILES_DIR, f"bearer_token_{str(uuid.uuid4())}" + ) + + create_bearer_token( + self.shell, + issued_at=1, + not_valid_before=1, + owner=get_last_address_from_wallet(user_wallet.wallet_path, WALLET_PASS), + out=path_to_bearer, + rpc_endpoint=self.cluster.default_rpc_endpoint, + eacl=create_eacl(cid, eacl, shell=self.shell), + lifetime=1 if expiration_flag == "lifetime" else None, + expire_at=current_epoch + 2 if expiration_flag == "expire_at" else None, + ) + + sign_bearer( + shell=self.shell, + wallet_path=user_wallet.wallet_path, + eacl_rules_file_from=path_to_bearer, + eacl_rules_file_to=path_to_bearer, + json=True, + ) + + self.tick_epochs(1) + + with allure.step( + f"Check {EACLRole.USER.value} with token has access to all operations with container" + ): + check_full_access_to_container( + user_wallet.wallet_path, + cid, + objects_oids.pop(), + file_path, + bearer=path_to_bearer, + wallet_config=user_wallet.config_path, + shell=self.shell, + cluster=self.cluster, + ) + + self.tick_epochs(1) + + with allure.step( + f"Check {EACLRole.USER.value} has no access to all operations with container" + ): + check_no_access_to_container( + user_wallet.wallet_path, + cid, + objects_oids.pop(), + file_path, + bearer=path_to_bearer, + wallet_config=user_wallet.config_path, + shell=self.shell, + cluster=self.cluster, + ) From fe46bc7142dea444c200190d4eea710adbe535b3 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 21:57:47 -0400 Subject: [PATCH 4/7] tests: update test_storagegroup_lifetime Add proper checks for lifetime, expire_at params Signed-off-by: Evgeniy Zayats --- .../testsuites/acl/storage_group/test_storagegroup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py index 13a18ff8b..0ca5bacea 100644 --- a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py +++ b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py @@ -6,7 +6,8 @@ import allure import pytest from cluster_test_base import ClusterTestBase -from common import ASSETS_DIR, TEST_FILES_DIR, FREE_STORAGE, WALLET_PASS +from common import ASSETS_DIR, FREE_STORAGE, TEST_FILES_DIR, WALLET_PASS +from epoch import get_epoch from file_helper import generate_file from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from neofs_testlib.utils.wallet import init_wallet @@ -233,7 +234,8 @@ def test_storagegroup_bearer_allow(self, object_size, max_object_size): ) @allure.title("Test to check Storage Group lifetime") - def test_storagegroup_lifetime(self, object_size): + @pytest.mark.parametrize("expiration_flag", ["lifetime", "expire_at"]) + def test_storagegroup_lifetime(self, object_size, expiration_flag, cluster): cid = create_container( self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint ) @@ -242,13 +244,15 @@ def test_storagegroup_lifetime(self, object_size): self.main_wallet, file_path, cid, shell=self.shell, cluster=self.cluster ) objects = [oid] + current_epoch = get_epoch(self.shell, cluster) storage_group = put_storagegroup( self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects, - lifetime=1, + lifetime=1 if expiration_flag == "lifetime" else None, + expire_at=current_epoch + 1 if expiration_flag == "expire_at" else None, ) with allure.step("Tick two epochs"): for _ in range(2): From cbab558a1e9d6428c3b15632062528c8d7a43ae4 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 21:58:32 -0400 Subject: [PATCH 5/7] tests: update test_object_api_lifetime Add proper checks for lifetime, expire_at params Signed-off-by: Evgeniy Zayats --- .../testsuites/object/test_object_lifetime.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pytest_tests/testsuites/object/test_object_lifetime.py b/pytest_tests/testsuites/object/test_object_lifetime.py index 9d4bdc4d8..634b05757 100644 --- a/pytest_tests/testsuites/object/test_object_lifetime.py +++ b/pytest_tests/testsuites/object/test_object_lifetime.py @@ -2,7 +2,7 @@ import allure import pytest -from epoch import get_epoch, tick_epoch +from epoch import get_epoch from file_helper import generate_file, get_file_hash from grpc_responses import OBJECT_NOT_FOUND from pytest import FixtureRequest @@ -19,12 +19,15 @@ class TestObjectApiLifetime(ClusterTestBase): @allure.title("Test object life time") @pytest.mark.parametrize( - "object_size", - [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], - ids=["simple object", "complex object"], + "object_size,expiration_flag", + [ + (pytest.lazy_fixture("simple_object_size"), "lifetime"), + (pytest.lazy_fixture("complex_object_size"), "expire_at"), + ], + ids=["simple object, lifetime", "complex object, expire_at"], ) def test_object_api_lifetime( - self, default_wallet: str, request: FixtureRequest, object_size: int + self, default_wallet: str, request: FixtureRequest, object_size: int, expiration_flag: str ): """ Test object deleted after expiration epoch. @@ -41,7 +44,13 @@ def test_object_api_lifetime( epoch = get_epoch(self.shell, self.cluster) oid = put_object_to_random_node( - wallet, file_path, cid, self.shell, self.cluster, expire_at=epoch + 1 + wallet, + file_path, + cid, + self.shell, + self.cluster, + expire_at=epoch + 1 if expiration_flag == "expire_at" else None, + lifetime=1 if expiration_flag == "lifetime" else None, ) got_file = get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster) assert get_file_hash(got_file) == file_hash From 3b5877a1b01ade9b583ee37da4fe176de217128c Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 22:05:24 -0400 Subject: [PATCH 6/7] tests: add test_session_token_expiration_flags To validate session token expiration flags - lifetime, expire_at Signed-off-by: Evgeniy Zayats --- .../test_object_session_token.py | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/pytest_tests/testsuites/session_token/test_object_session_token.py b/pytest_tests/testsuites/session_token/test_object_session_token.py index 9996fa15c..9034ec858 100644 --- a/pytest_tests/testsuites/session_token/test_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_object_session_token.py @@ -4,8 +4,9 @@ import pytest from cluster_test_base import ClusterTestBase from common import WALLET_PASS +from epoch import get_epoch from file_helper import generate_file -from grpc_responses import SESSION_NOT_FOUND +from grpc_responses import EXPIRED_SESSION_TOKEN, SESSION_NOT_FOUND from neofs_testlib.utils.wallet import get_last_address_from_wallet from python_keywords.container import create_container from python_keywords.neofs_verbs import delete_object, put_object, put_object_to_random_node @@ -144,3 +145,73 @@ def test_object_session_token(self, default_wallet, object_size): endpoint=non_container_node.get_rpc_endpoint(), session=session_token, ) + + @allure.title("Verify session token expiration flags") + @pytest.mark.skip(reason="https://github.com/nspcc-dev/neofs-node/issues/2539") + @pytest.mark.nspcc_dev__neofs_node__issue_2539 + @pytest.mark.parametrize("expiration_flag", ["lifetime", "expire_at"]) + def test_session_token_expiration_flags( + self, default_wallet, simple_object_size, expiration_flag, cluster + ): + rpc_endpoint = self.cluster.storage_nodes[0].get_rpc_endpoint() + + with allure.step("Create Session Token with Lifetime param"): + current_epoch = get_epoch(self.shell, cluster) + + session_token = create_session_token( + shell=self.shell, + owner=get_last_address_from_wallet(default_wallet, ""), + wallet_path=default_wallet, + wallet_password=WALLET_PASS, + rpc_endpoint=rpc_endpoint, + lifetime=1 if expiration_flag == "lifetime" else None, + expire_at=current_epoch + 1 if expiration_flag == "expire_at" else None, + ) + + with allure.step("Create Private Container"): + un_locode = self.cluster.storage_nodes[0].get_un_locode() + locode = "SPB" if un_locode == "RU LED" else un_locode.split()[1] + placement_policy = ( + f"REP 1 IN LOC_{locode}_PLACE CBF 1 SELECT 1 FROM LOC_{locode} " + f'AS LOC_{locode}_PLACE FILTER "UN-LOCODE" ' + f'EQ "{un_locode}" AS LOC_{locode}' + ) + cid = create_container( + default_wallet, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + rule=placement_policy, + ) + + with allure.step("Verify object operations with created session token are allowed"): + file_path = generate_file(simple_object_size) + oid = put_object( + wallet=default_wallet, + path=file_path, + cid=cid, + shell=self.shell, + endpoint=rpc_endpoint, + session=session_token, + ) + delete_object( + wallet=default_wallet, + cid=cid, + oid=oid, + shell=self.shell, + endpoint=rpc_endpoint, + session=session_token, + ) + + self.tick_epochs(2) + + with allure.step("Verify object operations with created session token are not allowed"): + file_path = generate_file(simple_object_size) + with pytest.raises(RuntimeError, match=EXPIRED_SESSION_TOKEN): + oid = put_object( + wallet=default_wallet, + path=file_path, + cid=cid, + shell=self.shell, + endpoint=rpc_endpoint, + session=session_token, + ) From ec7375fa68b25f27f503eef566334bd93b93c5ee Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Thu, 31 Aug 2023 22:10:30 -0400 Subject: [PATCH 7/7] Bump neofs-testlib 1.1.8 -> 1.1.10 Signed-off-by: Evgeniy Zayats --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 383304d4d..00706807d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ mmh3==3.0.0 multidict==6.0.2 mypy==0.950 mypy-extensions==0.4.3 -neofs-testlib==1.1.8 +neofs-testlib==1.1.10 netaddr==0.8.0 packaging==21.3 paramiko==2.10.3