Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests: enable network failover tests #818

Merged
merged 1 commit into from
Jun 24, 2024
Merged

tests: enable network failover tests #818

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 23 additions & 12 deletions pytest_tests/tests/failovers/test_failover_network.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import logging
import subprocess
import sys
from random import choices
from time import sleep

Expand All @@ -21,10 +22,10 @@

@pytest.mark.failover
@pytest.mark.failover_network
@pytest.mark.skip(reason="These tests require multiple hosts/inner rings to run")
@pytest.mark.skipif(sys.platform == "darwin", reason="not supported on macos runners")
class TestFailoverNetwork(NeofsEnvTestBase):
@pytest.fixture(autouse=True)
@allure.step("Restore network")
@pytest.fixture
def restore_network(self):
yield

Expand All @@ -36,8 +37,15 @@ def restore_network(self):
if not_empty:
wait_all_storage_nodes_returned(self.neofs_env)

for node in self.neofs_env.storage_nodes:
node.kill()
for node in self.neofs_env.storage_nodes:
node.start(fresh=False)

wait_all_storage_nodes_returned(self.neofs_env)

@allure.title("Block Storage node traffic")
def test_block_storage_node_traffic(self, default_wallet, simple_object_size):
def test_block_storage_node_traffic(self, default_wallet, simple_object_size, restore_network):
"""
Block storage nodes traffic using iptables and wait for replication for objects.
"""
Expand All @@ -56,7 +64,9 @@ def test_block_storage_node_traffic(self, default_wallet, simple_object_size):
)
oid = put_object_to_random_node(wallet.path, source_file_path, cid, shell=self.shell, neofs_env=self.neofs_env)

nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.neofs_env.storage_nodes)
nodes = wait_object_replication(
cid, oid, 2, shell=self.shell, nodes=self.neofs_env.storage_nodes, neofs_env=self.neofs_env
)

logger.info(f"Nodes are {nodes}")
nodes_to_block = nodes
Expand All @@ -79,11 +89,12 @@ def test_block_storage_node_traffic(self, default_wallet, simple_object_size):
2,
shell=self.shell,
nodes=list(set(self.neofs_env.storage_nodes) - set(excluded_nodes)),
neofs_env=self.neofs_env,
)
assert node not in new_nodes

with allure.step("Check object data is not corrupted"):
got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0].endpoint, shell=self.shell)
got_file_path = get_object(wallet.path, cid, oid, endpoint=new_nodes[0].endpoint, shell=self.shell)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)

for node in nodes_to_block:
Expand All @@ -93,14 +104,16 @@ def test_block_storage_node_traffic(self, default_wallet, simple_object_size):
sleep(wakeup_node_timeout)

with allure.step("Check object data is not corrupted"):
new_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.neofs_env.storage_nodes)
new_nodes = wait_object_replication(
cid, oid, 2, shell=self.shell, nodes=self.neofs_env.storage_nodes, neofs_env=self.neofs_env
)

got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].endpoint)
got_file_path = get_object(wallet.path, cid, oid, shell=self.shell, endpoint=new_nodes[0].endpoint)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)

@pytest.mark.sanity
@allure.title("RPC reconnection test")
def test_rpc_reconnection(self, default_wallet):
def test_rpc_reconnection(self, default_wallet, restore_network):
"""
When RPC connection fails (and it can), storage node reconnects to some other node and continues to operate.
"""
Expand All @@ -118,7 +131,6 @@ def test_rpc_reconnection(self, default_wallet):
"inner_ring_candidate_fee",
"maximum_object_size",
"withdrawal_fee",
"systemdns",
"homomorphic_hashing_disabled",
"maintenance_mode_allowed",
]
Expand All @@ -130,7 +142,7 @@ def test_rpc_reconnection(self, default_wallet):
morph_chain_port = self.neofs_env.inner_ring_nodes[0].rpc_address.split(":")[1]

with allure.step(
f"Disconnecting storage node {storage_node.name} " f"from {morph_chain_addr} {dport_repeat} times"
f"Disconnecting storage node {storage_node} " f"from {morph_chain_addr} {dport_repeat} times"
):
for repeat in range(dport_repeat):
with allure.step(f"Disconnect number {repeat}"):
Expand All @@ -156,8 +168,7 @@ def test_rpc_reconnection(self, default_wallet):
# Delay between shutdown attempts, emulates a real disconnection
sleep(1)
logger.info(
f"Disconnected storage node {storage_node.name} "
f"from {morph_chain_addr} {dport_repeat} times"
f"Disconnected storage node {storage_node} " f"from {morph_chain_addr} {dport_repeat} times"
)

for node in self.neofs_env.storage_nodes:
Expand Down
Loading