Enable pprof and prometheus for all services

closes #847

Signed-off-by: Evgeniy Zayats <[email protected]>

neofs-testlib/neofs_testlib/env/env.py

@@ -542,6 +542,8 @@ def __init__(self, neofs_env: NeoFSEnv):
         self.main_chain_boltdb = self.neofs_env._generate_temp_file(extension="db", prefix="main_chain_bolt_db")
         self.rpc_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.p2p_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.pprof_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.prometheus_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.stdout = "Not initialized"
         self.stderr = "Not initialized"
         self.process = None
@@ -555,6 +557,8 @@ def __str__(self):
             - BoltDB path: {self.main_chain_boltdb}
             - RPC address: {self.rpc_address}
             - P2P address: {self.p2p_address}
+            - Pprof address: {self.pprof_address}
+            - Prometheus address: {self.prometheus_address}
             - STDOUT: {self.stdout}
             - STDERR: {self.stderr}
         """
@@ -588,6 +592,8 @@ def start(self, wait_until_ready=True):
             p2p_address=self.p2p_address,
             rpc_address=self.rpc_address,
             sn_addresses=[sn.endpoint for sn in self.neofs_env.storage_nodes],
+            pprof_address=self.pprof_address,
+            prometheus_address=self.prometheus_address,
         )
         logger.info(f"Generating CLI config at: {self.cli_config}")
         NeoFSEnv.generate_config_file(
@@ -634,6 +640,8 @@ def __init__(self, neofs_env: NeoFSEnv):
         self.rpc_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.p2p_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.grpc_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.pprof_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.prometheus_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.ir_state_file = self.neofs_env._generate_temp_file(prefix="ir_state_file")
         self.stdout = "Not initialized"
         self.stderr = "Not initialized"
@@ -647,6 +655,8 @@ def __str__(self):
             - RPC address: {self.rpc_address}
             - P2P address: {self.p2p_address}
             - GRPC address: {self.grpc_address}
+            - Pprof address: {self.pprof_address}
+            - Prometheus address: {self.prometheus_address}
             - IR State file path: {self.ir_state_file}
             - STDOUT: {self.stdout}
             - STDERR: {self.stderr}
@@ -713,6 +723,8 @@ def start(self, wait_until_ready=True, with_main_chain=False):
             without_mainnet=f"{not with_main_chain}".lower(),
             main_chain_rpc="localhost:1234" if not with_main_chain else self.neofs_env.main_chain.rpc_address,
             neofs_contract_hash="123" if not with_main_chain else self.neofs_env.main_chain.neofs_contract_hash,
+            pprof_address=self.pprof_address,
+            prometheus_address=self.prometheus_address,
         )
         logger.info(f"Launching Inner Ring Node:{self}")
         self._launch_process()
@@ -766,6 +778,8 @@ def __init__(
         self.shards = [Shard(neofs_env), Shard(neofs_env)]
         self.endpoint = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.control_grpc_endpoint = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.pprof_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.prometheus_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.stdout = "Not initialized"
         self.stderr = "Not initialized"
         self.sn_number = sn_number
@@ -781,6 +795,8 @@ def __str__(self):
             Storage node:
             - Endpoint: {self.endpoint}
             - Control gRPC endpoint: {self.control_grpc_endpoint}
+            - Pprof address: {self.pprof_address}
+            - Prometheus address: {self.prometheus_address}
             - Attributes: {self.attrs}
             - STDOUT: {self.stdout}
             - STDERR: {self.stderr}
@@ -808,6 +824,8 @@ def start(self, fresh=True):
                 shards=self.shards,
                 wallet=self.wallet,
                 state_file=self.state_file,
+                pprof_address=self.pprof_address,
+                prometheus_address=self.prometheus_address,
             )
             logger.info(f"Generating cli config for storage node at: {self.cli_config}")
             NeoFSEnv.generate_config_file(
@@ -920,6 +938,8 @@ def __init__(self, neofs_env: NeoFSEnv):
             password=self.neofs_env.default_password,
         )
         self.address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.pprof_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
+        self.prometheus_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.tls_cert_path = self.neofs_env._generate_temp_file(prefix="s3gw_tls_cert")
         self.tls_key_path = self.neofs_env._generate_temp_file(prefix="s3gw_tls_key")
         self.stdout = "Not initialized"
@@ -931,6 +951,8 @@ def __str__(self):
             S3 Gateway:
             - Address: {self.address}
             - S3 GW Config path: {self.config_path}
+            - Pprof address: {self.pprof_address}
+            - Prometheus address: {self.prometheus_address}
             - STDOUT: {self.stdout}
             - STDERR: {self.stderr}
         """
@@ -978,6 +1000,8 @@ def _generate_config(self):
             tree_service_endpoint=self.neofs_env.storage_nodes[0].endpoint,
             listen_domain=self.neofs_env.domain,
             s3_gw_version=self._get_version(),
+            pprof_address=self.pprof_address,
+            prometheus_address=self.prometheus_address,
         )
 
     def _get_version(self) -> str:

neofs-testlib/neofs_testlib/env/templates/ir.yaml

@@ -138,3 +138,13 @@ settlement:
 
 contracts:
   neofs: {{ neofs_contract_hash }}      # Address of NeoFS contract in mainchain; ignore if mainchain is disabled
+
+pprof:
+  enabled: true
+  address: {{ pprof_address }}
+  shutdown_timeout: 30s
+
+prometheus:
+  enabled: true
+  address: {{ prometheus_address }}
+  shutdown_timeout: 30s

neofs-testlib/neofs_testlib/env/templates/main_chain.yaml

@@ -34,9 +34,13 @@ ApplicationConfiguration:
     EnableCORSWorkaround: false
     MaxGasInvoke: 1500000000
   Prometheus:
-    Enabled: false
+    Enabled: true
+    Addresses:
+      - "{{ prometheus_address }}"
   Pprof:
-    Enabled: false
+    Enabled: true
+    Addresses:
+      - "{{ pprof_address }}"
   Consensus:
     Enabled: true
     UnlockWallet:

neofs-testlib/neofs_testlib/env/templates/s3.yaml

@@ -51,3 +51,11 @@ tree:
 
 listen_domains:
   - {{ listen_domain }}
+
+pprof:
+  enabled: true
+  address: {{ pprof_address }}
+
+prometheus:
+  enabled: true
+  address: {{ prometheus_address }}

neofs-testlib/neofs_testlib/env/templates/sn.yaml

@@ -5,14 +5,14 @@ logger:
 
 # Profiler section
 pprof:
-  enabled: false
-  address: :6060  # Server address
+  enabled: true
+  address: {{ pprof_address }}  # Server address
   shutdown_timeout: 15s  # Timeout for profiling HTTP server graceful shutdown
 
 # Application metrics section
 prometheus:
-  enabled: false
-  address: :9090  # Server address
+  enabled: true
+  address: {{ prometheus_address }}  # Server address
   shutdown_timeout: 15s  # Timeout for metrics HTTP server graceful shutdown
 
 # Morph section

pytest_tests/tests/metrics/test_metrics.py

@@ -0,0 +1,75 @@
+import socket
+
+from neofs_testlib.env.env import NeoFSEnv
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+
+@retry(wait=wait_fixed(10), stop=stop_after_attempt(5), reraise=True)
+def is_port_in_use(host: str, port: str, error_msg: str):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            s.settimeout(1)
+            s.connect((host, int(port)))
+        except (socket.timeout, ConnectionRefusedError):
+            raise AssertionError(error_msg)
+
+
+def test_ports_for_metrics_are_utilized(neofs_env_with_mainchain: NeoFSEnv):
+    is_port_in_use(
+        neofs_env_with_mainchain.main_chain.pprof_address.split(":")[0],
+        neofs_env_with_mainchain.main_chain.pprof_address.split(":")[1],
+        "main chain pprof port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.main_chain.prometheus_address.split(":")[0],
+        neofs_env_with_mainchain.main_chain.prometheus_address.split(":")[1],
+        "main chain prometheus port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.inner_ring_nodes[0].pprof_address.split(":")[0],
+        neofs_env_with_mainchain.inner_ring_nodes[0].pprof_address.split(":")[1],
+        "inner ring node pprof port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.inner_ring_nodes[0].prometheus_address.split(":")[0],
+        neofs_env_with_mainchain.inner_ring_nodes[0].prometheus_address.split(":")[1],
+        "inner ring node prometheus port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.s3_gw.pprof_address.split(":")[0],
+        neofs_env_with_mainchain.s3_gw.pprof_address.split(":")[1],
+        "s3 gw pprof port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.s3_gw.prometheus_address.split(":")[0],
+        neofs_env_with_mainchain.s3_gw.prometheus_address.split(":")[1],
+        "s3 gw prometheus port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.rest_gw.pprof_address.split(":")[0],
+        neofs_env_with_mainchain.rest_gw.pprof_address.split(":")[1],
+        "rest gw pprof port is not utilized",
+    )
+
+    is_port_in_use(
+        neofs_env_with_mainchain.rest_gw.metrics_address.split(":")[0],
+        neofs_env_with_mainchain.rest_gw.metrics_address.split(":")[1],
+        "rest gw prometheus port is not utilized",
+    )
+
+    for sn in neofs_env_with_mainchain.storage_nodes:
+        is_port_in_use(
+            sn.pprof_address.split(":")[0], sn.pprof_address.split(":")[1], "storage node pprof port is not utilized"
+        )
+
+        is_port_in_use(
+            sn.prometheus_address.split(":")[0],
+            sn.prometheus_address.split(":")[1],
+            "storage node prometheus port is not utilized",
+        )