Merge pull request #35 from Maharacha/rpc-interface

Added interface for connecting parachain nodes to relaychain nodes.

README.md

@@ -127,6 +127,15 @@ The `cos_agent` interface is already supported by this Polkadot operator charm s
 
 Find more details on how to deploy and use COS [here](https://charmhub.io/topics/canonical-observability-stack/tutorials/instrumenting-machine-charms).
 
+#### Using an external relaychain node
+
+A parachain node can use an external relaychain node instead of the internal one. It's useful for scaling where multiple parachain nodes can share a relaychain node. It's also useful to get faster in sync since the parachain node does not have to sync a relaychain node by itself. This can be set with the service argument `--relay-chain-rpc-urls`, which takes one or more weboscket URLs to relaychain nodes to use. Setting multiple URLs is for fallback where the parachain node will try accessing the URLs in a round-robin fashion. Instead of setting this manually, the interface `rpc-url` can be used:
+
+    juju relate polkadot-relay:rpc-url polkadot-para:relay-rpc-url     # Juju 2.x
+    juju integrate polkadot-relay:rpc-url polkadot-para:relay-rpc-url  # Juju 3.x
+
+Relating to multiple relaychain nodes, to have fallbacks, is supported by the interface. It's also possible to both use the `rpc-url` interface and set URLs manually in the service arguments at the same time. In the charm, the URLs from using the interface are added to the beginning of the service arguments, in the same order as they are related. Adding URLs manually should thus be considered as a fallback since as it has been mentioned, the Polkadot client selects relay chain URL in a round-robin fashion. One can for example use an external providers relaychain node as a fallback in this way, a case where it is not possible to use Juju primitives.
+
 ## Resources
 
 - [Polkadot](https://polkadot.network/)

metadata.yaml

@@ -27,3 +27,7 @@ provides:
     interface: cos_agent
   rpc-url:
     interface: rpc-url
+
+requires:
+  relay-rpc-url:
+    interface: rpc-url

src/charm.py

@@ -21,6 +21,7 @@
 
 from interface_prometheus import PrometheusProvider
 from interface_rpc_url_provider import RpcUrlProvider
+from interface_rpc_url_requirer import RpcUrlRequirer
 from polkadot_rpc_wrapper import PolkadotRpcWrapper
 import utils
 from service_args import ServiceArgs
@@ -40,6 +41,7 @@ def __init__(self, *args):
         self.prometheus_node_provider = PrometheusProvider(self, 'node-prometheus', 9100, '/metrics')
         self.prometheus_polkadot_provider = PrometheusProvider(self, 'polkadot-prometheus', 9615, '/metrics')
         self.rpc_url_provider = RpcUrlProvider(self, 'rpc_url'),
+        self.rpc_url_requirer = RpcUrlRequirer(self, 'relay_rpc_url'),
 
         self.cos_agent_provider = COSAgentProvider(
             self,
@@ -70,11 +72,12 @@ def __init__(self, *args):
 
         self._stored.set_default(binary_url=self.config.get('binary-url'),
                                  docker_tag=self.config.get('docker-tag'),
-                                 service_args=self.config.get('service-args'))
+                                 service_args=self.config.get('service-args'),
+                                 relay_rpc_urls=dict())
 
     def _on_install(self, event: ops.InstallEvent) -> None:
         self.unit.status = ops.MaintenanceStatus("Begin installing charm")
-        service_args_obj = ServiceArgs(self.config.get('service-args'))
+        service_args_obj = ServiceArgs(self.config.get('service-args'), self._stored.relay_rpc_urls)
         # Setup polkadot group and user, disable login
         utils.setup_group_and_user()
         # Create environment file for polkadot service arguments
@@ -93,7 +96,7 @@ def _on_install(self, event: ops.InstallEvent) -> None:
 
     def _on_config_changed(self, event: ops.ConfigChangedEvent) -> None:
         try:
-            service_args_obj = ServiceArgs(self.config.get('service-args'))
+            service_args_obj = ServiceArgs(self.config.get('service-args'), self._stored.relay_rpc_urls)
         except ValueError as e:
             self.unit.status = ops.BlockedStatus(str(e))
             event.defer()
@@ -124,7 +127,7 @@ def _on_update_status(self, event: ops.UpdateStatusEvent) -> None:
 
     def update_status(self, connection_attempts: int = 4) -> None:
         if utils.service_started():
-            rpc_port = ServiceArgs(self._stored.service_args).rpc_port
+            rpc_port = ServiceArgs(self._stored.service_args, self._stored.relay_rpc_urls).rpc_port
             for i in range(connection_attempts):
                 time.sleep(5)
                 try:
@@ -143,7 +146,7 @@ def update_status(self, connection_attempts: int = 4) -> None:
                     break
                 except RequestsConnectionError as e:
                     logger.warning(e)
-                    self.unit.status = ops.MaintenanceStatus("Client not responding to HTTP (attempt {}/{})".format(i, connection_attempts))
+                    self.unit.status = ops.MaintenanceStatus("Client not responding to HTTP (attempt {}/{})".format(i+1, connection_attempts))
             if type(self.unit.status) != ops.ActiveStatus:
                 self.unit.status = ops.WaitingStatus("Service running, client starting up")
         else:
@@ -155,11 +158,11 @@ def _on_start(self, event: ops.StartEvent) -> None:
 
     def _on_stop(self, event: ops.StopEvent) -> None:
         utils.stop_service()
-        self.unit.status = ops.ActiveStatus("Service stopped")
+        self.update_status()
 
     def _on_get_session_key_action(self, event: ops.ActionEvent) -> None:
         event.log("Getting new session key through rpc...")
-        rpc_port = ServiceArgs(self._stored.service_args).rpc_port
+        rpc_port = ServiceArgs(self._stored.service_args, self._stored.relay_rpc_urls).rpc_port
         key = PolkadotRpcWrapper(rpc_port).get_session_key()
         if key:
             event.set_results(results={'session-key': key})
@@ -172,7 +175,7 @@ def _on_has_session_key_action(self, event: ops.ActionEvent) -> None:
         if not re.match(keypattern, key):
             event.fail("Illegal key pattern, did your key start with 0x ?")
         else:
-            rpc_port = ServiceArgs(self._stored.service_args).rpc_port
+            rpc_port = ServiceArgs(self._stored.service_args, self._stored.relay_rpc_urls).rpc_port
             has_session_key = PolkadotRpcWrapper(rpc_port).has_session_key(key)
             event.set_results(results={'has-key': has_session_key})
 
@@ -183,32 +186,36 @@ def _on_insert_key_action(self, event: ops.ActionEvent) -> None:
         if not re.match(keypattern, address):
             event.fail("Illegal key pattern, did your public key/address start with 0x ?")
         else:
-            rpc_port = ServiceArgs(self._stored.service_args).rpc_port
+            rpc_port = ServiceArgs(self._stored.service_args, self._stored.relay_rpc_urls).rpc_port
             PolkadotRpcWrapper(rpc_port).insert_key(mnemonic, address)
 
     def _on_restart_node_service_action(self, event: ops.ActionEvent) -> None:
         utils.restart_service()
         if not utils.service_started():
             event.fail("Could not restart service")
-        self.unit.status = ops.ActiveStatus("Node service restarted")
+        event.set_results(results={'message': 'Node service restarted'})
+        self.update_status()
 
     def _on_start_node_service_action(self, event: ops.ActionEvent) -> None:
         utils.start_service()
         if not utils.service_started():
             event.fail("Could not start service")
-        self.unit.status = ops.ActiveStatus("Node service started")
+        event.set_results(results={'message': 'Node service started'})
+        self.update_status()
 
     def _on_stop_node_service_action(self, event: ops.ActionEvent) -> None:
         utils.stop_service()
         if utils.service_started(iterations=1):
             event.fail("Could not stop service")
-        self.unit.status = ops.BlockedStatus("Node service stopped")
+        event.set_results(results={'message': 'Node service stopped'})
+        self.update_status()
 
     def _on_set_node_key_action(self, event: ops.ActionEvent) -> None:
         key = event.params['key']
         utils.stop_service()
         utils.write_node_key_file(key)
         utils.start_service()
+        self.update_status()
 
     def _on_find_validator_address_action(self, event: ops.ActionEvent) -> None:
         event.log("Checking sessions key through rpc...")
@@ -279,7 +286,7 @@ def _on_get_node_info_action(self, event: ops.ActionEvent) -> None:
             event.set_results(results={'node-relay': utils.get_relay_for_parachain()})
         # On-chain info
         try:
-            rpc_port = ServiceArgs(self._stored.service_args).rpc_port
+            rpc_port = ServiceArgs(self._stored.service_args, self._stored.relay_rpc_urls).rpc_port
             block_height = PolkadotRpcWrapper(rpc_port).get_block_height()
             if block_height:
                 event.set_results(results={'chain-block-height': block_height})

src/interface_rpc_url_provider.py

@@ -4,7 +4,11 @@
 
 from service_args import ServiceArgs
 from ops.framework import Object
-from ops.charm import RelationChangedEvent
+from ops.charm import RelationJoinedEvent
+import utils
+import logging
+
+logger = logging.getLogger(__name__)
 
 class RpcUrlProvider(Object):
     """RPC url provider interface."""
@@ -17,17 +21,24 @@ def __init__(self, charm, relation_name):
             charm.on[relation_name].relation_joined, self._on_relation_joined
         )
 
-    def _on_relation_joined(self, event: RelationChangedEvent) -> None:
+    def _on_relation_joined(self, event: RelationJoinedEvent) -> None:
         """This event is used to send the ws or http rpc url to another client."""
 
-        service_args_obj = ServiceArgs(self._charm.config.get('service-args'))
-        ingress_address = event.relation.data.get(self.model.unit)['ingress-address']
-        if service_args_obj.ws_port:
-            url = f'ws://{ingress_address}:{service_args_obj.ws_port}'
-        elif service_args_obj.rpc_port:
-            url = f'http://{ingress_address}:{service_args_obj.rpc_port}'
-        else:
+        service_args_obj = ServiceArgs(self._charm.config.get('service-args'), "")
+
+        ws_port = service_args_obj.ws_port
+        rpc_port = service_args_obj.rpc_port
+        if not ws_port and not rpc_port:
             event.defer()
             return
 
-        event.relation.data[self.model.unit]['url'] = url
+        # In newer version of Polkadot the ws options are removed, and ws and http uses the same port specified by --rpc-port instead.
+        if "--ws-port" not in utils.get_client_binary_help_output():
+            logger.info(f'Using same RPC port ({rpc_port}) for websocket and http due to newer version of Polkadot.')
+            ws_port = rpc_port
+
+        ingress_address = event.relation.data.get(self.model.unit)['ingress-address']
+        if rpc_port:
+            event.relation.data[self.model.unit]['rpc_url'] = f'http://{ingress_address}:{rpc_port}'
+        if ws_port:
+            event.relation.data[self.model.unit]['ws_url'] = f'ws://{ingress_address}:{ws_port}'

src/interface_rpc_url_requirer.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+
+"""RPC url interface (requirers side)."""
+
+from service_args import ServiceArgs
+from ops.framework import Object
+from ops.charm import RelationChangedEvent, RelationDepartedEvent
+import utils
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class RpcUrlRequirer(Object):
+    """RPC url requirer interface."""
+
+    def __init__(self, charm, relation_name):
+        super().__init__(charm, relation_name)
+        self._charm = charm
+        self._relation_name = relation_name
+        self.framework.observe(
+            charm.on[relation_name].relation_changed, self._on_relation_changed
+        )
+        self.framework.observe(
+            charm.on[relation_name].relation_departed, self._on_relation_departed
+        )
+
+    def _on_relation_changed(self, event: RelationChangedEvent) -> None:
+        """This event is used to receive the Websocket RPC url from another client."""
+
+        if not event.unit in event.relation.data:
+            event.defer()
+            return
+
+        # The --relay-chain-rpc-urls option currently only supports ws, hence using ws_url and not rpc_url.
+        try:
+            ws_url = event.relation.data[event.unit]["ws_url"]
+        except KeyError:
+            logger.warning(f'Did not receive websocket URL from {event.unit} to use as a RPC endpoint.')
+            event.defer()
+            return
+        logger.info(f'Received websocket URL {ws_url} from {event.unit} to use as a RPC endpoint.')
+        # Storing the unitname+relation_id is a workaround because the relation data is already removed before the departed hook is called.
+        # This is to know which url to remove. Issue for the bug: https://github.com/canonical/operator/issues/1109
+        dict_key = event.unit.name + ':' + str(event.relation.id)
+        self._charm._stored.relay_rpc_urls[dict_key] = ws_url
+        service_args_obj = ServiceArgs(self._charm.config.get('service-args'), self._charm._stored.relay_rpc_urls)
+        utils.update_service_args(service_args_obj.service_args_string)
+        self._charm.update_status()
+
+    def _on_relation_departed(self, event: RelationDepartedEvent) -> None:
+        dict_key = event.unit.name + ':' + str(event.relation.id)
+        self._charm._stored.relay_rpc_urls.pop(dict_key)
+        service_args_obj = ServiceArgs(self._charm.config.get('service-args'), self._charm._stored.relay_rpc_urls)
+        utils.update_service_args(service_args_obj.service_args_string)
+        self._charm.update_status()

src/service_args.py

@@ -8,8 +8,9 @@
 
 class ServiceArgs():
 
-    def __init__(self, service_args: str):
+    def __init__(self, service_args: str, relay_rpc_urls: dict):
         service_args = self.__encode_for_emoji(service_args)
+        self._relay_rpc_urls = relay_rpc_urls
         self.__check_service_args(service_args)
         self.service_args_list = self.__service_args_to_list(service_args)
         self.__check_service_args(self.service_args_list)
@@ -90,6 +91,8 @@ def __add_secondchain_args(self, args: list):
 
     def __customize_service_args(self):
         self.__add_firstchain_args(['--node-key-file', utils.NODE_KEY_PATH])
+        if self._relay_rpc_urls:
+            self.__add_firstchain_args(['--relay-chain-rpc-urls'] + list(self._relay_rpc_urls.values()))
 
         if self.chain_name == 'peregrine':
             self.__peregrine()