Add initial support for EoIP, GRE and IPIP

mktxp/cli/config/config.py

@@ -49,6 +49,9 @@ class CollectorKeys:
     USER_COLLECTOR = 'UserCollector'
     BGP_COLLECTOR = 'BGPCollector'
     ROUTING_STATS_COLLECTOR = 'RoutingStatsCollector'
+    EOIP_COLLECTOR = 'EOIPCollector'
+    GRE_COLLECTOR = 'GRECollector'
+    IPIP_COLLECTOR = 'IPIPCollector'
     IPSEC_COLLECTOR = 'IPSecCollector'
     LTE_COLLECTOR = 'LTECollector'
     SWITCH_PORT_COLLECTOR = 'SwitchPortCollector'
@@ -99,6 +102,9 @@ class MKTXPConfigKeys:
     FE_PUBLIC_IP_KEY = 'public_ip'
     FE_NETWATCH_KEY = 'netwatch'
 
+    FE_EOIP_KEY = 'eoip'
+    FE_GRE_KEY = 'gre'
+    FE_IPIP_KEY = 'ipip'
     FE_IPSEC_KEY = 'ipsec'
     FE_LTE_KEY = "lte"
     FE_SWITCH_PORT_KEY = "switch_port"
@@ -164,7 +170,7 @@ class MKTXPConfigKeys:
 
     BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, FE_KID_CONTROL_DYNAMIC,
                        SSL_CERTIFICATE_VERIFY, FE_IPV6_ROUTE_KEY, FE_IPV6_DHCP_POOL_KEY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY,
-                       FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY, FE_CERTIFICATE_KEY, FE_DNS_KEY}
+                       FE_EOIP_KEY, FE_GRE_KEY, FE_IPIP_KEY, FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY, FE_CERTIFICATE_KEY, FE_DNS_KEY}
 
     # Feature keys enabled by default
     BOOLEAN_KEYS_YES = {PLAINTEXT_LOGIN_KEY, FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, 
@@ -197,7 +203,7 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY, MKTXPConfigKeys.FE_DNS_KEY,
                                                        MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,                                               
                                                        MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_BGP_KEY,
-                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, 
+                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_EOIP_KEY, MKTXPConfigKeys.FE_GRE_KEY, MKTXPConfigKeys.FE_IPIP_KEY, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY,
                                                        MKTXPConfigKeys.FE_ROUTING_STATS_KEY, MKTXPConfigKeys.FE_CERTIFICATE_KEY
                                                        ])
     MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,

mktxp/cli/config/mktxp.conf

@@ -59,6 +59,9 @@
     capsman = True                  # CAPsMAN general metrics
     capsman_clients = True          # CAPsMAN clients metrics
 
+    eoip = False                    # EoIP status metrics
+    gre = False                     # GRE status metrics
+    ipip = False                    # IPIP status metrics
     lte = False                     # LTE signal and status metrics (requires additional 'test' permission policy on RouterOS v6)
     ipsec = False                   # IPSec active peer metrics
     switch_port = False             # Switch Port metrics

mktxp/collector/eoip_collector.py

@@ -0,0 +1,56 @@
+# coding=utf8
+## Copyright (c) 2024 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.interface_ds import InterfaceMetricsDataSource
+from mktxp.utils.utils import parse_mkt_uptime
+
+
+class EOIPCollector(BaseCollector):
+    ''' EoIP Metrics collector
+    '''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.eoip:
+            return
+
+        default_labels = ['name', 'local_address', 'remote_address', 'tunnel_id']
+        translation_table = {
+            'running': lambda value: '1' if value == 'true' else '0',
+            'disabled': lambda value: '1' if value == 'true' else '0'
+        }
+        monitor_records = InterfaceMetricsDataSource.metric_records(
+            router_entry,
+            kind='eoip',
+            additional_proplist=['actual-mtu', 'l2mtu', 'local-address', 'mtu', 'remote-address', 'tunnel-id'],
+            translation_table=translation_table
+        )
+
+        if monitor_records:
+            yield BaseCollector.gauge_collector('interface_status',
+                                                'Current status of the interface',
+                                                monitor_records,
+                                                metric_key='running',
+                                                metric_labels=default_labels + ['disabled'])
+
+            yield BaseCollector.gauge_collector('interface_l2mtu',
+                                                'Current used layer 2 mtu for this interface',
+                                                monitor_records,
+                                                metric_key='actual_mtu',
+                                                metric_labels=default_labels)
+
+            yield BaseCollector.gauge_collector('interface_mtu',
+                                                'Current used mut for this interface',
+                                                monitor_records,
+                                                metric_key='actual_mtu',
+                                                metric_labels=default_labels + ['mtu'])

mktxp/collector/gre_collector.py

@@ -0,0 +1,50 @@
+# coding=utf8
+## Copyright (c) 2024 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.interface_ds import InterfaceMetricsDataSource
+from mktxp.utils.utils import parse_mkt_uptime
+
+
+class GRECollector(BaseCollector):
+    ''' GRE Metrics collector
+    '''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.gre:
+            return
+
+        default_labels = ['name', 'local_address', 'remote_address']
+        translation_table = {
+            'running': lambda value: '1' if value == 'true' else '0',
+            'disabled': lambda value: '1' if value == 'true' else '0'
+        }
+        monitor_records = InterfaceMetricsDataSource.metric_records(
+            router_entry,
+            kind='gre',
+            additional_proplist=['mtu', 'actual-mtu', 'local-address', 'remote-address'],
+            translation_table=translation_table
+        )
+
+        if monitor_records:
+            yield BaseCollector.gauge_collector('interface_status',
+                                                'Current status of the interface',
+                                                monitor_records,
+                                                metric_key='running',
+                                                metric_labels=default_labels + ['disabled'])
+
+            yield BaseCollector.gauge_collector('interface_mtu',
+                                                'Current used MTU for this interface',
+                                                monitor_records,
+                                                metric_key='actual_mtu',
+                                                metric_labels=default_labels + ['mtu'])

mktxp/collector/ipip_collector.py

@@ -0,0 +1,50 @@
+# coding=utf8
+## Copyright (c) 2024 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.interface_ds import InterfaceMetricsDataSource
+from mktxp.utils.utils import parse_mkt_uptime
+
+
+class IPIPCollector(BaseCollector):
+    ''' IPIP Metrics collector
+    '''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.ipip:
+            return
+
+        default_labels = ['name', 'local_address', 'remote_address']
+        translation_table = {
+            'running': lambda value: '1' if value == 'true' else '0',
+            'disabled': lambda value: '1' if value == 'true' else '0'
+        }
+        interface_records = InterfaceMetricsDataSource.metric_records(
+            router_entry,
+            kind='ipip',
+            additional_proplist=['mtu', 'actual-mtu', 'local-address', 'remote-address'],
+            translation_table=translation_table
+        )
+
+        if interface_records:
+            yield BaseCollector.gauge_collector('interface_status',
+                                                'Current status of the interface',
+                                                interface_records,
+                                                metric_key='running',
+                                                metric_labels=default_labels + ['disabled'])
+
+            yield BaseCollector.gauge_collector('interface_mtu',
+                                                'Current used MTU for this interface',
+                                                interface_records,
+                                                metric_key='actual_mtu',
+                                                metric_labels=default_labels + ['mtu'])

mktxp/datasource/interface_ds.py

@@ -17,6 +17,38 @@
 from mktxp.utils.utils import routerOS7_version
 
 
+class InterfaceMetricsDataSource:
+    ''' Interface Monitor Metrics data provider
+    '''
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = None, kind = 'ethernet', additional_proplist=None, translation_table=None):
+        if metric_labels is None:
+            metric_labels = []
+
+        if additional_proplist is None:
+            additional_proplist = []
+
+        call_params = {
+            'proplist': ','.join(['name', 'running', 'disabled'] + additional_proplist)
+        }
+
+        try:
+
+            interface_records = router_entry.api_connection.router_api().get_resource(
+                f'/interface/{kind}'
+            ).call(
+                'print',
+                call_params
+            )
+            return BaseDSProcessor.trimmed_records(router_entry,
+                                                   router_records=interface_records,
+                                                   metric_labels=metric_labels,
+                                                   translation_table=translation_table)
+        except Exception as exc:
+            print(f'Error getting {kind} interface info from router {router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
 class InterfaceTrafficMetricsDataSource:
     ''' Interface Traffic Metrics data provider
     '''

mktxp/flow/collector_registry.py

@@ -40,6 +40,9 @@
 from mktxp.collector.kid_control_device_collector import KidDeviceCollector
 from mktxp.collector.bgp_collector import BGPCollector
 from mktxp.collector.routing_stats_collector import RoutingStatsCollector
+from mktxp.collector.eoip_collector import EOIPCollector
+from mktxp.collector.gre_collector import GRECollector
+from mktxp.collector.ipip_collector import IPIPCollector
 from mktxp.collector.lte_collector import LTECollector
 from mktxp.collector.switch_collector import SwitchPortCollector
 from mktxp.collector.certificate_collector import CertificateCollector
@@ -84,6 +87,9 @@ def __init__(self):
 
         self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect)
         self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect)
+        self.register(CollectorKeys.EOIP_COLLECTOR, EOIPCollector.collect)
+        self.register(CollectorKeys.GRE_COLLECTOR, GRECollector.collect)
+        self.register(CollectorKeys.IPIP_COLLECTOR, IPIPCollector.collect)
         self.register(CollectorKeys.LTE_COLLECTOR, LTECollector.collect)
         self.register(CollectorKeys.SWITCH_PORT_COLLECTOR, SwitchPortCollector.collect)
 

mktxp/flow/router_entry.py

@@ -75,6 +75,9 @@ def __init__(self, router_name):
                             CollectorKeys.USER_COLLECTOR: 0,
                             CollectorKeys.BGP_COLLECTOR: 0,
                             CollectorKeys.ROUTING_STATS_COLLECTOR: 0,
+                            CollectorKeys.EOIP_COLLECTOR: 0,
+                            CollectorKeys.GRE_COLLECTOR: 0,
+                            CollectorKeys.IPIP_COLLECTOR: 0,
                             CollectorKeys.LTE_COLLECTOR: 0,
                             CollectorKeys.SWITCH_PORT_COLLECTOR: 0,
                             CollectorKeys.MKTXP_COLLECTOR: 0,