Merge branch 'akpw:main' into main

akpw · Mar 12, 2024 · d1a7fce · d1a7fce
2 parents 28dae30 + d3d782e
commit d1a7fce
Show file tree

Hide file tree

Showing 13 changed files with 179 additions and 37 deletions.
diff --git a/README.md b/README.md
@@ -91,6 +91,8 @@ The default configuration file comes with a sample configuration, making it easy
 
     user = True                     # Active Users metrics
     queue = True                    # Queues metrics
+
+    bgp = False                     # BGP sessions metrics
     
     remote_dhcp_entry = None        # An MKTXP entry for remote DHCP info resolution (capsman/wireless)
 
@@ -211,7 +213,7 @@ mktxp edit -i
 
 ```
 [MKTXP]
-    port = 49090                    
+    listen = '0.0.0.0:49090'         # Space separated list of socket addresses to listen to, both IPV4 and IPV6
     socket_timeout = 2
     
     initial_delay_on_failure = 120
@@ -273,7 +275,7 @@ optional arguments:
 While most of the [mktxp options](https://github.com/akpw/mktxp#getting-started) are self explanatory, some might require a bit of a context.
 
 ### Remote DHCP resolution
-When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readibility / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option:
+When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readability / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option:
 ```
 remote_dhcp_entry = None        # An MKTXP entry for remote DHCP info resolution in capsman/wireless
 ```
@@ -321,11 +323,13 @@ total_max_scrape_duration = 30      # Max overall duration of all metrics collec
 To keeps things within expected boundaries, the last two parameters allows for controlling both individual and overall scrape durations
 
 
-### mktxp port
-By default, mktxp runs it's HTTP metrics endpoint on port 49090. You can change it via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration):
+### mktxp endpoint listen addresses
+By default, mktxp runs it's HTTP metrics endpoint on any IPv4 address on port 49090. However, it is also able to listen on multiple socket addresses, both IPv4 and IPv6. 
+You can configure this behaviour via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration), setting ```listen``` to a space-separated list of sockets to listen to, e.g.:
 ```
-port = 49090 
+listen = '0.0.0.0:49090 [::1]:49090'
 ```
+A wildcard for the hostname is supported as well, and binding to both IPv4/IPv6 as available.
 
 ## Setting up MKTXP to run as a Linux Service
 If you've installed MKTXP on a Linux system, you can run it with system boot via adding a service. \

diff --git a/mktxp/cli/checks/chk_pv.py b/mktxp/cli/checks/chk_pv.py
@@ -24,20 +24,20 @@ def check_version():
         print(\
         '''
         Mikrotik Prometheus Exporter requires
-                           Python version 3.6 or later.
+                           Python version 3.8 or later.
 
-        You can create an isolated Python 3.6 environment
+        You can create an isolated Python 3.8 environment
         with the virtualenv tool:
           http://docs.python-guide.org/en/latest/dev/virtualenvs
 
         ''')
         sys.exit(0)
-    elif sys.version_info.major == 3 and sys.version_info.minor < 6:
+    elif sys.version_info.major == 3 and sys.version_info.minor < 8:
         print(\
         '''
 
         Mikrotik Prometheus Exporter requires
-                            Python version 3.6 or later.
+                            Python version 3.8 or later.
 
         Please upgrade to the latest Python 3.x version.
 

diff --git a/mktxp/cli/config/_mktxp.conf b/mktxp/cli/config/_mktxp.conf
@@ -12,15 +12,15 @@
 
 
 [MKTXP]
-    port = 49090
+    listen = '0.0.0.0:49090'         # Space separated list of socket addresses to listen to, both IPV4 and IPV6
     socket_timeout = 2
 
     initial_delay_on_failure = 120
     max_delay_on_failure = 900
     delay_inc_div = 5
 
     bandwidth = False                # Turns metrics bandwidth metrics collection on / off    
-    bandwidth_test_interval = 600    # Interval for colllecting bandwidth metrics
+    bandwidth_test_interval = 600    # Interval for collecting bandwidth metrics
     minimal_collect_interval = 5     # Minimal metric collection interval
 
     verbose_mode = False             # Set it on for troubleshooting

diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py
@@ -46,6 +46,7 @@ class CollectorKeys:
     QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector'
     KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector'
     USER_COLLECTOR = 'UserCollector'
+    BGP_COLLECTOR = 'BGPCollector'
     MKTXP_COLLECTOR = 'MKTXPCollector'
 
 
@@ -56,6 +57,7 @@ class MKTXPConfigKeys:
     ENABLED_KEY = 'enabled'
     HOST_KEY = 'hostname'
     PORT_KEY = 'port'
+    LISTEN_KEY = 'listen'
     USER_KEY = 'username'
     PASSWD_KEY = 'password'
 
@@ -87,6 +89,8 @@ class MKTXPConfigKeys:
 
     FE_USER_KEY = 'user'
     FE_QUEUE_KEY = 'queue'
+    FE_BGP_KEY = 'bgp'
+
     FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry'
 
     FE_CHECK_FOR_UPDATES = 'check_for_updates'
@@ -133,7 +137,7 @@ class MKTXPConfigKeys:
 
 
     BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE,
-                       SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY}
+                       SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY}
 
     # Feature keys enabled by default
     BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -150,7 +154,7 @@ class MKTXPConfigKeys:
                       MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
                       MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
 
-    # MKTXP config entry nane
+    # MKTXP config entry name
     MKTXP_CONFIG_ENTRY_NAME = 'MKTXP'
 
 
@@ -162,9 +166,9 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
                                                        MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
                                                        MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
-                                                       MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE,
+                                                       MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_BGP_KEY,
                                                        ])
-    MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
+    MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
                                                        MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
                                                        MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_KEY,
                                                        MKTXPConfigKeys.MKTXP_VERBOSE_MODE, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL,
@@ -288,10 +292,10 @@ def system_entry(self):
     def _read_from_disk(self):
         ''' (Force-)Read conf data from disk
         '''
-        self.config = ConfigObj(self.usr_conf_data_path)
+        self.config = ConfigObj(self.usr_conf_data_path, indent_type = '    ')
         self.config.preserve_comments = True
 
-        self._config = ConfigObj(self.mktxp_conf_path)
+        self._config = ConfigObj(self.mktxp_conf_path, indent_type = '    ')
         self._config.preserve_comments = True
 
     def _create_os_path(self, os_path, resource_path):
@@ -360,7 +364,8 @@ def _system_entry_reader(self):
                 system_entry_reader[key] = self._config[entry_name].as_int(key)
             else:
                 system_entry_reader[key] = self._default_value_for_key(key)
-                new_keys.append(key) # read from disk next time
+                if key not in (MKTXPConfigKeys.PORT_KEY):  # Port key has been depricated
+                    new_keys.append(key) # read from disk next time
 
         for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES):
             if self._config[entry_name].get(key) is not None:
@@ -369,9 +374,17 @@ def _system_entry_reader(self):
                 system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False
                 new_keys.append(key) # read from disk next time
 
+        # listen 
+        if self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY):
+            system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY)
+        else:
+            system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = f'0.0.0.0:{system_entry_reader[MKTXPConfigKeys.PORT_KEY]}'
+            new_keys.append(MKTXPConfigKeys.LISTEN_KEY) # read from disk next time
+
         if new_keys:
             self._config[entry_name] = system_entry_reader
             try:
+                self._config[entry_name].pop(MKTXPConfigKeys.PORT_KEY, None) # Port key has been depricated
                 self._config.write()
                 if self._config[entry_name].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE):
                     print(f'Updated system entry {entry_name} with new system keys {new_keys}')    

diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf
@@ -52,6 +52,8 @@
 
     user = True                     # Active Users metrics
     queue = True                    # Queues metrics
+
+    bgp = False                     # BGP sessions metrics
 
     remote_dhcp_entry = None        # An MKTXP entry for remote DHCP info resolution (capsman/wireless)
 

diff --git a/mktxp/collector/bgp_collector.py b/mktxp/collector/bgp_collector.py
@@ -0,0 +1,80 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.flow.processor.output import BaseOutputProcessor
+from mktxp.datasource.bgp_ds import BGPMetricsDataSource
+
+
+class BGPCollector(BaseCollector):
+    '''BGP collector'''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.bgp:
+            return
+
+        bgp_labels = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi', 'remote_messages', 'remote_bytes', 'local_messages', 'local_bytes', 'prefix_count', 'established', 'uptime']
+        bgp_records = BGPMetricsDataSource.metric_records(router_entry, metric_labels=bgp_labels)
+
+
+        if bgp_records:
+            # translate records to appropriate values
+            translated_fields = ['established', 'uptime']        
+            for bgp_record in bgp_records:
+                for translated_field in translated_fields:
+                    value = bgp_record.get(translated_field, None)    
+                    if value:            
+                        bgp_record[translated_field] = BGPCollector._translated_values(translated_field, value)
+
+            session_info_labes = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi']
+            bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_info_labes)
+            yield bgp_sessions_metrics
+
+            session_id_labes = ['name']
+            remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote_messages', session_id_labes)
+            yield remote_messages_metrics
+
+
+            local_messages_metrics = BaseCollector.counter_collector('bgp_local_messages', 'Number of local messages', bgp_records, 'local_messages', session_id_labes)
+            yield local_messages_metrics
+
+
+            remote_bytes_metrics = BaseCollector.counter_collector('bgp_remote_bytes', 'Number of remote bytes', bgp_records, 'remote_bytes', session_id_labes)
+            yield remote_bytes_metrics
+
+
+            local_bytes_metrics = BaseCollector.counter_collector('bgp_local_bytes', 'Number of local bytes', bgp_records, 'local_bytes', session_id_labes)
+            yield local_bytes_metrics
+
+
+            prefix_count_metrics = BaseCollector.gauge_collector('bgp_prefix_count', 'BGP prefix count', bgp_records, 'prefix_count', session_id_labes)
+            yield prefix_count_metrics
+
+
+            established_metrics = BaseCollector.gauge_collector('bgp_established', 'BGP established', bgp_records, 'established', session_id_labes)
+            yield established_metrics
+
+
+            uptime_metrics = BaseCollector.gauge_collector('bgp_uptime', 'BGP uptime in milliseconds', bgp_records, 'uptime', session_id_labes)
+            yield uptime_metrics
+
+
+    # Helpers
+    @staticmethod
+    def _translated_values(translated_field, value):
+        return {
+                'established': lambda value: '1' if value=='true' else '0',
+                'uptime': lambda value: BaseOutputProcessor.parse_timedelta_milliseconds(value)
+                }[translated_field](value)
+
diff --git a/mktxp/datasource/base_ds.py b/mktxp/datasource/base_ds.py
@@ -23,15 +23,14 @@ def trimmed_records(router_entry, *, router_records = None, metric_labels = None
         if metric_labels is None:
             metric_labels = []   
         if translation_table is None:
-            translation_table = {}                 
-        dash2_ = lambda x : x.replace('-', '_')
+            translation_table = {}         
         if len(metric_labels) == 0 and len(router_records) > 0:
-            metric_labels = [dash2_(key) for key in router_records[0].keys()]
+            metric_labels = [BaseDSProcessor._normalise_keys(key) for key in router_records[0].keys()]
         metric_labels = set(metric_labels)      
 
         labeled_records = []
         for router_record in router_records:
-            translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
+            translated_record = {BaseDSProcessor._normalise_keys(key): value for (key, value) in router_record.items() if BaseDSProcessor._normalise_keys(key) in metric_labels}
 
             if add_router_id:
                 for key, value in router_entry.router_id.items():
@@ -43,3 +42,13 @@ def trimmed_records(router_entry, *, router_records = None, metric_labels = None
             labeled_records.append(translated_record)
 
         return labeled_records
+
+
+    @staticmethod
+    def _normalise_keys(key):
+        chars = ".-"
+        for chr in chars:
+            if chr in key:
+                key = key.replace(chr, "_")     
+        return key
+
diff --git a/mktxp/datasource/bgp_ds.py b/mktxp/datasource/bgp_ds.py
@@ -0,0 +1,31 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class BGPMetricsDataSource:
+    ''' Wireless Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
+        if metric_labels is None:
+            metric_labels = []                
+        try:
+            bgp_records = router_entry.api_connection.router_api().get_resource('/routing/bgp/session').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = bgp_records, metric_labels = metric_labels, add_router_id = add_router_id)
+        except Exception as exc:
+            print(f'Error getting BGP sessions info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py
@@ -37,6 +37,7 @@
 from mktxp.collector.queue_collector import QueueTreeCollector
 from mktxp.collector.queue_collector import QueueSimpleCollector
 from mktxp.collector.kid_control_device_collector import KidDeviceCollector
+from mktxp.collector.bgp_collector import BGPCollector
 
 class CollectorRegistry:
     ''' MKTXP Collectors Registry
@@ -74,6 +75,8 @@ def __init__(self):
         self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect)
 
         self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect)
+        self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect)
+
 
         self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)
 

diff --git a/mktxp/flow/processor/base_proc.py b/mktxp/flow/processor/base_proc.py
@@ -12,10 +12,9 @@
 ## GNU General Public License for more details.
 
 
-from http.server import HTTPServer
 from datetime import datetime
 from prometheus_client.core import REGISTRY
-from prometheus_client import MetricsHandler
+from prometheus_client import make_wsgi_app
 
 from mktxp.cli.config.config import config_handler
 from mktxp.flow.collector_handler import CollectorHandler
@@ -27,23 +26,18 @@
 from mktxp.cli.output.dhcp_out import DHCPOutput
 from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput
 
+from waitress import serve
+
 
 class ExportProcessor:
     ''' Base Export Processing
     '''    
     @staticmethod
     def start():
         REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
-        ExportProcessor.run(port=config_handler.system_entry().port)
-
-    @staticmethod
-    def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None):
-        server_address = ('', port)
-        httpd = server_class(server_address, handler_class)
         current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{current_time} Running HTTP metrics server on port {port}')
-        httpd.serve_forever()
-
+        print(f'{current_time} Running HTTP metrics server on: {config_handler.system_entry().listen}')
+        serve(make_wsgi_app(), listen = config_handler.system_entry().listen)
 
 class OutputProcessor:
     ''' Base CLI Processing