Skip to content

Commit

Permalink
Merge pull request #130 from Astro1247/main
Browse files Browse the repository at this point in the history 
NAT and mangle firewall metrics collection
  • Loading branch information
@akpw
akpw authored Apr 30, 2024
2 parents 9293402 + 218f5a9 commit a1d3ff4
Show file tree
Hide file tree
Showing 2 changed files with 179 additions and 143 deletions.
152 changes: 88 additions & 64 deletions mktxp/collector/firewall_collector.py
View file
Original file line number Diff line number Diff line change
@@ -1,64 +1,88 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.


from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.firewall_ds import FirewallMetricsDataSource


class FirewallCollector(BaseCollector):
''' Firewall rules traffic metrics collector
'''
@staticmethod
def collect(router_entry):
# Initialize all pool counts, including those currently not used
# These are the same for both IPv4 and IPv6
firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log']

if router_entry.config_entry.firewall:
# ~*~*~*~*~*~ IPv4 ~*~*~*~*~*~
firewall_filter_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels)
if firewall_filter_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metrics_records, 'bytes', ['name', 'log'])
yield firewall_filter_metrics

firewall_raw_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, raw = True)
if firewall_raw_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metrics_records, 'bytes', ['name', 'log'])
yield firewall_raw_metrics

# ~*~*~*~*~*~ IPv6 ~*~*~*~*~*~
if router_entry.config_entry.ipv6_firewall:
firewall_filter_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels)
if firewall_filter_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records_ipv6]
firewall_filter_metrics_ipv6 = BaseCollector.counter_collector('firewall_filter_ipv6', 'Total amount of bytes matched by firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield firewall_filter_metrics_ipv6

firewall_raw_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, raw = True)
if firewall_raw_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records_ipv6]
firewall_raw_metrics_ipv6 = BaseCollector.counter_collector('firewall_raw_ipv6', 'Total amount of bytes matched by raw firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield firewall_raw_metrics_ipv6

# Helpers
@staticmethod
def metric_record(router_entry, firewall_record):
name = f"| {firewall_record.get('chain', ' ')} | {firewall_record.get('action', ' ')} | {firewall_record.get('comment', ' ')}"
bytes = firewall_record.get('bytes', 0)
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'name': name, 'log': firewall_record['log'], 'bytes': bytes}
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.


from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.firewall_ds import FirewallMetricsDataSource


class FirewallCollector(BaseCollector):
''' Firewall rules traffic metrics collector
'''
@staticmethod
def collect(router_entry):
# Initialize all pool counts, including those currently not used
# These are the same for both IPv4 and IPv6
firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log']

if router_entry.config_entry.firewall:
# ~*~*~*~*~*~ IPv4 ~*~*~*~*~*~
firewall_filter_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, filter_path='filter')
if firewall_filter_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metrics_records, 'bytes', ['name', 'log'])
yield firewall_filter_metrics

firewall_raw_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, filter_path='raw')
if firewall_raw_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metrics_records, 'bytes', ['name', 'log'])
yield firewall_raw_metrics

filter_nat_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, filter_path='nat')
if filter_nat_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in filter_nat_records]
filter_nat_metrics = BaseCollector.counter_collector('firewall_nat', 'Total amount of bytes matched by NAT rules', metrics_records, 'bytes', ['name', 'log'])
yield filter_nat_metrics

filter_mangle_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, filter_path='mangle')
if filter_mangle_records:
metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in filter_mangle_records]
filter_mangle_metrics = BaseCollector.counter_collector('firewall_mangle', 'Total amount of bytes matched by Mangle rules', metrics_records, 'bytes', ['name', 'log'])
yield filter_mangle_metrics

# ~*~*~*~*~*~ IPv6 ~*~*~*~*~*~
if router_entry.config_entry.ipv6_firewall:
firewall_filter_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, filter_path='filter')
if firewall_filter_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records_ipv6]
firewall_filter_metrics_ipv6 = BaseCollector.counter_collector('firewall_filter_ipv6', 'Total amount of bytes matched by firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield firewall_filter_metrics_ipv6

firewall_raw_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, filter_path='raw')
if firewall_raw_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records_ipv6]
firewall_raw_metrics_ipv6 = BaseCollector.counter_collector('firewall_raw_ipv6', 'Total amount of bytes matched by raw firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield firewall_raw_metrics_ipv6

filter_nat_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, filter_path='nat')
if filter_nat_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in filter_nat_records_ipv6]
filter_nat_metrics_ipv6 = BaseCollector.counter_collector('firewall_nat_ipv6', 'Total amount of bytes matched by NAT rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield filter_nat_metrics_ipv6

filter_mangle_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, filter_path='mangle')
if filter_mangle_records_ipv6:
metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in filter_mangle_records_ipv6]
filter_mangle_metrics_ipv6 = BaseCollector.counter_collector('firewall_mangle_ipv6', 'Total amount of bytes matched by Mangle rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log'])
yield filter_mangle_metrics_ipv6

# Helpers
@staticmethod
def metric_record(router_entry, firewall_record):
name = f"| {firewall_record.get('chain', ' ')} | {firewall_record.get('action', ' ')} | {firewall_record.get('comment', ' ')}"
bytes = firewall_record.get('bytes', 0)
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'name': name, 'log': firewall_record['log'], 'bytes': bytes}
170 changes: 91 additions & 79 deletions mktxp/datasource/firewall_ds.py
View file
Original file line number Diff line number Diff line change
@@ -1,79 +1,91 @@
# coding=utf8
# Copyright (c) 2020 Arseniy Kuznetsov
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.


from mktxp.datasource.base_ds import BaseDSProcessor
from mktxp.flow.router_entry import RouterEntry

TRANSLATION_TABLE = {
'comment': lambda value: value if value else '',
'log': lambda value: '1' if value == 'true' else '0'
}


class FirewallMetricsDataSource:
''' Firewall Metrics data provider
This datasource supports both IPv4 and IPv6
'''
@staticmethod
def _get_records(router_entry: RouterEntry, filter_path: str, args: dict, matching_only: bool = False):
"""
Get firewall records from a Mikrotik ROS device.
:param router_entry: The ROS API entry used to connect to the API
:param filter_path: The path to query the records for (e.g. /ip/firewall/filter)
:param args: A dictionary of arguments to pass to the print function used for export.
Looks like: '{'stats': '', 'all': ''}'
"""
firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', args)
if matching_only:
firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
return firewall_records

@staticmethod
def metric_records_ipv4(router_entry, *, metric_labels=None, raw=False, matching_only=True):
if metric_labels is None:
metric_labels = []
try:
filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw'
firewall_records = FirewallMetricsDataSource._get_records(
router_entry,
filter_path,
{'stats': '', 'all': ''},
matching_only=matching_only
)

return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE)
except Exception as exc:
print(
f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}'
)
return None

@staticmethod
def metric_records_ipv6(router_entry, metric_labels=None, raw=False, matching_only=True):
metric_labels = metric_labels or []
try:
filter_path = '/ipv6/firewall/filter' if not raw else '/ipv6/firewall/raw'
firewall_records = FirewallMetricsDataSource._get_records(
router_entry,
filter_path,
{'stats': ''},
matching_only=matching_only
)

return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE)
except Exception as exc:
print(
f'Error getting IPv6 firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}'
)
return None
# coding=utf8
# Copyright (c) 2020 Arseniy Kuznetsov
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.


from mktxp.datasource.base_ds import BaseDSProcessor
from mktxp.flow.router_entry import RouterEntry

TRANSLATION_TABLE = {
'comment': lambda value: value if value else '',
'log': lambda value: '1' if value == 'true' else '0'
}


class FirewallMetricsDataSource:
''' Firewall Metrics data provider
This datasource supports both IPv4 and IPv6
'''
@staticmethod
def _get_records(router_entry: RouterEntry, filter_path: str, args: dict, matching_only: bool = False):
"""
Get firewall records from a Mikrotik ROS device.
:param router_entry: The ROS API entry used to connect to the API
:param filter_path: The path to query the records for (e.g. /ip/firewall/filter)
:param args: A dictionary of arguments to pass to the print function used for export.
Looks like: '{'stats': '', 'all': ''}'
"""
firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', args)
if matching_only:
firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
return firewall_records

@staticmethod
def metric_records_ipv4(router_entry, *, metric_labels=None, matching_only=True, filter_path='filter'):
if metric_labels is None:
metric_labels = []
try:
filter_paths = {
'filter': '/ip/firewall/filter',
'raw': '/ip/firewall/raw',
'nat': '/ip/firewall/nat',
'mangle': '/ip/firewall/mangle'
}
filter_path = filter_paths[filter_path]
firewall_records = FirewallMetricsDataSource._get_records(
router_entry,
filter_path,
{'stats': '', 'all': ''},
matching_only=matching_only
)

return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE)
except Exception as exc:
print(
f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}'
)
return None

@staticmethod
def metric_records_ipv6(router_entry, metric_labels=None, matching_only=True, filter_path='filter'):
metric_labels = metric_labels or []
try:
filter_paths = {
'filter': '/ipv6/firewall/filter',
'raw': '/ipv6/firewall/raw',
'nat': '/ipv6/firewall/nat',
'mangle': '/ipv6/firewall/mangle'
}
filter_path = filter_paths[filter_path]
firewall_records = FirewallMetricsDataSource._get_records(
router_entry,
filter_path,
{'stats': ''},
matching_only=matching_only
)

return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE)
except Exception as exc:
print(
f'Error getting IPv6 firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}'
)
return None

0 comments on commit a1d3ff4

Please sign in to comment.