From 897fcd7aa0596c0cfbee427117da8d6f5e86219f Mon Sep 17 00:00:00 2001 From: Hicham El Gharbi Date: Tue, 8 Nov 2022 11:43:53 +0100 Subject: [PATCH 1/4] Add option to serve metrics as http We modified the default behaviour to expose Prometheus metrics in HTTP(using the -d flag). Without the -d flag, it behaves as it did before. We also added a few other flags to provide more flexibility. --- src/pkg_exporter/pkgmanager/apt.py | 7 ++-- src/pkg_exporter/textfile.py | 66 ++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/src/pkg_exporter/pkgmanager/apt.py b/src/pkg_exporter/pkgmanager/apt.py index 85c3b08..969911f 100644 --- a/src/pkg_exporter/pkgmanager/apt.py +++ b/src/pkg_exporter/pkgmanager/apt.py @@ -3,9 +3,10 @@ import apt.progress from pathlib import Path - class AptPkgManager: - def __init__(self): + def __init__(self, rootdir=None): + self.rootdir = rootdir + self.metricDict = {} self.metricDict["installed"] = {"description": "Installed packages per origin"} self.metricDict["upgradable"] = { @@ -21,7 +22,7 @@ def __init__(self): self.metaMetrics["update_start_time"] = 0 self.metaMetrics["update_end_time"] = 0 - self.cache = apt.Cache() + self.cache = apt.Cache(rootdir=self.rootdir) self.cache.open(None) def labelValues(self, origin): diff --git a/src/pkg_exporter/textfile.py b/src/pkg_exporter/textfile.py index e16ed4e..653590c 100755 --- a/src/pkg_exporter/textfile.py +++ b/src/pkg_exporter/textfile.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -from prometheus_client import CollectorRegistry, Gauge, write_to_textfile +from prometheus_client import CollectorRegistry, Gauge, write_to_textfile, start_http_server +from time import sleep from pkg_exporter.pkgmanager import apt from pkg_exporter import reboot +import argparse import os +import sys - -def main(): - registry = CollectorRegistry() - +def populate_registry(registry, rootdir=None): # get updates from apt - pkgmanager = apt.AptPkgManager() + pkgmanager = apt.AptPkgManager(rootdir=rootdir) # initially, check which metrics and labels are available metrics = pkgmanager.getMetricDict() @@ -50,12 +50,64 @@ def main(): rebootmanager.query() reboot_gauge.set(rebootmanager.getMetricValue()) - exporter_file = os.getenv("PKG_EXPORTER_FILE", "/var/prometheus/pkg-exporter.prom") +def write_registry_to_file(registry, exporter_file=None): + if not exporter_file: + exporter_file = os.getenv("PKG_EXPORTER_FILE", + "/var/prometheus/pkg-exporter.prom") exporter_dir = os.path.dirname(exporter_file) os.makedirs(exporter_dir, exist_ok=True) write_to_textfile(exporter_file, registry) +def serve(registry, addr, port, timewait, rootdir): + start_http_server(addr=addr, port=port, registry=registry) + while True: + sleep(timewait) + registry = CollectorRegistry() + populate_registry(registry, rootdir) + + +def processArgs(): + parser = argparse.ArgumentParser( + description="Collect metrics from apt and export it as a service") + group = parser.add_mutually_exclusive_group() + + group.add_argument("-f", "--exporter-file", + type=str, + default=os.getenv('PKG_EXPORTER_FILE', "/var/prometheus/pkg-exporter.prom"), + help="File to export, if used the content will not be served") + group.add_argument("-d", "--daemon", + action='store_true', + help="Run as daemon and server metric via http") + parser.add_argument("-a", "--bind-addr", + type=str, + default=os.getenv('PKG_EXPORTER_ADDR', '0.0.0.0'), + help="Bind address") + parser.add_argument("-p", "--port", + type=int, + default=os.getenv('PKG_EXPORTER_PORT', 8089), + help="Bind port") + parser.add_argument("-r", "--rootdir", + type=str, + default=os.getenv('PKG_EXPORTER_ROOT_DIR', None), + help="Custom root directory for dpkg") + parser.add_argument("-t", "--time-wait", + type=int, + default=os.getenv('PKG_EXPORTER_TIME_WAIT', 300), + help="time (in second) to wait between data updates") + return parser.parse_args() + +def main(): + args = processArgs() + registry = CollectorRegistry() + populate_registry(registry, args.rootdir) + + if not args.daemon: + write_registry_to_file(registry, args.exporter_file) + else: + serve(registry, args.bind_addr, args.port, args.time_wait, args.rootdir) + if __name__ == "__main__": main() + From 92052f3063f8b8aac67201afd148dda9479d8135 Mon Sep 17 00:00:00 2001 From: Hicham El Gharbi Date: Fri, 18 Nov 2022 03:35:17 +0100 Subject: [PATCH 2/4] Fix Gauge creation and registry usage Check if Gauge exists before creating it. Use the same registry during all the serve blocking process instead of recreating and repopulating it every time. --- src/pkg_exporter/textfile.py | 109 ++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 40 deletions(-) diff --git a/src/pkg_exporter/textfile.py b/src/pkg_exporter/textfile.py index 653590c..d762081 100755 --- a/src/pkg_exporter/textfile.py +++ b/src/pkg_exporter/textfile.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -from prometheus_client import CollectorRegistry, Gauge, write_to_textfile, start_http_server +from prometheus_client import Gauge, write_to_textfile, start_http_server +from prometheus_client import GC_COLLECTOR, PLATFORM_COLLECTOR, PROCESS_COLLECTOR +from prometheus_client.core import REGISTRY from time import sleep from pkg_exporter.pkgmanager import apt from pkg_exporter import reboot import argparse import os -import sys -def populate_registry(registry, rootdir=None): + +def populate_registry(rootdir=None): # get updates from apt pkgmanager = apt.AptPkgManager(rootdir=rootdir) @@ -20,20 +22,22 @@ def populate_registry(registry, rootdir=None): # also add reboot metrics rebootmanager = reboot.RebootManager() - reboot_gauge = Gauge( - "pkg_reboot_required", "Node Requires an Reboot", [], registry=registry - ) + reboot_gauge = REGISTRY._names_to_collectors.get("pkg_reboot_required", None) + if not reboot_gauge: + reboot_gauge = Gauge("pkg_reboot_required", "Node Requires an Reboot", []) # add update statistics meta_metric = pkgmanager.getMetaMetricDict() for key, value in meta_metric.items(): - meta_gauges[key] = Gauge(f"pkg_{key}", value["description"], registry=registry) + meta_gauges[key] = REGISTRY._names_to_collectors.get(f"pkg_{key}", None) + if not meta_gauges[key]: + meta_gauges[key] = Gauge(f"pkg_{key}", value["description"]) # Create all the gauge metrics for key, value in metrics.items(): - gauges[key] = Gauge( - f"pkg_{key}", value["description"], labels, registry=registry - ) + gauges[key] = REGISTRY._names_to_collectors.get(f"pkg_{key}", None) + if not gauges[key]: + gauges[key] = Gauge(f"pkg_{key}", value["description"], labels) # let the pkgmanager query its internal metrics pkgmanager.query() @@ -50,64 +54,89 @@ def populate_registry(registry, rootdir=None): rebootmanager.query() reboot_gauge.set(rebootmanager.getMetricValue()) + def write_registry_to_file(registry, exporter_file=None): if not exporter_file: - exporter_file = os.getenv("PKG_EXPORTER_FILE", - "/var/prometheus/pkg-exporter.prom") + exporter_file = os.getenv( + "PKG_EXPORTER_FILE", "/var/prometheus/pkg-exporter.prom" + ) exporter_dir = os.path.dirname(exporter_file) os.makedirs(exporter_dir, exist_ok=True) write_to_textfile(exporter_file, registry) -def serve(registry, addr, port, timewait, rootdir): - start_http_server(addr=addr, port=port, registry=registry) + +def serve(addr, port, timewait, rootdir): + start_http_server(addr=addr, port=port) while True: sleep(timewait) - registry = CollectorRegistry() - populate_registry(registry, rootdir) + populate_registry(rootdir) def processArgs(): parser = argparse.ArgumentParser( - description="Collect metrics from apt and export it as a service") + description="Collect metrics from apt and export it as a service" + ) group = parser.add_mutually_exclusive_group() - group.add_argument("-f", "--exporter-file", + group.add_argument( + "-f", + "--exporter-file", type=str, - default=os.getenv('PKG_EXPORTER_FILE', "/var/prometheus/pkg-exporter.prom"), - help="File to export, if used the content will not be served") - group.add_argument("-d", "--daemon", - action='store_true', - help="Run as daemon and server metric via http") - parser.add_argument("-a", "--bind-addr", + default=os.getenv("PKG_EXPORTER_FILE", "/var/prometheus/pkg-exporter.prom"), + help="File to export, if used the content will not be served", + ) + group.add_argument( + "-d", + "--daemon", + action="store_true", + help="Run as daemon and server metrics via http", + ) + parser.add_argument( + "-a", + "--bind-addr", type=str, - default=os.getenv('PKG_EXPORTER_ADDR', '0.0.0.0'), - help="Bind address") - parser.add_argument("-p", "--port", + default=os.getenv("PKG_EXPORTER_ADDR", "0.0.0.0"), + help="Bind address", + ) + parser.add_argument( + "-p", + "--port", type=int, - default=os.getenv('PKG_EXPORTER_PORT', 8089), - help="Bind port") - parser.add_argument("-r", "--rootdir", + default=os.getenv("PKG_EXPORTER_PORT", 8089), + help="Bind port", + ) + parser.add_argument( + "-r", + "--rootdir", type=str, - default=os.getenv('PKG_EXPORTER_ROOT_DIR', None), - help="Custom root directory for dpkg") - parser.add_argument("-t", "--time-wait", + default=os.getenv("PKG_EXPORTER_ROOT_DIR", None), + help="Custom root directory for dpkg", + ) + parser.add_argument( + "-t", + "--time-wait", type=int, - default=os.getenv('PKG_EXPORTER_TIME_WAIT', 300), - help="time (in second) to wait between data updates") + default=os.getenv("PKG_EXPORTER_TIME_WAIT", 300), + help="time (in second) to wait between data updates", + ) return parser.parse_args() + def main(): args = processArgs() - registry = CollectorRegistry() - populate_registry(registry, args.rootdir) + + REGISTRY.unregister(GC_COLLECTOR) + REGISTRY.unregister(PLATFORM_COLLECTOR) + REGISTRY.unregister(PROCESS_COLLECTOR) + + populate_registry(args.rootdir) if not args.daemon: - write_registry_to_file(registry, args.exporter_file) + write_registry_to_file(REGISTRY, args.exporter_file) else: - serve(registry, args.bind_addr, args.port, args.time_wait, args.rootdir) + serve(args.bind_addr, args.port, args.time_wait, args.rootdir) if __name__ == "__main__": main() - From 2382ade94fead593ea3ec997eeeb5f22304fc6f1 Mon Sep 17 00:00:00 2001 From: sudeephb Date: Tue, 10 Jan 2023 16:27:21 +0545 Subject: [PATCH 3/4] Update Readme to add information about CLI options --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index d7664a9..8f9adee 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,16 @@ If the directory is not already present, it will be created by the exporter. The command `pkg_exporter` provided by the package or the binary shall be executed in a appropriate interval, e.g. using cron or systemd timers. The exporter needs to be executed with appropriate privileges, which are not necessarily root privileges. +Several command line options can be used along with the `pkg_exporter` command, which are as follows: +| Option | Description | +| ------- | --------- | +| -f, --exporter-file | The file where prometheus metrics are exported, default is `/var/prometheus/pkg-exporter.prom` | +| -d, --daemon | Run as daemon and serve metrics via http. | +| -a, --bind-addr | The address to serve metrics via http. Default is `0.0.0.0` | +| -p, --port | The port to serve metrics via http. Default is `8089` | +| -r, --rootdir | Custom root directory for dpkg, if any. | +| -t, --time-wait | Time, in seconds, to wait between metrics updates. Default is 300 | + An example configuration will be provided in this repository in the future. ### apt hook From 6be809bb8f8e3b0f5493292f71920f15ec0e0d70 Mon Sep 17 00:00:00 2001 From: sudeephb Date: Thu, 19 Jan 2023 14:57:08 +0545 Subject: [PATCH 4/4] Lint fix --- src/pkg_exporter/pkgmanager/apt.py | 1 + src/pkg_exporter/textfile.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/pkg_exporter/pkgmanager/apt.py b/src/pkg_exporter/pkgmanager/apt.py index 969911f..084b76e 100644 --- a/src/pkg_exporter/pkgmanager/apt.py +++ b/src/pkg_exporter/pkgmanager/apt.py @@ -3,6 +3,7 @@ import apt.progress from pathlib import Path + class AptPkgManager: def __init__(self, rootdir=None): self.rootdir = rootdir diff --git a/src/pkg_exporter/textfile.py b/src/pkg_exporter/textfile.py index d762081..b320b8d 100755 --- a/src/pkg_exporter/textfile.py +++ b/src/pkg_exporter/textfile.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from prometheus_client import Gauge, write_to_textfile, start_http_server -from prometheus_client import GC_COLLECTOR, PLATFORM_COLLECTOR, PROCESS_COLLECTOR +from prometheus_client import GC_COLLECTOR, PLATFORM_COLLECTOR, PROCESS_COLLECTOR # noqa E501 from prometheus_client.core import REGISTRY from time import sleep from pkg_exporter.pkgmanager import apt @@ -22,14 +22,19 @@ def populate_registry(rootdir=None): # also add reboot metrics rebootmanager = reboot.RebootManager() - reboot_gauge = REGISTRY._names_to_collectors.get("pkg_reboot_required", None) + reboot_gauge = REGISTRY._names_to_collectors.get( + "pkg_reboot_required", None) if not reboot_gauge: - reboot_gauge = Gauge("pkg_reboot_required", "Node Requires an Reboot", []) + reboot_gauge = Gauge( + "pkg_reboot_required", + "Node Requires an Reboot", + []) # add update statistics meta_metric = pkgmanager.getMetaMetricDict() for key, value in meta_metric.items(): - meta_gauges[key] = REGISTRY._names_to_collectors.get(f"pkg_{key}", None) + meta_gauges[key] = REGISTRY._names_to_collectors.get( + f"pkg_{key}", None) if not meta_gauges[key]: meta_gauges[key] = Gauge(f"pkg_{key}", value["description"]) @@ -83,7 +88,9 @@ def processArgs(): "-f", "--exporter-file", type=str, - default=os.getenv("PKG_EXPORTER_FILE", "/var/prometheus/pkg-exporter.prom"), + default=os.getenv( + "PKG_EXPORTER_FILE", + "/var/prometheus/pkg-exporter.prom"), help="File to export, if used the content will not be served", ) group.add_argument(