From 7fac976d6cd2a2d3e76e41bd600341f83bfc0df1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 30 Jul 2024 23:06:41 +0000 Subject: [PATCH] Setup mtail --- CHANGELOG.md | 11 ++++ chatmaild/src/chatmaild/config.py | 1 + chatmaild/src/chatmaild/filtermail.py | 8 ++- chatmaild/src/chatmaild/ini/chatmail.ini.f | 13 ++++ cmdeploy/src/cmdeploy/__init__.py | 40 +++++++++++++ .../src/cmdeploy/mtail/delivered_mail.mtail | 59 +++++++++++++++++++ cmdeploy/src/cmdeploy/mtail/mtail.service.j2 | 10 ++++ 7 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail create mode 100644 cmdeploy/src/cmdeploy/mtail/mtail.service.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 551c29e9..e803eca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## untagged +There is a new required setting in `chatmail.ini`: `mtail_address`. +This defines the address on which [`mtail`](https://google.github.io/mtail/) +exposes its metrics collected from the logs. +If you want to collect the metrics with Prometheus, +setup a private network (e.g. WireGuard interface) +and assign an IP address from this network to the host. +If unsure, set this setting to `127.0.0.1`. + +- add mtail + ([#388](https://github.com/deltachat/chatmail/pull/388)) + - fix checking for required DNS records ([#412](https://github.com/deltachat/chatmail/pull/412)) diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index f8109520..b870d7a1 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -30,6 +30,7 @@ def __init__(self, inipath, params): self.passthrough_recipients = params["passthrough_recipients"].split() self.filtermail_smtp_port = int(params["filtermail_smtp_port"]) self.postfix_reinject_port = int(params["postfix_reinject_port"]) + self.mtail_address = params["mtail_address"] self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true" self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true" self.iroh_relay = params.get("iroh_relay") diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 140e7172..f6b32275 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -183,15 +183,21 @@ def check_DATA(self, envelope): mail_encrypted = check_encrypted(message) _, from_addr = parseaddr(message.get("from").strip()) + envelope_from_domain = from_addr.split("@").pop() + logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}") if envelope.mail_from.lower() != from_addr.lower(): return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>" + if mail_encrypted: + print("Filtering encrypted mail.", file=sys.stderr) + else: + print("Filtering unencrypted mail.", file=sys.stderr) + if envelope.mail_from in self.config.passthrough_senders: return passthrough_recipients = self.config.passthrough_recipients - envelope_from_domain = from_addr.split("@").pop() for recipient in envelope.rcpt_tos: if envelope.mail_from == recipient: # Always allow sending emails to self. diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index b2a5ff12..48b9f800 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -55,6 +55,19 @@ # if set to "True" IPv6 is disabled disable_ipv6 = False +# Address on which `mtail` listens, +# e.g. 127.0.0.1 or some private network +# address like 192.168.10.1. +# You can point Prometheus +# or some other OpenMetrics-compatible +# collector to +# http://{{mtail_address}}:3903/metrics +# and display collected metrics with Grafana. +# +# WARNING: do not expose this service +# to the public IP address. +mtail_address = 127.0.0.1 + # # Debugging options # diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index ace9e47b..5c4c2d1c 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -441,6 +441,44 @@ def check_config(config): return config +def deploy_mtail(config): + apt.packages( + name="Install mtail", + packages=["mtail"], + ) + + # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`. + # This allows to read from journalctl instead of log files. + files.template( + src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"), + dest="/etc/systemd/system/mtail.service", + user="root", + group="root", + mode="644", + address=config.mtail_address, + port=3903, + ) + + mtail_conf = files.put( + name="Mtail configuration", + src=importlib.resources.files(__package__).joinpath( + "mtail/delivered_mail.mtail" + ), + dest="/etc/mtail/delivered_mail.mtail", + user="root", + group="root", + mode="644", + ) + + systemd.service( + name="Start and enable mtail", + service="mtail.service", + running=True, + enabled=True, + restarted=mtail_conf.changed, + ) + + def deploy_chatmail(config_path: Path) -> None: """Deploy a chat-mail instance. @@ -636,3 +674,5 @@ def deploy_chatmail(config_path: Path) -> None: name="Ensure cron is installed", packages=["cron"], ) + + deploy_mtail(config) diff --git a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail new file mode 100644 index 00000000..115bb6fc --- /dev/null +++ b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail @@ -0,0 +1,59 @@ +counter delivered_mail +/saved mail to INBOX$/ { + delivered_mail++ +} + +counter quota_exceeded +/Quota exceeded \(mailbox for user is full\)$/ { + quota_exceeded++ +} + +# Essentially the number of outgoing messages. +counter dkim_signed +/DKIM-Signature field added/ { + dkim_signed++ +} + +counter created_accounts +counter created_ci_accounts +counter created_nonci_accounts + +/: Created address: (?P.*)$/ { + created_accounts++ + + $addr =~ /ci-/ { + created_ci_accounts++ + } else { + created_nonci_accounts++ + } +} + +counter postfix_timeouts +/timeout after DATA/ { + postfix_timeouts++ +} + +counter postfix_noqueue +/postfix\/.*NOQUEUE/ { + postfix_noqueue++ +} + +counter warning_count +/warning/ { + warning_count++ +} + + +counter filtered_mail_count + +counter encrypted_mail_count +/Filtering encrypted mail\./ { + encrypted_mail_count++ + filtered_mail_count++ +} + +counter unencrypted_mail_count +/Filtering unencrypted mail\./ { + unencrypted_mail_count++ + filtered_mail_count++ +} diff --git a/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 b/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 new file mode 100644 index 00000000..97d209d1 --- /dev/null +++ b/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=mtail + +[Service] +Type=simple +ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -" +Restart=on-failure + +[Install] +WantedBy=multi-user.target