From 357162d2620b15271f21a2a6110af9bd26b34f39 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 16:44:21 +0200 Subject: [PATCH 01/13] Path /etc/crowdsec is no longer used --- imageroot/actions/destroy-module/75remove-crowdsec-wrapper | 2 -- 1 file changed, 2 deletions(-) diff --git a/imageroot/actions/destroy-module/75remove-crowdsec-wrapper b/imageroot/actions/destroy-module/75remove-crowdsec-wrapper index 1c21fc5..3a1b228 100755 --- a/imageroot/actions/destroy-module/75remove-crowdsec-wrapper +++ b/imageroot/actions/destroy-module/75remove-crowdsec-wrapper @@ -10,5 +10,3 @@ set -e # remove the wrapper to use the container rm -vf /usr/local/sbin/cscli -# remove residual config -rm -rvf /etc/crowdsec From ab16cb36ba5077361ad5ce67f5731fdd05d3e3e4 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 16:50:59 +0200 Subject: [PATCH 02/13] Avoid -t flag in actions Do not attach a terminal, otherwise output can contain terminal sequences. --- imageroot/actions/create-module/10initialize | 12 ++++++------ .../actions/create-module/35register-local-bouncer | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/imageroot/actions/create-module/10initialize b/imageroot/actions/create-module/10initialize index ca2f015..a166581 100755 --- a/imageroot/actions/create-module/10initialize +++ b/imageroot/actions/create-module/10initialize @@ -20,8 +20,8 @@ Requisite=${MODULE_ID}.service [Service] Type=oneshot -ExecStart=/usr/bin/podman exec -ti ${MODULE_ID} cscli hub update -ExecStart=/usr/bin/podman exec -ti ${MODULE_ID} cscli hub upgrade +ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub update +ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub upgrade SyslogIdentifier=%N EOF install -v -m 644 "${tmpfile}" /etc/systemd/system/${MODULE_ID}-upgrade-hub.service @@ -47,10 +47,10 @@ systemctl enable --now "${MODULE_ID}.service" systemctl enable --now "${MODULE_ID}-upgrade-hub.timer" # Install default collections -podman exec -ti ${MODULE_ID} cscli hub update -podman exec -ti ${MODULE_ID} cscli hub upgrade +podman exec -i ${MODULE_ID} cscli hub update +podman exec -i ${MODULE_ID} cscli hub upgrade -podman exec -ti ${MODULE_ID} cscli collections install \ +podman exec -i ${MODULE_ID} cscli collections install \ crowdsecurity/apache2 \ crowdsecurity/base-http-scenarios \ crowdsecurity/dovecot \ @@ -70,4 +70,4 @@ podman exec -ti ${MODULE_ID} cscli collections install \ crowdsecurity/whitelist-good-actors # we need it if we want to ban with IP from country -podman exec -ti ${MODULE_ID} cscli parsers install crowdsecurity/geoip-enrich +podman exec -i ${MODULE_ID} cscli parsers install crowdsecurity/geoip-enrich diff --git a/imageroot/actions/create-module/35register-local-bouncer b/imageroot/actions/create-module/35register-local-bouncer index 79e6522..53da010 100755 --- a/imageroot/actions/create-module/35register-local-bouncer +++ b/imageroot/actions/create-module/35register-local-bouncer @@ -10,4 +10,4 @@ set -e echo "Register the localhost bouncer" secret=$(cat secrets/bouncer_keys_firewall.secret) -podman exec -ti ${MODULE_ID} cscli bouncers add localhost -k "$secret" +podman exec -i ${MODULE_ID} cscli bouncers add localhost -k "$secret" From d88a251332e1e82bf9121fb0cb516ef9487b573e Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 17:11:47 +0200 Subject: [PATCH 03/13] Run cscli under the agent environment Avoid installing app executables under the system dirs. --- README.md | 15 +++++++++++---- .../actions/create-module/70crowdsec-wrapper | 19 ------------------- .../destroy-module/75remove-crowdsec-wrapper | 12 ------------ imageroot/bin/cscli | 12 ++++++++++++ 4 files changed, 23 insertions(+), 35 deletions(-) delete mode 100755 imageroot/actions/create-module/70crowdsec-wrapper delete mode 100755 imageroot/actions/destroy-module/75remove-crowdsec-wrapper create mode 100755 imageroot/bin/cscli diff --git a/README.md b/README.md index b1034a2..dd2259c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Display the configuration By default whitelist is enabled to never ban IP on the local network, for test purpose you could disable it - cscli parsers remove crowdsecurity/whitelists + runagent -m crowdsec1 cscli parsers remove crowdsecurity/whitelists systemctl restart crowdsec1 ## List Banned IP in nftables sets @@ -86,8 +86,15 @@ Banned IP are contained inside nft sets that you can list by the command line be ### cscli -crowdsec come with a cli, do `cscli --help`, if you want to know on a specific command `cscli --help` +Crowdsec come with a cli tool, available within the application environment. Get a shell with: + runagent -m crowdsec1 bash -l + +Then run the tool as + + cscli --help + +- if you want to know on a specific command `cscli --help` - get a glance : `cscli metrics` - see the state of installed bouncers : `cscli bouncers list` - see the active decisions(ban): `cscli decisions list` @@ -111,12 +118,12 @@ crowdsec come with a cli, do `cscli --help`, if you want to know on a specific c You can see the metrics of crowdsec at https://app.crowdsec.net/, for this purpose you need to create a login for a single user or an organization in the website, then in the top right menu click in `enroll an instance` and retrieve the keys, then enroll your container and restart it. - cscli console enroll + runagent -m crowdsec1 cscli console enroll systemctl restart crowdsec1 you can force the enrollment with another key - cscli console enroll --overwrite + runagent -m crowdsec1 cscli console enroll --overwrite systemctl restart crowdsec1 Once done you need to accept inside the website the `Instance enroll request` diff --git a/imageroot/actions/create-module/70crowdsec-wrapper b/imageroot/actions/create-module/70crowdsec-wrapper deleted file mode 100755 index d8467c0..0000000 --- a/imageroot/actions/create-module/70crowdsec-wrapper +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# -exec 1>&2 # never generate action output. Write to stderr instead. -set -e # stop at the first error - -# Create a wrapper to use the container : cscli metrics - -tmpfile=$(mktemp) -trap "rm -f \${tmpfile}" EXIT -cat <${tmpfile} -#!/bin/bash - -podman exec -ti ${MODULE_ID} cscli "\${@}" -EOF -install -v -m 0755 "${tmpfile}" /usr/local/sbin/cscli diff --git a/imageroot/actions/destroy-module/75remove-crowdsec-wrapper b/imageroot/actions/destroy-module/75remove-crowdsec-wrapper deleted file mode 100755 index 3a1b228..0000000 --- a/imageroot/actions/destroy-module/75remove-crowdsec-wrapper +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -exec 1>&2 # Send any output to stderr, to not alter the action response protocol -set -e - -# remove the wrapper to use the container -rm -vf /usr/local/sbin/cscli diff --git a/imageroot/bin/cscli b/imageroot/bin/cscli new file mode 100755 index 0000000..55e29ef --- /dev/null +++ b/imageroot/bin/cscli @@ -0,0 +1,12 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +if [[ -t 1 ]]; then + with_tty=1 +fi + +exec podman exec -${with_tty:+t}i "${MODULE_ID}" cscli "${@}" From 43879ff6079de8f3d4e6aa51a848567961ff91b6 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 17:16:26 +0200 Subject: [PATCH 04/13] Remove additional files on cleanup - system-wide cscli wrapper - /etc/crowdsec orphan directory --- README.md | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index dd2259c..f3f0936 100644 --- a/README.md +++ b/README.md @@ -75,15 +75,6 @@ By default whitelist is enabled to never ban IP on the local network, for test p runagent -m crowdsec1 cscli parsers remove crowdsecurity/whitelists systemctl restart crowdsec1 -## List Banned IP in nftables sets - -Banned IP are contained inside nft sets that you can list by the command line below - -- ipv4 -`nft list set ip crowdsec crowdsec-blacklists` -- ipv6 -`nft list set ip6 crowdsec6 crowdsec6-blacklists` - ### cscli Crowdsec come with a cli tool, available within the application environment. Get a shell with: @@ -94,7 +85,7 @@ Then run the tool as cscli --help -- if you want to know on a specific command `cscli --help` +- help on a specific command: `cscli --help` - get a glance : `cscli metrics` - see the state of installed bouncers : `cscli bouncers list` - see the active decisions(ban): `cscli decisions list` @@ -134,29 +125,31 @@ To uninstall the instance: remove-module --no-preserve crowdsec1 -## Uninstall the crowdsec binary bouncer +## Uninstall the old crowdsec binary bouncer Previous to the version 1.0.6 the bouncer was installed on the host following a repository method, after this version the bouncer is shipped in a full container. With the upgrade the service `crowdsec-firewall-bouncer` has been stopped but not removed from the host. For a full cleaning you can - remove firewalld permanent sets: - `firewall-cmd --permanent --delete-ipset=crowdsec-blacklists` - `firewall-cmd --permanent --delete-ipset=crowdsec6-blacklists` + + firewall-cmd --permanent --delete-ipset=crowdsec-blacklists + firewall-cmd --permanent --delete-ipset=crowdsec6-blacklists - remove the bouncer on rocky linux - `dnf remove -y crowdsec-firewall-bouncer-iptables` - `rm /etc/yum.repos.d/crowdsec_crowdsec.repo` + + dnf remove -y crowdsec-firewall-bouncer-iptables + rm -rvf /etc/yum.repos.d/crowdsec_crowdsec.repo /etc/crowdsec /usr/local/sbin/cscli - remove the bouncer on debian - `apt-get -y remove crowdsec-firewall-bouncer-iptables` - `rm /etc/apt/sources.list.d/crowdsec_crowdsec.list` + + apt-get -y remove crowdsec-firewall-bouncer-iptables + rm -rvf /etc/apt/sources.list.d/crowdsec_crowdsec.list /etc/crowdsec /usr/local/sbin/cscli ## Testing Test the module using the `test-module.sh` script: - ./test-module.sh ghcr.io/nethserver/crowdsec:latest The tests are made using [Robot Framework](https://robotframework.org/) @@ -168,4 +161,4 @@ Translated with [Weblate](https://hosted.weblate.org/projects/ns8/). To setup the translation process: - add [GitHub Weblate app](https://docs.weblate.org/en/latest/admin/continuous.html#github-setup) to your repository -- add your repository to [hosted.weblate.org]((https://hosted.weblate.org) or ask a NethServer developer to add it to ns8 Weblate project +- add your repository to [hosted.weblate.org](https://hosted.weblate.org) or ask a NethServer developer to add it to ns8 Weblate project From 961da21005dce9912c000e14c7206136d5e6b424 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 17:19:53 +0200 Subject: [PATCH 05/13] Remove unused code 1. The ExecReload hook is not used. 2. The implementation cannot work because the configuration file is copied in the container only when it is restarted. --- imageroot/crowdsec-firewall-bouncer.service | 2 -- 1 file changed, 2 deletions(-) diff --git a/imageroot/crowdsec-firewall-bouncer.service b/imageroot/crowdsec-firewall-bouncer.service index 549481c..bff3556 100644 --- a/imageroot/crowdsec-firewall-bouncer.service +++ b/imageroot/crowdsec-firewall-bouncer.service @@ -36,8 +36,6 @@ ExecStart=/usr/bin/podman run \ ${CROWDSEC_FIREWALL_BOUNCER_IMAGE} ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N.cid -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N.cid -ExecReload=runagent -m ${MODULE_ID} expand-bouncer-configuration -ExecReload=/usr/bin/podman kill -s HUP %N PIDFile=%t/%N.pid Type=forking From 7be76ff5d7cffe5915cc02fd5c1001e1dd23dca0 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 17:37:49 +0200 Subject: [PATCH 06/13] ui. Fix docs URL --- ui/public/metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/public/metadata.json b/ui/public/metadata.json index e300ae9..472d4cf 100644 --- a/ui/public/metadata.json +++ b/ui/public/metadata.json @@ -12,7 +12,7 @@ } ], "docs": { - "documentation_url": "https://doc.crowdsec.net/docs/intro/", + "documentation_url": "https://docs.nethserver.org/projects/ns8/en/latest/crowdsec.html", "bug_url": "https://github.com/NethServer/dev", "code_url": "https://github.com/NethServer/ns8-crowdsec" }, From 3d812a63c142c6bb0c48c8e2ae902d6cc6f3529a Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 18:09:04 +0200 Subject: [PATCH 07/13] Remove obsolete event handlers Since the journal is not filtered by modules, restart is no longer required. --- .../events/module-added/00event_validation | 17 ----------------- imageroot/events/module-added/10handler | 13 ------------- .../events/module-removed/00event_validation | 1 - imageroot/events/module-removed/10handler | 13 ------------- 4 files changed, 44 deletions(-) delete mode 100755 imageroot/events/module-added/00event_validation delete mode 100755 imageroot/events/module-added/10handler delete mode 120000 imageroot/events/module-removed/00event_validation delete mode 100755 imageroot/events/module-removed/10handler diff --git a/imageroot/events/module-added/00event_validation b/imageroot/events/module-added/00event_validation deleted file mode 100755 index 1678432..0000000 --- a/imageroot/events/module-added/00event_validation +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -import json -import sys -import agent -import os - -event = json.load(sys.stdin) - -if str(event['node']) != os.environ['NODE_ID']: - print(agent.SD_DEBUG + "Event ignored: source is another node") - sys.exit(2) diff --git a/imageroot/events/module-added/10handler b/imageroot/events/module-added/10handler deleted file mode 100755 index 0b663f7..0000000 --- a/imageroot/events/module-added/10handler +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -set -e - -exec 1>&2 # Send any output to stderr, to not alter the action response protocol - -# when crowdsec restart we look after modules in redis to filter with journald (SYSLOG_IDENTIFIER) -systemctl restart ${MODULE_ID}.service diff --git a/imageroot/events/module-removed/00event_validation b/imageroot/events/module-removed/00event_validation deleted file mode 120000 index 3983cd9..0000000 --- a/imageroot/events/module-removed/00event_validation +++ /dev/null @@ -1 +0,0 @@ -../module-added/00event_validation \ No newline at end of file diff --git a/imageroot/events/module-removed/10handler b/imageroot/events/module-removed/10handler deleted file mode 100755 index 0b663f7..0000000 --- a/imageroot/events/module-removed/10handler +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -set -e - -exec 1>&2 # Send any output to stderr, to not alter the action response protocol - -# when crowdsec restart we look after modules in redis to filter with journald (SYSLOG_IDENTIFIER) -systemctl restart ${MODULE_ID}.service From f8da055565d9d571f3e6c1a654daa653ae56ef02 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 19:21:27 +0200 Subject: [PATCH 08/13] Refresh Systemd config on update Systemd units are templates: automatize their expansion on each update, to ease the current upgrade and future ones. Restart both crowdsec and bouncer services. --- imageroot/actions/create-module/10initialize | 35 +------------------ .../actions/create-module/50start-bouncer | 7 ---- imageroot/bin/install-systemd-units | 21 +++++++++++ imageroot/crowdsec-upgrade-hub.service | 9 +++++ imageroot/crowdsec-upgrade-hub.timer | 9 +++++ .../update-module.d/10install_systemd_units | 8 +++++ ...update-bouncer-to-container => 15fix_6900} | 9 +++-- imageroot/update-module.d/20restart | 4 +-- 8 files changed, 53 insertions(+), 49 deletions(-) create mode 100755 imageroot/bin/install-systemd-units create mode 100644 imageroot/crowdsec-upgrade-hub.service create mode 100644 imageroot/crowdsec-upgrade-hub.timer create mode 100755 imageroot/update-module.d/10install_systemd_units rename imageroot/update-module.d/{10update-bouncer-to-container => 15fix_6900} (53%) diff --git a/imageroot/actions/create-module/10initialize b/imageroot/actions/create-module/10initialize index a166581..d9c8384 100755 --- a/imageroot/actions/create-module/10initialize +++ b/imageroot/actions/create-module/10initialize @@ -8,41 +8,8 @@ set -e exec 1>&2 # Send any output to stderr, to not alter the action response protocol -install -m 644 "${AGENT_INSTALL_DIR}/crowdsec.service" "/etc/systemd/system/${MODULE_ID}.service" +install-systemd-units -# create the service to update the crowdsec hub for collections -tmpfile=$(mktemp) -trap "rm -f \${tmpfile}" EXIT -cat <${tmpfile} -[Unit] -Description=Update the crowdsec HUB -Requisite=${MODULE_ID}.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub update -ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub upgrade -SyslogIdentifier=%N -EOF -install -v -m 644 "${tmpfile}" /etc/systemd/system/${MODULE_ID}-upgrade-hub.service - -# create the timer to update the crowdsec hub for collections -tmpfile=$(mktemp) -trap "rm -f \${tmpfile}" EXIT -cat <${tmpfile} -[Unit] -Description=Timer of crowdsec hub update - -[Timer] -OnActiveSec=15 minutes -OnUnitInactiveSec=15 days - -[Install] -WantedBy=timers.target -EOF -install -v -m 644 "${tmpfile}" /etc/systemd/system/${MODULE_ID}-upgrade-hub.timer - -systemctl daemon-reload systemctl enable --now "${MODULE_ID}.service" systemctl enable --now "${MODULE_ID}-upgrade-hub.timer" diff --git a/imageroot/actions/create-module/50start-bouncer b/imageroot/actions/create-module/50start-bouncer index 7f881bd..a1649ff 100755 --- a/imageroot/actions/create-module/50start-bouncer +++ b/imageroot/actions/create-module/50start-bouncer @@ -8,13 +8,6 @@ set -e exec 1>&2 # Send any output to stderr, to not alter the action response protocol -tmpfile=$(mktemp) -trap "rm -f \${tmpfile}" EXIT -envsubst >${tmpfile} <"${AGENT_INSTALL_DIR}/crowdsec-firewall-bouncer.service" -install -m 644 "${tmpfile}" "/etc/systemd/system/${MODULE_ID}-firewall-bouncer.service" -# reload and start service -systemctl daemon-reload - # API server could be slow to start: # ignore bouncer connect error if it fails to start on first run systemctl enable --now ${MODULE_ID}-firewall-bouncer.service diff --git a/imageroot/bin/install-systemd-units b/imageroot/bin/install-systemd-units new file mode 100755 index 0000000..feacaf6 --- /dev/null +++ b/imageroot/bin/install-systemd-units @@ -0,0 +1,21 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# install the main controller service +install -m 644 "${AGENT_INSTALL_DIR}/crowdsec.service" "/etc/systemd/system/${MODULE_ID}.service" + +# install the firewall bouncer service +envsubst <"${AGENT_INSTALL_DIR}/crowdsec-firewall-bouncer.service" >"/etc/systemd/system/${MODULE_ID}-firewall-bouncer.service" + +# create the service to update the crowdsec hub for collections +envsubst <"${AGENT_INSTALL_DIR}/crowdsec-upgrade-hub.service" >"/etc/systemd/system/${MODULE_ID}-upgrade-hub.service" + +# create the timer to update the crowdsec hub for collections +envsubst <"${AGENT_INSTALL_DIR}/crowdsec-upgrade-hub.timer" >"/etc/systemd/system/${MODULE_ID}-upgrade-hub.timer" + +# reload and start service +systemctl daemon-reload \ No newline at end of file diff --git a/imageroot/crowdsec-upgrade-hub.service b/imageroot/crowdsec-upgrade-hub.service new file mode 100644 index 0000000..7510711 --- /dev/null +++ b/imageroot/crowdsec-upgrade-hub.service @@ -0,0 +1,9 @@ +[Unit] +Description=Update the crowdsec HUB +Requisite=${MODULE_ID}.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub update +ExecStart=/usr/bin/podman exec -i ${MODULE_ID} cscli hub upgrade +SyslogIdentifier=%N diff --git a/imageroot/crowdsec-upgrade-hub.timer b/imageroot/crowdsec-upgrade-hub.timer new file mode 100644 index 0000000..0b3ab5f --- /dev/null +++ b/imageroot/crowdsec-upgrade-hub.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Timer of crowdsec hub update + +[Timer] +OnActiveSec=15 minutes +OnUnitInactiveSec=15 days + +[Install] +WantedBy=timers.target diff --git a/imageroot/update-module.d/10install_systemd_units b/imageroot/update-module.d/10install_systemd_units new file mode 100755 index 0000000..e1e1a01 --- /dev/null +++ b/imageroot/update-module.d/10install_systemd_units @@ -0,0 +1,8 @@ +#!/bin/bash + +# +# Copyright (C) 2022 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +install-systemd-units diff --git a/imageroot/update-module.d/10update-bouncer-to-container b/imageroot/update-module.d/15fix_6900 similarity index 53% rename from imageroot/update-module.d/10update-bouncer-to-container rename to imageroot/update-module.d/15fix_6900 index 4be1bbf..e28e38d 100755 --- a/imageroot/update-module.d/10update-bouncer-to-container +++ b/imageroot/update-module.d/15fix_6900 @@ -1,4 +1,5 @@ #!/bin/bash + # # Copyright (C) 2024 Nethesis S.r.l. # SPDX-License-Identifier: GPL-3.0-or-later @@ -7,10 +8,8 @@ # Needed to upgrade from crowdsec:1.0.6 if systemctl is-active -q crowdsec-firewall-bouncer.service; then - # stop the bouncer - echo "Stop the crowdsec bouncer and disable it, removal of firewalld rules and bouncer binary can be done manually" + echo "Uninstall the old crowdsec binary bouncer" + echo "Check the app README for manual clean up instructions" systemctl disable --now crowdsec-firewall-bouncer.service - # start the bouncer - echo "Enable and start the systemd service of the crowdsec bouncer container" - ../actions/create-module/50start-bouncer + systemctl enable --now "${MODULE_ID}-firewall-bouncer.service" fi diff --git a/imageroot/update-module.d/20restart b/imageroot/update-module.d/20restart index cf1fe1a..58b2568 100755 --- a/imageroot/update-module.d/20restart +++ b/imageroot/update-module.d/20restart @@ -7,7 +7,5 @@ set -e -exec 1>&2 # Send any output to stderr, to not alter the action response protocol - # we need to start again to expand configuration -systemctl restart "${MODULE_ID}.service" +systemctl try-restart "${MODULE_ID}.service" "${MODULE_ID}-firewall-bouncer.service" From 5aa7f5611552dae683d76123e00f70d01f37272c Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 4 Apr 2024 19:38:06 +0200 Subject: [PATCH 09/13] Fix bouncer signal handling - Ensure the bouncer do not daemonize itself - Run container with --init to ensure zombies are reaped - Bouncer: systemd unit ignore exit code 1, returned also with SIGTERM - Controller: systemd unit ignore exit code 143 (SIGTERM) --- imageroot/crowdsec-firewall-bouncer.service | 2 ++ imageroot/crowdsec.service | 2 ++ imageroot/templates/crowdsec-firewall-bouncer.yaml.local | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/imageroot/crowdsec-firewall-bouncer.service b/imageroot/crowdsec-firewall-bouncer.service index bff3556..a97099a 100644 --- a/imageroot/crowdsec-firewall-bouncer.service +++ b/imageroot/crowdsec-firewall-bouncer.service @@ -23,8 +23,10 @@ LimitNOFILE=65536 TimeoutStopSec=70 ExecStartPre=/bin/rm -f %t/%N.pid %t/%N.cid ExecStartPre=runagent -m ${MODULE_ID} expand-bouncer-configuration +SuccessExitStatus=1 ExecStart=/usr/bin/podman run \ --detach \ + --init \ --conmon-pidfile %t/%N.pid \ --cidfile %t/%N.cid \ --cgroups=no-conmon \ diff --git a/imageroot/crowdsec.service b/imageroot/crowdsec.service index 8131639..d788f6b 100644 --- a/imageroot/crowdsec.service +++ b/imageroot/crowdsec.service @@ -24,8 +24,10 @@ ExecStartPre=/bin/mkdir -vp crowdsec_config/postoverflows/s01-whitelist ExecStartPre=/bin/mkdir -vp crowdsec_config/parsers/s02-enrich ExecStartPre=/usr/local/bin/runagent -m %N expand-configuration ExecStartPre=/usr/local/bin/runagent -m %N expand-smarthost +SuccessExitStatus=143 ExecStart=/usr/bin/podman run \ --detach \ + --init \ --privileged \ --conmon-pidfile %t/%N.pid \ --cidfile %t/%N.cid \ diff --git a/imageroot/templates/crowdsec-firewall-bouncer.yaml.local b/imageroot/templates/crowdsec-firewall-bouncer.yaml.local index 4a120b4..d8e5bab 100644 --- a/imageroot/templates/crowdsec-firewall-bouncer.yaml.local +++ b/imageroot/templates/crowdsec-firewall-bouncer.yaml.local @@ -1,7 +1,7 @@ mode: nftables pid_dir: /var/run/ update_frequency: 10s -daemonize: true +daemonize: false log_mode: stdout log_dir: /var/log/ log_level: info From 07e4ad5f9785307046070f892bb9e85668ae1b93 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Apr 2024 11:38:43 +0200 Subject: [PATCH 10/13] Fix deprecation warning The bouncer "daemonize" option is deprecated --- imageroot/templates/crowdsec-firewall-bouncer.yaml.local | 1 - 1 file changed, 1 deletion(-) diff --git a/imageroot/templates/crowdsec-firewall-bouncer.yaml.local b/imageroot/templates/crowdsec-firewall-bouncer.yaml.local index d8e5bab..74b0012 100644 --- a/imageroot/templates/crowdsec-firewall-bouncer.yaml.local +++ b/imageroot/templates/crowdsec-firewall-bouncer.yaml.local @@ -1,7 +1,6 @@ mode: nftables pid_dir: /var/run/ update_frequency: 10s -daemonize: false log_mode: stdout log_dir: /var/log/ log_level: info From 1b07f0bf98a5e08a8c111256176e7db8df99295b Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Apr 2024 11:40:20 +0200 Subject: [PATCH 11/13] First crowdsec conf provisioning run Run an initialization container with TEST_MODE=true, just to generate the initial Crowdsec configuration. Configuration is written inside the ./crowdsec_config directory, which is mounted as a volume. --- .../02create-first-configuration | 13 -------- imageroot/actions/create-module/10initialize | 12 +++++++ .../create-module/40crowdsec-configuration | 33 ------------------- imageroot/bin/expand-configuration | 9 +++-- 4 files changed, 16 insertions(+), 51 deletions(-) delete mode 100755 imageroot/actions/create-module/02create-first-configuration delete mode 100755 imageroot/actions/create-module/40crowdsec-configuration diff --git a/imageroot/actions/create-module/02create-first-configuration b/imageroot/actions/create-module/02create-first-configuration deleted file mode 100755 index 3f5faa1..0000000 --- a/imageroot/actions/create-module/02create-first-configuration +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -set -e - -# build folder to store configuration -mkdir -vp crowdsec_config -# first build of the configuration -exec expand-configuration diff --git a/imageroot/actions/create-module/10initialize b/imageroot/actions/create-module/10initialize index d9c8384..ace926b 100755 --- a/imageroot/actions/create-module/10initialize +++ b/imageroot/actions/create-module/10initialize @@ -8,6 +8,18 @@ set -e exec 1>&2 # Send any output to stderr, to not alter the action response protocol +# Run with TEST_MODE=true to generate the initial configuration +mkdir -vp crowdsec_config +podman run -i --rm --replace --name "${MODULE_ID}-init" \ + --log-driver=none \ + --privileged \ + --network=host \ + --env=TEST_MODE=true \ + --volume ./crowdsec_config:/etc/crowdsec:Z \ + --volume "${MODULE_ID}-data":/var/lib/crowdsec/data:Z \ + --volume "${CROWDSEC_JOURNAL}":/run/log/journal \ + ${CROWDSEC_IMAGE} + install-systemd-units systemctl enable --now "${MODULE_ID}.service" diff --git a/imageroot/actions/create-module/40crowdsec-configuration b/imageroot/actions/create-module/40crowdsec-configuration deleted file mode 100755 index c2b4486..0000000 --- a/imageroot/actions/create-module/40crowdsec-configuration +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-3.0-or-later -# - -import os -import json -import agent -import agent.tasks - -from jinja2 import Environment, FileSystemLoader, select_autoescape - -files =["crowdsec_config/config.yaml.local"] -for f in files: - try: - os.remove(f) - except FileNotFoundError: - pass - -jenv = Environment( - loader=FileSystemLoader(os.getenv("AGENT_INSTALL_DIR")+"/templates"), - autoescape=select_autoescape(), - keep_trailing_newline=True, -) -# placeholder for later -properties = {} - -template = jenv.get_template('config.yaml.local') -output = template.render(properties) -with open("crowdsec_config/config.yaml.local","w") as f: - f.write(output) diff --git a/imageroot/bin/expand-configuration b/imageroot/bin/expand-configuration index 598cf84..a3dbc5a 100755 --- a/imageroot/bin/expand-configuration +++ b/imageroot/bin/expand-configuration @@ -35,10 +35,8 @@ template = jenv.get_template('acquis.yaml') with open("crowdsec_config/acquis.yaml","w") as f: f.write(template.render()) -# The first start crowdsec expects other configuration files -# if these files are not present then the service fails to start -# we start the first time with the default configuration -if os.path.isfile("crowdsec_config/config.yaml.local"): +# expand config.yaml.local +if True: files =["crowdsec_config/config.yaml.local"] for f in files: try: @@ -58,8 +56,9 @@ if os.path.isfile("crowdsec_config/config.yaml.local"): output = template.render(properties) with open("crowdsec_config/config.yaml.local","w") as f: f.write(output) + ## template of local_api_credentials.yaml.local -if os.path.isfile("crowdsec_config/config.yaml.local"): +if True: files =["crowdsec_config/local_api_credentials.yaml.local"] for f in files: try: From 00c069112af06fd8a6e1d0ed1ef6299c1ca83744 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Apr 2024 11:48:41 +0200 Subject: [PATCH 12/13] Ensure the old bouncer is disabled too The unit of the old package-based bouncer must be disabled and the update code triggered even if the service is only stopped. --- imageroot/update-module.d/15fix_6900 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imageroot/update-module.d/15fix_6900 b/imageroot/update-module.d/15fix_6900 index e28e38d..42762b4 100755 --- a/imageroot/update-module.d/15fix_6900 +++ b/imageroot/update-module.d/15fix_6900 @@ -7,7 +7,7 @@ # Stop the bouncer installed on the host and start the bouncer container # Needed to upgrade from crowdsec:1.0.6 -if systemctl is-active -q crowdsec-firewall-bouncer.service; then +if systemctl is-active -q crowdsec-firewall-bouncer.service || systemctl is-enabled crowdsec-firewall-bouncer.service &>/dev/null ; then echo "Uninstall the old crowdsec binary bouncer" echo "Check the app README for manual clean up instructions" systemctl disable --now crowdsec-firewall-bouncer.service From 865a3466371e7a763dc0453ad346d2353064e113 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Fri, 5 Apr 2024 13:03:23 +0200 Subject: [PATCH 13/13] ui. Fix notification text The text label refers to the old "Smarthost" name. --- ui/public/i18n/en/translation.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/public/i18n/en/translation.json b/ui/public/i18n/en/translation.json index 35fd22f..ed27f98 100644 --- a/ui/public/i18n/en/translation.json +++ b/ui/public/i18n/en/translation.json @@ -46,7 +46,7 @@ "helo_host": "Helo_host", "helo_host_must_be_relevant_for_smtp": "This might be needed to properly receive email notifications. If your antispam system adds score due to default 'localhost' Helo name, you can set a specific Helo FQDN here", "helo_host_placeholder": "sub.domain.com", - "enable_smarthosts_for_notifications": "Smarthost settings", + "enable_smarthosts_for_notifications": "Go to cluster Settings", "bad_email_address": "Bad email address", "bad_IP_or_hostname": "Bad CIDR, IP or FQDN", "processing": "Processing...", @@ -54,7 +54,7 @@ "token_not_valid":"The token provided is not valid", "whitelists_tips": "No ban will occur for members of this list", "smarthost_is_disabled": "Email notifications are disabled", - "smarthosts_is_needed_to_send_notifications": "Enable Smarthost to receive CrowdSec email notifications" + "smarthosts_is_needed_to_send_notifications": "To receive mail notifications from Crowdsec, change the cluster's email settings" }, "unban":{ "title": "Banned IP",