Merge pull request #7 from NethServer/sdl-6895

Update mail server configuration and random TCP port NethServer/dev#6895

README.md

@@ -23,7 +23,6 @@ Launch `configure-module`, by setting the following parameters:
 - `host`: a fully qualified domain name for the application
 - `http2https`: enable or disable HTTP to HTTPS redirection
 - `lets_encrypt`: enable or disable Let's Encrypt certificate
-- `imap_host`: set a hostname of an imap server to authenticate user from it
 
 Example:
 
@@ -33,7 +32,7 @@ Example:
       "host": "piler.domain.com",
       "http2https": true,
       "lets_encrypt": false,
-      "imap_host": "imap.domain.com"
+      "mail_server": "c990d0d0-6216-4651-9d0b-d393117d0f7e"
     }
 EOF
 ```
@@ -52,10 +51,10 @@ api-cli run get-configuration --agent module/piler1 --data null | jq
 ```
 {
   "host": "piler.domain.com",
-  "imap_host": "imap.domain.com",
   "http2https": true,
   "lets_encrypt": false,
-  "tcp_port_archive": "2525"
+  "mail_server": "c990d0d0-6216-4651-9d0b-d393117d0f7e",
+  "mail_server_URL": []
 }
 ```
 
@@ -76,22 +75,22 @@ The tests are made using [Robot Framework](https://robotframework.org/)
 
 ## send email to piler
 
-Piler is waiting email on a TCP port on 2525, to send email to the archive system you need
+Piler is waiting email on a TCP port on a random port(>20000/tcp), to send email to the archive system you need
 
 - Bcc email of your domain to [email protected]
 - adapt your email server to send email to piler.domain.com on the custom smtp port
 
 This can be done by adapting the `/etc/postfix/transport/`
 
-`piler.domain.com smtp:piler.domain.com:2525`
+`piler.domain.com smtp:piler.domain.com:20001`
 
 - postmap the configuration file (if needed) : `postmap /etc/postfix/transport`
 - restart postfix : `systemctl restart postfix`
 
+## Import emails to piler
 
-On NS7 you can set it by
+Previous emails are sent automatically one time to piler after the first configuration, but if you want to launch manually the synchronisation, you can trigger this service in the terminal:
 
-```
-db smarthosts set @piler.domain.com recipient Host piler.domain.com Password '' Port 2525 TlsStatus enabled Username '' status enabled
-signal-event nethserver-mail-smarthost-save
-```
+    runagent -m piler1 import-emails
+
+Piler understands and manage  duplicated emails.

build-images.sh

@@ -32,8 +32,8 @@ buildah add "${container}" imageroot /imageroot
 buildah add "${container}" ui/dist /ui
 # Setup the entrypoint, ask to reserve one TCP port with the label and set a rootless container
 buildah config --entrypoint=/ \
-    --label="org.nethserver.authorizations=node:fwadm traefik@any:routeadm" \
-    --label="org.nethserver.tcp-ports-demand=1" \
+    --label="org.nethserver.authorizations=node:fwadm traefik@node:routeadm mail@any:mailadm" \
+    --label="org.nethserver.tcp-ports-demand=2" \
     --label="org.nethserver.rootfull=0" \
     --label="org.nethserver.images=docker.io/sutoj/piler:1.4.4 docker.io/mariadb:10.11.7 docker.io/memcached:1.6.26-alpine" \
     "${container}"

imageroot/actions/clone-module/50configure-module

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import agent
+import os
+
+configure_retval = agent.tasks.run(agent_id=os.environ['AGENT_ID'], action='configure-module', data={
+    "mail_server": os.environ["MAIL_SERVER"],
+    "lets_encrypt": os.environ["TRAEFIK_LETS_ENCRYPT"] == 'True',
+    "host": os.environ["TRAEFIK_HOST"],
+    "http2https": os.environ["TRAEFIK_HTTP2HTTPS"] == 'True',
+    "restore": True,
+})
+agent.assert_exp(configure_retval['exit_code'] == 0, "The configure-module subtask failed!")

imageroot/actions/configure-module/01_validate_domain

@@ -1,60 +1,26 @@
 #!/usr/bin/env python3
 
 #
-# Copyright (C) 2022 Nethesis S.r.l.
+# Copyright (C) 2024 Nethesis S.r.l.
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
 
 import os
 import sys
 import json
 import agent
-import urllib.request
-import urllib.error
-import ssl
 
+agent.set_weight(os.path.basename(__file__), 0) # Validation step, no task progress at all
 
 # retrieve json data
 data = json.load(sys.stdin)
 
 # Setup default values
-host = data.get("host")
+hostname = data.get("host")
 # do not test if it is the same host
 oldHost = os.environ.get('TRAEFIK_HOST','')
-if host == oldHost:
-    exit(0)
 
-# set error validation
-agent.set_weight(os.path.basename(__file__), 0)
-
-http = 0
-https = 0
-
-# do not verify the certificate
-ctx = ssl.create_default_context()
-ctx.check_hostname = False
-ctx.verify_mode = ssl.CERT_NONE
-# try on http
-try:
-    req = urllib.request.Request('http://127.0.0.1')
-    req.add_header('Host', host)
-    urllib.request.urlopen(req, context=ctx)
-except urllib.error.HTTPError as e:
-    http = e.code
-
-# try on https
-try:
-    req = urllib.request.Request('https://127.0.0.1')
-    req.add_header('Host', host)
-    urllib.request.urlopen(req, context=ctx)
-except urllib.error.HTTPError as d:
-    https = d.code
-
-if http == 404 and  https == 404:
-    # there is no website on http or https -> OK
-    exit(0)
-else:
-    # path exists -> nok
+if hostname != oldHost and agent.http_route_in_use(domain=hostname):
     agent.set_status('validation-failed')
-    json.dump([{'field':'host','parameter':'host','value':host,'error':'domain_already_used_in_traefik'}],fp=sys.stdout)
+    json.dump([{'field':'host','parameter':'host','value':hostname,'error':'domain_already_used_in_traefik'}],fp=sys.stdout)
     sys.exit(2)

imageroot/actions/configure-module/01_validate_mail_server

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2022 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import agent
+
+agent.set_weight(os.path.basename(__file__), 0) # Validation step, no task progress at all
+
+# Try to parse the stdin as JSON.
+# If parsing fails, output everything to stderr
+data = json.load(sys.stdin)
+
+# Connect to redis
+rdb = agent.redis_connect()
+
+providers = agent.list_service_providers(rdb, 'imap', 'tcp', {
+    'module_uuid': data["mail_server"],
+})
+
+if not providers:
+    agent.set_status('validation-failed')
+    json.dump([{'field':'mail_server','parameter':'mail_server','value': data["mail_server"],'error':'mail_server_is_not_valid'}], fp=sys.stdout)
+    sys.exit(3)

imageroot/actions/configure-module/02_validate_bcc_server

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import sys
+import os
+import agent
+
+agent.set_weight(os.path.basename(__file__), 0) # Validation step, no task progress at all
+
+# Try to parse the stdin as JSON.
+# If parsing fails, output everything to stderr
+data = json.load(sys.stdin)
+# we are restoring, do not test
+if data.get('restore', False):
+    sys.exit(0)
+
+# Connect to redis
+rdb = agent.redis_connect()
+
+providers = agent.list_service_providers(rdb, 'imap', 'tcp', {
+    'module_uuid': data["mail_server"],
+})
+# function to test get-alaways-bcc
+def get_always_bcc(email):
+    mail_id = providers[0]["module_id"]
+    response = agent.tasks.run(f"module/{mail_id}", action='get-always-bcc')
+    agent.assert_exp(response['exit_code'] == 0)
+    # check if the always_bcc is not set
+    if response['output']['bcc'] == '':
+        sys.exit(0)
+    # check if the always_bcc is set to the email
+    elif response['output']['bcc'] == email:
+        sys.exit(0)
+    else:
+        agent.set_status('validation-failed')
+        json.dump([{'field':'mail_server','parameter':'mail_server','value': response['output']['bcc'],'error':'always_bcc_is_already_set'}], fp=sys.stdout)
+        sys.exit(2)
+
+
+if providers:
+    # we need to set bcc address for the mail server
+    get_always_bcc('archive@'+os.environ['MODULE_ID'])

imageroot/actions/configure-module/10configure

@@ -14,10 +14,14 @@ import agent
 # Try to parse the stdin as JSON.
 # If parsing fails, output everything to stderr
 data = json.load(sys.stdin)
-imap_host = data.get("imap_host")
 
-agent.set_env("IMAP_HOST", imap_host)
+# Connect to redis
+rdb = agent.redis_connect()
 
-# Make sure everything is saved inside the environment file
-# just before starting systemd unit
-agent.dump_env()
+providers = agent.list_service_providers(rdb, 'imap', 'tcp', {
+    'module_uuid': data["mail_server"],
+})
+
+if providers:
+    # Set the IMAP_HOST (IPV4) environment variable to the host of the first provider
+    agent.set_env("IMAP_HOST", providers[0]["host"])

imageroot/actions/configure-module/20traefik

@@ -26,10 +26,6 @@ agent.set_env("TRAEFIK_HOST", host)
 agent.set_env("TRAEFIK_HTTP2HTTPS", h2hs)
 agent.set_env("TRAEFIK_LETS_ENCRYPT", le)
 
-# Make sure everything is saved inside the environment file
-# just before starting systemd unit
-agent.dump_env()
-
 # Find default traefik instance for current node
 default_traefik_id = agent.resolve_agent_id('traefik@node')
 if default_traefik_id is None:
@@ -41,7 +37,7 @@ response = agent.tasks.run(
     action='set-route',
     data={
         'instance': os.environ['MODULE_ID'],
-        'url':  'http://127.0.0.1:' + os.environ["TCP_PORT"],
+        'url':  'http://127.0.0.1:' + os.environ["TCP_PORT_TRAEFIK"],
         'host': host,
         'http2https': h2hs,
         'lets_encrypt': le

imageroot/actions/configure-module/80start_services

@@ -9,3 +9,14 @@
 
 systemctl --user enable piler.service
 systemctl --user restart piler.service
+
+# before to send the action add-relay-rule or aler-relay-rule we need to wait until the service is active
+# does not work as expected in systemd post startup
+for ((i = 0; i < 60; i++)); do
+    if podman exec piler-app /etc/init.d/rc.piler status | grep 'piler is running'; then
+        break
+    fi
+    echo "Waiting for piler-app to be active..."
+    sleep 1
+done
+