From a870098c7e43db38003cee0447e3ce66c6e906cf Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 14 Nov 2024 09:22:47 +0100 Subject: [PATCH] Gradually increase TCP ports up to 1024 - Set an high limit to TCP ports allocation (1024). - Deallocate the predefined TCP_PORTS_RANGE in create-module. - Dynamically allocate the TCP ports, up to 1024 limit. - Reuse previously allocated ports for existing installations. --- build-images.sh | 3 +- .../actions/create-module/10deallocate_ports | 14 +++++ imageroot/bin/allocate-ports | 56 ++++++++----------- imageroot/update-module.d/10dynamic_ports | 29 ++++++++++ 4 files changed, 68 insertions(+), 34 deletions(-) create mode 100755 imageroot/actions/create-module/10deallocate_ports create mode 100755 imageroot/update-module.d/10dynamic_ports diff --git a/build-images.sh b/build-images.sh index 5ecc9c4f..151a386d 100644 --- a/build-images.sh +++ b/build-images.sh @@ -10,7 +10,8 @@ container=$(buildah from scratch) buildah add "${container}" imageroot /imageroot buildah add "${container}" ui /ui buildah config \ - --label='org.nethserver.tcp-ports-demand=8' \ + --label="org.nethserver.authorizations=node:portsadm" \ + --label='org.nethserver.tcp-ports-demand=1024' \ --label='org.nethserver.flags=core_module no_data_backup' \ --label="org.nethserver.images=docker.io/library/nginx:1.27.2-alpine" \ --entrypoint=/ "${container}" diff --git a/imageroot/actions/create-module/10deallocate_ports b/imageroot/actions/create-module/10deallocate_ports new file mode 100755 index 00000000..b790efc5 --- /dev/null +++ b/imageroot/actions/create-module/10deallocate_ports @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import agent + +# Clean-up the initial TCP ports allocation +agent.deallocate_ports("tcp") +agent.unset_env("TCP_PORT") +agent.unset_env("TCP_PORTS") +agent.unset_env("TCP_PORTS_RANGE") diff --git a/imageroot/bin/allocate-ports b/imageroot/bin/allocate-ports index d6b4cc9f..0481dd3e 100755 --- a/imageroot/bin/allocate-ports +++ b/imageroot/bin/allocate-ports @@ -22,61 +22,51 @@ import agent import cluster.userdomains -import itertools -import sys import os import json agent_id = os.environ['AGENT_ID'] node_id = int(os.environ['NODE_ID']) -rbegin, rend = os.environ['TCP_PORTS_RANGE'].split('-') -ports = set([int(port) for port in range(int(rbegin),int(rend)+1)]) -ports |= set([int(port) for port in os.environ['TCP_PORTS'].split(',')]) - -rdb = agent.redis_connect(use_replica=True) +domains = cluster.userdomains.list_domains(agent.redis_connect()) +rdb = agent.redis_connect(privileged=True) domain_port = rdb.hgetall(f'{agent_id}/data/domain_port') +released_ports = rdb.smembers(f'{agent_id}/data/released_ports') user_domain_changed_events = [] -domains = cluster.userdomains.list_domains(rdb) -for domain in list(domain_port.keys()): - # If the domain does not exist anymore in the cluster, release its port +# copy the key list, to alter the dict in the FOR-loop: +domains_with_port = list(domain_port.keys()) +for domain in domains_with_port: + # If the domain does not exist anymore in the cluster, release its + # allocated port. if not domain in domains: - print(f"Release TCP port {domain_port[domain]} allocated for domain {domain}") - domain_port.pop(domain) + print(f"Release TCP port {domain_port[domain]} allocated for old domain {domain}") + released_ports.add(domain_port.pop(domain)) # save port number for the future user_domain_changed_events.append({"node_id": node_id, "domain": domain}) -try: - iports = [int(p) for p in domain_port.values()] - allocated_ports = set(iports) -except AttributeError: - allocated_ports = set() - -free_ports = ports - allocated_ports - for domain in domains: + # If the domain has no port allocation, allocate a TCP port to it. if not domain in domain_port: - try: - # Allocate the first available port number - nport = free_ports.pop() - except KeyError: - print(agent.SD_ERR + f"TCP ports range exhausted! {domain} was not allocated a port.") - break - - domain_port[domain] = nport + if released_ports: + nport = released_ports.pop() # reuse old ports + else: + # Ask the local node to allocate us one more port + nport, _ = agent.allocate_ports(ports_number=1, protocol="tcp", keep_existing=True) + domain_port[domain] = str(nport) user_domain_changed_events.append({"node_id": node_id, "domain": domain}) - print(f"Allocating TCP port {nport} for {domain}...") - -rdb.close() # close the read-only connection + print(f"Allocating TCP port {nport} to {domain}...") # -# Save the new domain/port allocations any change occurred +# Persist changes to Redis and raise events for applications # if len(user_domain_changed_events) > 0: - with agent.redis_connect(privileged=True).pipeline() as trx: + with rdb.pipeline() as trx: trx.delete(f'{agent_id}/data/domain_port') if domain_port: trx.hset(f'{agent_id}/data/domain_port', mapping=domain_port) + trx.delete(f'{agent_id}/data/released_ports') + if released_ports: + trx.sadd(f'{agent_id}/data/released_ports', *released_ports) for domevent in user_domain_changed_events: trx.publish(f'{agent_id}/event/user-domain-changed', json.dumps(domevent)) trx.execute() diff --git a/imageroot/update-module.d/10dynamic_ports b/imageroot/update-module.d/10dynamic_ports new file mode 100755 index 00000000..aad21f66 --- /dev/null +++ b/imageroot/update-module.d/10dynamic_ports @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import agent +import sys +import os + +if not 'TCP_PORTS' in os.environ: + sys.exit(0) + +agent_id = os.environ['AGENT_ID'] +rdb = agent.redis_connect(privileged=True) +domain_port = rdb.hgetall(f'{agent_id}/data/domain_port') + +# Versions up to 1.0.2 have a fixed range of 8 ports allocated by the core +# when the module instance is created. They are listed in "TCP_PORTS" +# environment variable. We consider unused ports as "released", so the +# "allocate-ports" procedure can reuse them before asking for new ports to +# the local node. +unused_ports = set(os.environ['TCP_PORTS'].split(",")) - set(domain_port.values()) +rdb.sadd(f'{agent_id}/data/released_ports', *unused_ports) + +agent.unset_env("TCP_PORT") +agent.unset_env("TCP_PORTS") +agent.unset_env("TCP_PORTS_RANGE")