chore(scripts): add some python scripts

.gitignore

@@ -16,6 +16,9 @@ src/e2e/ExportData/*
 src/main/app/.nvmrc
 /bin
 
+### Python virtual environment ###
+scripts/python/.venv
+
 ### INTELLIJ ###
 *.iml
 .idea
@@ -43,4 +46,5 @@ docker-compose.override.yml
 check-for-running-containers.sh
 fix-permissions.sh
 
+### Docker ###
 .DS_store

scripts/python/dependencies/brp_version_extractor.py

@@ -0,0 +1,38 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+from version_data import VersionData
+
+
+class BrpVersionExtractor:
+    def __init__(self, url):
+        self.url = url
+
+    def get_latest_version(self):
+        # Send a GET request to the web page
+        response = requests.get(self.url)
+
+        # Parse the web page content
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Find all version numbers matching the pattern 'Versi(on|e) x.y.z'
+        version_pattern = re.compile(r'Versi[e|o]n? ([\d.]+)')
+        versions = version_pattern.findall(soup.text)
+
+        # Extract the version numbers from a tags within the ul
+        version_list = []
+        for version_number in versions:
+            if re.match(r'[\d.]+', version_number):
+                version_data = VersionData(version_number, self.url)
+                version_list.append(version_data)
+
+
+        # Return the latest version
+        if version_list:
+            return max(version_list)
+        else:
+            return None

scripts/python/dependencies/github_tag_version_extractor.py

@@ -0,0 +1,41 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+from version_data import VersionData
+
+class GitHubTagVersionExtractor:
+    def __init__(self, url, prefix = ''):
+        self.url = url
+        self.prefix = prefix
+
+    def get_latest_version(self):
+        # Send a GET request to the web page
+        response = requests.get(self.url)
+
+        # Parse the web page content
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Find all h2.a tags containing the version information
+        commits = soup.find_all('div', class_='commit')
+
+        # Extract the version numbers from a tags within the ul
+        version_list = []
+        for commit in commits:
+            a_tag = commit.find('a')
+            if a_tag:
+                version_number = a_tag.text.strip()
+                if re.match(rf'{re.escape(self.prefix)}[\d.]+', version_number):
+                    href = a_tag['href']
+                    if not href.startswith('http'):
+                        href = requests.compat.urljoin(self.url, href)
+                    version_list.append(VersionData(version_number.lstrip(self.prefix), href))
+
+        # Return the latest version
+        if version_list:
+            return max(version_list)
+        else:
+            return None

scripts/python/dependencies/postgresql_version_extractor.py

@@ -0,0 +1,40 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+
+import requests
+from bs4 import BeautifulSoup
+import re
+
+from version_data import VersionData
+
+class PostgresqlVersionExtractor:
+    def __init__(self, url):
+        self.url = url
+
+    def get_latest_version(self):
+        # Send a GET request to the web page
+        response = requests.get(self.url)
+
+        # Parse the web page content
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        # Find all ul tags containing the release versions
+        commits = soup.find_all('ul' , class_='release-notes-list')
+
+        # Extract the version numbers from a tags within the ul
+        version_list = []
+        for commit in commits:
+            a_tag = commit.find('a')
+            if a_tag:
+                version_number = a_tag.text.strip()
+                if re.match(r'[\d.]+', version_number):
+                    href = a_tag['href']
+                    if not href.startswith('http'):
+                        href = requests.compat.urljoin(self.url, href)
+                    version_list.append(VersionData(version_number, href))
+
+        # Return the latest version
+        if version_list:
+            return max(version_list)
+        else:
+            return None

scripts/python/dependencies/version_data.py

@@ -0,0 +1,51 @@
+#
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+#
+class VersionData:
+    def __init__(self, version, url):
+        self.version = version
+        self.url = url
+        parts = self.version.split('.')
+        self.major = 0
+        self.minor = 0
+        self.patch = 0
+        self.sub = self.version
+        if len(parts) > 0 and parts[0].isdigit():
+            self.major = int(parts[0])
+            self.sub = self.sub.replace(str(self.major), '', 1).lstrip('.-')
+            if len(parts) > 1 and parts[1].isdigit():
+                self.minor = int(parts[1])
+                self.sub = self.sub.replace(str(self.minor), '', 1).lstrip('.-')
+                if len(parts) > 2 and parts[2].isdigit():
+                    self.patch = int(parts[2])
+                    self.sub = self.sub.replace(str(self.patch), '', 1).lstrip('.-')
+
+    def __str__(self):
+        return f'{self.version} - {self.url}'
+
+    def __eq__(self, other):
+        return self.version == other.version and self.url == other.url
+
+    def __lt__(self, other):
+        if self.major < other.major:
+            return True
+        elif self.major == other.major:
+            if self.minor < other.minor:
+                return True
+            elif self.minor == other.minor:
+                if self.patch < other.patch:
+                    return True
+        return False
+
+    def __gt__(self, other):
+        if self.major > other.major:
+            return True
+        elif self.major == other.major:
+            if self.minor > other.minor:
+                return True
+            elif self.minor == other.minor:
+                if self.patch > other.patch:
+                    return True
+        return False
+

scripts/python/dependencies/versions-of-components.py

@@ -0,0 +1,67 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+
+from brp_version_extractor import BrpVersionExtractor
+from github_tag_version_extractor import GitHubTagVersionExtractor
+from postgresql_version_extractor import PostgresqlVersionExtractor
+
+if __name__ == '__main__':
+    # Extract the latest version of Haal-Centraal-BRP from the release notes
+    brp_version_extractor = BrpVersionExtractor('https://brp-api.github.io/Haal-Centraal-BRP-bevragen/releasenotes')
+    brp_latest_version = brp_version_extractor.get_latest_version()
+    print(f'Haal-Centraal-BRP latest version: {brp_latest_version}')
+
+    # Extract the latest version of Haal-Centraal-BRP from the GitHub tags
+    open_notificaties_extractor = GitHubTagVersionExtractor('https://github.com/open-zaak/open-notificaties/tags')
+    open_notificaties_latest_version = open_notificaties_extractor.get_latest_version()
+    print(f'Open Notificaties latest version: {open_notificaties_latest_version}')
+
+    # Extract the latest version of Haal-Centraal-BRP from the GitHub tags
+    open_zaak_extractor = GitHubTagVersionExtractor('https://github.com/open-zaak/open-zaak/tags')
+    open_zaak_latest_version = open_zaak_extractor.get_latest_version()
+    print(f'Open Zaak latest version: {open_zaak_latest_version}')
+
+    # Extract the latest version of Haal-Centraal-BRP from the GitHub tags
+    open_klant_extractor = GitHubTagVersionExtractor('https://github.com/maykinmedia/open-klant/tags')
+    open_klant_latest_version = open_klant_extractor.get_latest_version()
+    print(f'Open Klant latest version: {open_klant_latest_version}')
+
+    # Extract the latest version of Open Formulieren the GitHub tags
+    open_formulieren_extractor = GitHubTagVersionExtractor('https://github.com/open-formulieren/open-forms/tags')
+    open_formulieren_latest_version = open_formulieren_extractor.get_latest_version()
+    print(f'Open Formulieren latest version: {open_formulieren_latest_version}')
+
+    # Extract the latest version of Objecten API from the GitHub tags
+    open_objecten_extractor = GitHubTagVersionExtractor('https://github.com/maykinmedia/objects-api/tags')
+    open_objecten_latest_version = open_objecten_extractor.get_latest_version()
+    print(f'Objecten API latest version: {open_objecten_latest_version}')
+
+    # Extract the latest version of Object Types API from the GitHub tags
+    open_objecttypes_extractor = GitHubTagVersionExtractor('https://github.com/maykinmedia/objecttypes-api/tags')
+    open_objecttypes_latest_version = open_objecttypes_extractor.get_latest_version()
+    print(f'Object Types API latest version: {open_objecttypes_latest_version}')
+
+    # Extract the latest version of open-inwoner from the GitHub tags
+    open_inwoner_extractor = GitHubTagVersionExtractor('https://github.com/maykinmedia/open-inwoner/tags', 'v')
+    open_inwoner_latest_version = open_inwoner_extractor.get_latest_version()
+    print(f'Open Inwoner latest version: {open_inwoner_latest_version}')
+
+    # Extract the latest version of KISS from the GitHub tags
+    open_objecttypes_extractor = GitHubTagVersionExtractor('https://github.com/Klantinteractie-Servicesysteem/KISS-frontend/releases','v')
+    open_objecttypes_latest_version = open_objecttypes_extractor.get_latest_version()
+    print(f'KISS latest version: v{open_objecttypes_latest_version}')
+
+    # Extract the latest version of Keycloak from the GitHub tags
+    keycloak_extractor = GitHubTagVersionExtractor('https://github.com/keycloak/keycloak/tags')
+    keycloak_latest_version = keycloak_extractor.get_latest_version()
+    print(f'Keycloak latest version: {keycloak_latest_version}')
+
+    # Extract the latest version of Clamav from the GitHub tags
+    clamav_extractor = GitHubTagVersionExtractor('https://github.com/Cisco-Talos/clamav/tags','clamav-')
+    clamav_latest_version = clamav_extractor.get_latest_version()
+    print(f'ClamAV latest version: clamav-{clamav_latest_version}')
+
+    # Extract the latest version of Postgresql from it's release page
+    postgresql_extractor = PostgresqlVersionExtractor('https://www.postgresql.org/docs/release/')
+    postgresql_latest_version = postgresql_extractor.get_latest_version()
+    print(f'Postgresql latest version: {postgresql_latest_version}')

scripts/python/init-pyenv.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# SPDX-FileCopyrightText: 2024 Lifely
+# SPDX-License-Identifier: EUPL-1.2+
+#
+
+# Create a virtual environment for Python
+if [ -d .venv ]; then
+    echo "Virtual environment already exists"
+else
+    echo "Create virtual environment"
+    python3 -m venv .venv
+fi
+
+# Activate the virtual environment and install the dependencies
+source .venv/bin/activate
+pip install --upgrade pip
+pip install -r requirements.txt

scripts/python/podiumd/component_version.py

@@ -0,0 +1,14 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+#
+
+class ComponentVersion:
+    def __init__(self, name, version):
+        self.name = name
+        self.version = version
+
+    def __str__(self):
+        return f'{self.name}: {self.version}'
+
+    def __lt__(self, other):
+        return self.name < other.name

scripts/python/podiumd/podiumd_version.py

@@ -0,0 +1,15 @@
+#
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+#
+
+class PodiumdVersion:
+    def __init__(self, tag_url, chart_version, app_version, component_versions = None):
+        self.tag_url = tag_url
+        self.chart_version = chart_version
+        self.app_version = app_version
+        self.component_versions = component_versions if component_versions else {}
+
+    def __str__(self):
+        components_str = '\n'.join([str(v) for v in self.component_versions.items()])
+        return f'PodiumD: {self.app_version}\nChart: {self.chart_version}\nComponents:\n{components_str}'

scripts/python/podiumd/podiumd_version_builder.py

@@ -0,0 +1,62 @@
+#
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+#
+import urllib.request
+import yaml
+
+from podiumd_version import PodiumdVersion
+from component_version import ComponentVersion
+
+
+class PodiumdVersionBuilder:
+    def __init__(self, version=None):
+        self.podiumd_root_path = "https://raw.githubusercontent.com/Dimpact-Samenwerking/helm-charts/refs/heads/main"
+        if version:
+            self.podiumd_root_path = f"https://raw.githubusercontent.com/Dimpact-Samenwerking/helm-charts/refs/tags/podiumd-{version}"
+        self.chart_file = f"{self.podiumd_root_path}/charts/podiumd/Chart.yaml"
+        self.values_file = f"{self.podiumd_root_path}/charts/podiumd/values.yaml"
+        self.products_table = {
+            'openzaak': 'Open Zaak',
+            'opennotificaties': 'Open Notificaties',
+            'objecten': 'Objecten',
+            'objecttypen': 'Objecttypen',
+            'openklant': 'Open Klant',
+            'openformulieren': 'Open Formulieren',
+            'openinwoner': 'Open Inwoner',
+            'kiss.frontend': 'Contact',
+            'clamav': 'ClamAV'
+        }
+
+    def read_chart_version(self):
+        with urllib.request.urlopen(self.chart_file) as char_text:
+            chart_data = yaml.safe_load(char_text)
+            return { "chart_version": chart_data['version'], "app_version": chart_data['appVersion'] }
+
+    def build_component_version(self, product_name, version):
+        if product_name in self.products_table:
+            return { "component": self.products_table[product_name], "version": version }
+        else:
+            return { "component": product_name, "version": version }
+
+    def read_component_versions(self):
+        component_version = {}
+        with urllib.request.urlopen(self.values_file) as values_text:
+            values_data = yaml.safe_load(values_text)
+            for top_element, top_value in values_data.items():
+                if isinstance(top_value, dict) and 'image' in top_value:
+                    image_tag_value = top_value['image']['tag']
+                    v = self.build_component_version(f'{top_element}', image_tag_value)
+                    component_version[v['component']]=ComponentVersion(v['component'], v['version'])
+                elif isinstance(top_value, dict):
+                    for sub_element, sub_value in top_value.items():
+                        if isinstance(sub_value, dict) and 'image' in sub_value:
+                            image_tag_value = sub_value['image']['tag']
+                            v = self.build_component_version(f'{top_element}.{sub_element}', image_tag_value)
+                            component_version[v['component']]=ComponentVersion(v['component'], v['version'])
+        return component_version
+
+    def build(self):
+        chart_version = self.read_chart_version()
+        component_versions = self.read_component_versions()
+        return PodiumdVersion(self.podiumd_root_path, chart_version["chart_version"], chart_version["app_version"], component_versions)

scripts/python/podiumd/podiumd_version_comparator.py

@@ -0,0 +1,22 @@
+#  SPDX-FileCopyrightText: 2024 Lifely
+#  SPDX-License-Identifier: EUPL-1.2+
+
+class PodiumdVersionComparator:
+    def __init__(self, version1, version2):
+        self.version1 = version1
+        self.version2 = version2
+
+    def print_version_updated_table(self):
+        print('\n# PodiumD version updates:')
+        print(f'| Component | PodiumD {self.version1.app_version} | PodiumD {self.version2.app_version} |')
+        print(f'|---|---|---|')
+
+        for component_name in sorted(self.version1.component_versions):
+            v1 = self.version1.component_versions[component_name].version
+            v2 = ''
+            if component_name in self.version2.component_versions:
+                v2 = self.version2.component_versions[component_name].version
+            if v1 == v2:
+                print(f'| {component_name} | {v1} | {v2} |')
+            else:
+                print(f'| {component_name} | **{v1}** | {v2} |')