diff --git a/.github/workflows/python/github.py b/.github/workflows/python/github.py index 89e72b5991..1abbbb03c9 100644 --- a/.github/workflows/python/github.py +++ b/.github/workflows/python/github.py @@ -31,20 +31,20 @@ def __init__(self): def get_reviews(self): """ Get a list of reviews on a Github pull request as json object """ - reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id)) + reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews') reviews.raise_for_status() return reviews.json() def update_review(self, review_id, body): """ Update a review given by `review_id` and set its body to `body` """ payload = {'body': body} - resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload) + resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload) resp.raise_for_status() return resp.json() def post_review(self, body): """ Post a pull request review containing `body` and requesting changes """ payload = {'body': body, 'event': "REQUEST_CHANGES"} - resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload) + resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload) resp.raise_for_status() return resp.json() diff --git a/.github/workflows/python/pycodestyle_comment.py b/.github/workflows/python/pycodestyle_comment.py index 89eca1936f..4cce4d4028 100644 --- a/.github/workflows/python/pycodestyle_comment.py +++ b/.github/workflows/python/pycodestyle_comment.py @@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str: """ Format the list of pycodestyle errors and return them a one string. """ ret = '' for error in style_error_list: - ret += '* {}\n'.format(error) + ret += f'* {error}\n' return ret @@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str: style_errors = list_style_errors() if style_errors: - print("Found {} errors.".format(len(style_errors))) + print(f"Found {len(style_errors)} errors.") gh = github.Github() diff --git a/.github/workflows/scripts/setup-full.sh b/.github/workflows/scripts/setup-full.sh index 63e9353952..e546c0e459 100644 --- a/.github/workflows/scripts/setup-full.sh +++ b/.github/workflows/scripts/setup-full.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: 2020 Birger Schacht +# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology # SPDX-License-Identifier: AGPL-3.0-or-later set -x @@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch sudo systemctl start elasticsearch +sudo apt update +if [ $python_version == '3.8' ]; then + # for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539 + DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev +fi +# for psql (used below) +DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client-14 + # Install the dependencies of all the bots pip install wheel for file in intelmq/bots/*/*/REQUIREMENTS.txt; do @@ -30,7 +38,16 @@ done # Setup sudo and install intelmq sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers sudo pip install . -sudo intelmqsetup --skip-ownership + +intelmq_user_exists=$(getent passwd intelmq ||:) +if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then + # create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work + groupadd -r intelmq + useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq + sudo intelmqsetup +else + sudo intelmqsetup --skip-ownership +fi # Initialize the postgres database intelmq_psql_initdb diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 1dee5735f5..4325e96b1c 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] type: ['full', 'basic'] services: @@ -59,6 +59,7 @@ jobs: PGPORT: 5432 PGUSER: intelmq PGPASSWORD: intelmq + python_version: ${{ matrix.python-version }} run: bash .github/workflows/scripts/setup-full.sh - name: Install test dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b31d70355..3bb8766f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Configuration ### Core +- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner). ### Development @@ -29,12 +30,20 @@ #### Experts #### Outputs +- `intelmq.bots.outputs.cif3.output`: + - The requirement can only be installed on Python version < 3.12. + - Add a check on the Python version and exit if incompatible. + - Add a deprecation warning (PR#2544 by Sebastian Wagner) ### Documentation ### Packaging ### Tests +- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner). +- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner). +- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner). +- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner). ### Tools diff --git a/NEWS.md b/NEWS.md index 6f79b6ea7d..a3c242979a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes. --------------------------------- ### Requirements +Python 3.8 or newer is required. + +## Bots +#### CIF 3 API +The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance. +See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information. ### Tools @@ -36,7 +42,7 @@ No changes are required by administrators. ---------------------------------- ### Documentation -The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. +The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. We now have a slick, modern mkdocs based documentation. Please do check it out! @@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can: - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema) - - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) + - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) - parse the new reports **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here: @@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP ### General changes and bug fixes Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks! -This speeds up IntelMQ API calls. +This speeds up IntelMQ API calls. ### Data Format diff --git a/contrib/example-extension-package/setup.py b/contrib/example-extension-package/setup.py index 3d6774693f..d662eca34d 100644 --- a/contrib/example-extension-package/setup.py +++ b/contrib/example-extension-package/setup.py @@ -28,7 +28,7 @@ entry_point = '.'.join(file.with_suffix('').parts) file = Path(str(file).replace('intelmq/bots', 'mybots/bots')) module = '.'.join(file.with_suffix('').parts) - BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module)) + BOTS.append(f'{entry_point} = {module}:BOT.run') setup( name='intelmq-example-extension', diff --git a/intelmq/bots/collectors/rsync/collector_rsync.py b/intelmq/bots/collectors/rsync/collector_rsync.py index 112deb1f17..3829c8a3f8 100644 --- a/intelmq/bots/collectors/rsync/collector_rsync.py +++ b/intelmq/bots/collectors/rsync/collector_rsync.py @@ -13,7 +13,7 @@ from intelmq.lib.bot import CollectorBot -class Time(object): +class Time: def __init__(self, delta=None): """ Delta is a datetime.timedelta JSON string, ex: '{days=-1}'. """ self.time = datetime.now() diff --git a/intelmq/bots/experts/modify/expert.py b/intelmq/bots/experts/modify/expert.py index a9945449a4..f6cdbbb439 100644 --- a/intelmq/bots/experts/modify/expert.py +++ b/intelmq/bots/experts/modify/expert.py @@ -17,10 +17,7 @@ def is_re_pattern(value): """ Checks if the given value is a re compiled pattern """ - if sys.version_info > (3, 7): - return isinstance(value, re.Pattern) - else: - return hasattr(value, "pattern") + return isinstance(value, re.Pattern) class MatchGroupMapping: diff --git a/intelmq/bots/outputs/cif3/REQUIREMENTS.txt b/intelmq/bots/outputs/cif3/REQUIREMENTS.txt index 9b898b618a..0a3bef49d9 100644 --- a/intelmq/bots/outputs/cif3/REQUIREMENTS.txt +++ b/intelmq/bots/outputs/cif3/REQUIREMENTS.txt @@ -1,4 +1,4 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -# SPDX-FileCopyrightText: 2022 REN-ISAC +# SPDX-FileCopyrightText: 2022 REN-ISAC, 2024 Institute for Common Good Technology -cifsdk>=3.0.0rc4,<4.0 +cifsdk>=3.0.0rc4,<4.0; python_version < '3.12' diff --git a/intelmq/bots/outputs/cif3/output.py b/intelmq/bots/outputs/cif3/output.py index 537ac40b13..f7682c4107 100644 --- a/intelmq/bots/outputs/cif3/output.py +++ b/intelmq/bots/outputs/cif3/output.py @@ -32,6 +32,7 @@ JsonLib = None from datetime import datetime +from sys import version_info from typing import Optional, List from intelmq.lib.bot import OutputBot @@ -91,6 +92,9 @@ class CIF3OutputBot(OutputBot): _is_multithreadable = False def init(self): + raise DeprecationWarning("The CIFv3 API is deprecated and will be remove in IntelMQ version 4.0. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html") + if version_info >= (3, 12): + raise ValueError("This bot is not compatible with Python >= 3.12. See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html") try: cifsdk_version = int(get_cifsdk_version().get('version').split('.')[0]) except NameError: diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py index d24d114671..5661ea7339 100644 --- a/intelmq/bots/outputs/smtp_batch/output.py +++ b/intelmq/bots/outputs/smtp_batch/output.py @@ -123,7 +123,7 @@ def cli_run(self): with open(self.mail_template) as f: self.mail_contents = f.read() if self.alternative_mails: - with open(self.alternative_mails, "r") as f: + with open(self.alternative_mails) as f: self.alternative_mail = {row[0]: row[1] for row in csv.reader(f, delimiter=",")} print("Preparing mail queue...") diff --git a/intelmq/tests/lib/test_pipeline.py b/intelmq/tests/lib/test_pipeline.py index 39f75eb0fb..51c460668c 100644 --- a/intelmq/tests/lib/test_pipeline.py +++ b/intelmq/tests/lib/test_pipeline.py @@ -266,8 +266,8 @@ def test_reject(self): self.pipe.reject_message() self.assertEqual(SAMPLES['normal'][1], self.pipe.receive()) - @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] == (3, 8), - 'Fails on CI with Python 3.8') + @unittest.skipIf(os.getenv('CI') == 'true' and sys.version_info[:2] in ((3, 8), (3, 11)), + 'Fails on CI with Python 3.8 and 3.11') def test_acknowledge(self): self.pipe.send(SAMPLES['normal'][0]) self.pipe.receive() diff --git a/setup.py b/setup.py index 7bbe279fc2..55725c37e2 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ 'redis>=2.10', 'requests>=2.2.0', 'ruamel.yaml', - 'importlib-metadata; python_version < "3.8"' ] TESTS_REQUIRES = [ @@ -56,7 +55,7 @@ version=__version__, # noqa: F821 maintainer='Sebastian Wagner', maintainer_email='intelmq-dev@lists.cert.at', - python_requires='>=3.7', + python_requires='>=3.8', install_requires=REQUIRES, tests_require=TESTS_REQUIRES, test_suite='intelmq.tests', @@ -85,11 +84,11 @@ 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security',