From c217010bba61bbf8587030d4a4cb98fd7ced3ef7 Mon Sep 17 00:00:00 2001 From: tomasfratrik Date: Tue, 18 Jun 2024 10:22:35 +0200 Subject: [PATCH] Fix IPU being blocked by resource limitations First resource limit is maximum number of open file descriptors limit, second one being limit for maximum writable file size. Plus add unit tests. Resolves: RHEL-26459 and RHEL-16881 --- commands/command_utils.py | 38 ++++++++++++++++++ commands/preupgrade/__init__.py | 2 + commands/tests/test_upgrade_paths.py | 60 ++++++++++++++++++++++++++++ commands/upgrade/__init__.py | 3 ++ 4 files changed, 103 insertions(+) diff --git a/commands/command_utils.py b/commands/command_utils.py index 4f6f99eb09..2810a542cb 100644 --- a/commands/command_utils.py +++ b/commands/command_utils.py @@ -1,6 +1,7 @@ import json import os import re +import resource from leapp.exceptions import CommandError from leapp.utils import path @@ -140,3 +141,40 @@ def vet_upgrade_path(args): flavor=flavor, choices=','.join(supported_target_versions))) return (target_release, flavor) + + +def set_resource_limits(): + """ + Set resource limits for the maximum number of open file descriptors and the maximum writable file size. + + :raises: `CommandError` if the resource limits cannot be set + """ + + def set_resource_limit(resource_type, soft, hard): + rtype_string = ( + 'open file descriptors' if resource_type == resource.RLIMIT_NOFILE + else 'writable file size' if resource_type == resource.RLIMIT_FSIZE + else 'unknown resource' + ) + try: + resource.setrlimit(resource_type, (soft, hard)) + except ValueError as err: + raise CommandError( + 'Failure occurred while attempting to set soft limit higher than the hard limit. ' + 'Resource type: {}, error: {}'.format(rtype_string, err) + ) + except OSError as err: + raise CommandError( + 'Failed to set resource limit. Resource type: {}, error: {}'.format(rtype_string, err) + ) + + soft_nofile, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + soft_fsize, _ = resource.getrlimit(resource.RLIMIT_FSIZE) + nofile_limit = 1024*16 + fsize_limit = resource.RLIM_INFINITY + + if soft_nofile < nofile_limit: + set_resource_limit(resource.RLIMIT_NOFILE, nofile_limit, nofile_limit) + + if soft_fsize != fsize_limit: + set_resource_limit(resource.RLIMIT_FSIZE, fsize_limit, fsize_limit) diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py index 5a89069f6a..a9fa40e077 100644 --- a/commands/preupgrade/__init__.py +++ b/commands/preupgrade/__init__.py @@ -59,6 +59,8 @@ def preupgrade(args, breadcrumbs): except LeappError as exc: raise CommandError(exc.message) + command_utils.set_resource_limits() + workflow = repositories.lookup_workflow('IPUWorkflow')() util.warn_if_unsupported(configuration) util.process_whitelist_experimental(repositories, workflow, configuration, logger) diff --git a/commands/tests/test_upgrade_paths.py b/commands/tests/test_upgrade_paths.py index 53f081a56c..f1312f6685 100644 --- a/commands/tests/test_upgrade_paths.py +++ b/commands/tests/test_upgrade_paths.py @@ -1,3 +1,5 @@ +import resource + import mock import pytest @@ -50,3 +52,61 @@ def test_vet_upgrade_path(mock_open, monkeypatch): monkeypatch.setenv('LEAPP_DEVEL_TARGET_RELEASE', '9.0') args = mock.Mock(target='1.2') assert command_utils.vet_upgrade_path(args) == ('9.0', 'default') + + +def _mock_getrlimit_factory(nofile_limits=(1024, 4096), fsize_limits=(1024, 4096)): + """ + Factory function to create a mock `getrlimit` function with configurable return values. + The default param values are lower than the expected values. + + :param nofile_limits: Tuple representing (soft, hard) limits for `RLIMIT_NOFILE` + :param fsize_limits: Tuple representing (soft, hard) limits for `RLIMIT_FSIZE` + :return: A mock `getrlimit` function + """ + def mock_getrlimit(resource_type): + if resource_type == resource.RLIMIT_NOFILE: + return nofile_limits + if resource_type == resource.RLIMIT_FSIZE: + return fsize_limits + return (0, 0) + + return mock_getrlimit + + +@pytest.mark.parametrize("nofile_limits, fsize_limits, expected_calls", [ + # Case where both limits need to be increased + ((1024, 4096), (1024, 4096), [ + (resource.RLIMIT_NOFILE, (1024*16, 1024*16)), + (resource.RLIMIT_FSIZE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) + ]), + # Case where neither limit needs to be changed + ((1024*16, 1024*16), (resource.RLIM_INFINITY, resource.RLIM_INFINITY), []) +]) +def test_set_resource_limits_increase(monkeypatch, nofile_limits, fsize_limits, expected_calls): + setrlimit_called = [] + + def mock_setrlimit(resource_type, limits): + setrlimit_called.append((resource_type, limits)) + + monkeypatch.setattr(resource, "getrlimit", _mock_getrlimit_factory(nofile_limits, fsize_limits)) + monkeypatch.setattr(resource, "setrlimit", mock_setrlimit) + + command_utils.set_resource_limits() + + assert setrlimit_called == expected_calls + + +@pytest.mark.parametrize("errortype, expected_message", [ + (OSError, "Failed to set resource limit"), + (ValueError, "Failure occurred while attempting to set soft limit higher than the hard limit") +]) +def test_set_resource_limits_exceptions(monkeypatch, errortype, expected_message): + monkeypatch.setattr(resource, "getrlimit", _mock_getrlimit_factory()) + + def mock_setrlimit(*args, **kwargs): + raise errortype("mocked error") + + monkeypatch.setattr(resource, "setrlimit", mock_setrlimit) + + with pytest.raises(CommandError, match=expected_message): + command_utils.set_resource_limits() diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 1e15b59c45..c7487fded8 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -89,6 +89,9 @@ def upgrade(args, breadcrumbs): repositories = util.load_repositories() except LeappError as exc: raise CommandError(exc.message) + + command_utils.set_resource_limits() + workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration)