From 19674034429714992cba1edd0cc6e0fed03771ac Mon Sep 17 00:00:00 2001 From: Davide Palma Date: Thu, 11 Apr 2024 22:34:59 +0200 Subject: [PATCH] limit number of openable files (#131) Unlimited (e.g. 2**30) ulimit "nofile" will cause slow performance with programs that iterate all possible file descriptor, like fakeroot. Let extract.py spawn the container with a sane default ulimit, overriding host's limit, and let the container itself warn if it was started with unsane limits. --- extract.py | 2 +- fact_extractor/docker_extraction.py | 3 ++- fact_extractor/helperFunctions/program_setup.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/extract.py b/extract.py index 9ef11333..dd065af6 100755 --- a/extract.py +++ b/extract.py @@ -82,7 +82,7 @@ def call_docker(input_file, container, target, report_file, memory_limit, tmpdir shutil.copy(input_file, str(Path(tmpdir.name, 'input', Path(input_file).name))) - command = f'docker run --rm -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}' + command = f'docker run --rm --ulimit nofile=20000:50000 -m {memory_limit}m -v {tmpdir.name}:/tmp/extractor -v /dev:/dev --privileged {container} {arguments}' subprocess.run(command, shell=True) with suppress(shutil.Error): diff --git a/fact_extractor/docker_extraction.py b/fact_extractor/docker_extraction.py index 24724290..f38946d4 100755 --- a/fact_extractor/docker_extraction.py +++ b/fact_extractor/docker_extraction.py @@ -22,7 +22,7 @@ from helperFunctions.config import get_config_dir from helperFunctions.file_system import change_owner_of_output_files -from helperFunctions.program_setup import load_config, setup_logging +from helperFunctions.program_setup import check_ulimits, load_config, setup_logging from unpacker.unpack import unpack @@ -43,6 +43,7 @@ def _parse_args(): def main(args): config = load_config(f'{get_config_dir()}/main.cfg') setup_logging(debug=False) + check_ulimits() input_dir = Path(config.get('unpack', 'data_folder'), 'input') input_file = list(input_dir.iterdir())[0] diff --git a/fact_extractor/helperFunctions/program_setup.py b/fact_extractor/helperFunctions/program_setup.py index c5e2e0cc..994bdcec 100644 --- a/fact_extractor/helperFunctions/program_setup.py +++ b/fact_extractor/helperFunctions/program_setup.py @@ -1,6 +1,7 @@ import argparse import configparser import logging +import resource from common_helper_files import create_dir_for_file @@ -39,6 +40,16 @@ def setup_logging(debug, log_file=None, log_level=None): logger.addHandler(console_log) +def check_ulimits(): + # Get number of openable files + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft < 1024: + resource.setrlimit(resource.RLIMIT_NOFILE, (min(1024, hard), hard)) + logging.info(f'The number of openable files has been raised from {soft} to {min(1024, hard)}.') + elif soft == resource.RLIM_INFINITY or soft > 100000: + logging.warning('Warning: A very high (or no) nofile limit will slow down fakeroot and cause other problems.') + + def load_config(config_file): config = configparser.ConfigParser() config.read(config_file)