From 133840f941d803933188879f0a699cb401afbf0c Mon Sep 17 00:00:00 2001 From: Jiri Kyjovsky Date: Mon, 9 Sep 2024 16:41:16 +0200 Subject: [PATCH] rpmbuilds: dynamically specify tmp_fs size This moves specifying tmp fs size to rpmbuild from [1] in order to be able to automatically generate its size for performance builders. [1] - https://pagure.io/fedora-infra/ansible/blob/main/f/roles/copr/backend/files/provision/files/mock/site-defaults.cfg#_18 See #3268 --- rpmbuild/copr_rpmbuild/builders/mock.py | 9 ++++++++- rpmbuild/copr_rpmbuild/config.py | 17 +++++++++++++++++ rpmbuild/copr_rpmbuild/helpers.py | 12 ++++++++++++ rpmbuild/copr_rpmbuild/providers/base.py | 15 ++++++++++++--- rpmbuild/copr_rpmbuild/providers/custom.py | 1 + rpmbuild/main.py | 22 ++++++++++++++-------- rpmbuild/mock-custom-build.cfg.j2 | 6 ++++++ rpmbuild/mock.cfg.j2 | 6 ++++++ rpmbuild/tests/test_mock.py | 16 ++++++++++++++++ 9 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 rpmbuild/copr_rpmbuild/config.py diff --git a/rpmbuild/copr_rpmbuild/builders/mock.py b/rpmbuild/copr_rpmbuild/builders/mock.py index b3cc31461..2adf031e8 100644 --- a/rpmbuild/copr_rpmbuild/builders/mock.py +++ b/rpmbuild/copr_rpmbuild/builders/mock.py @@ -12,6 +12,7 @@ get_mock_uniqueext, GentlyTimeoutedPopen, macros_for_task, + mock_snippets_for_tags, ) log = logging.getLogger("__main__") @@ -19,7 +20,7 @@ MOCK_CALL = ['unbuffer', 'mock'] class MockBuilder(object): - def __init__(self, task, sourcedir, resultdir, config): + def __init__(self, task, sourcedir, resultdir, config, rpmbuild_config): self.task_id = task.get("task_id") self.build_id = re.sub("-.*", "", self.task_id) self.chroot = task.get("chroot") @@ -42,6 +43,9 @@ def __init__(self, task, sourcedir, resultdir, config): self.macros = macros_for_task(task, config) self.uniqueext = get_mock_uniqueext() self.allow_user_ssh = task.get("allow_user_ssh") + self.tags = task.get("tags", []) + + self.rpmbuild_config = rpmbuild_config def run(self): open(self.logfile, 'w').close() # truncate logfile @@ -82,6 +86,9 @@ def render_config_template(self): copr_build_id=self.build_id, isolation=self.isolation, macros=self.macros, + mock_snippets=mock_snippets_for_tags( + self.rpmbuild_config.job_tag_to_mock_snippets, self.tags + ) or None, ) def produce_srpm(self, spec, sources, resultdir): diff --git a/rpmbuild/copr_rpmbuild/config.py b/rpmbuild/copr_rpmbuild/config.py new file mode 100644 index 000000000..95adbea00 --- /dev/null +++ b/rpmbuild/copr_rpmbuild/config.py @@ -0,0 +1,17 @@ +import os +import yaml + +CONFIG_PATH = "/etc/copr-rpmbuild/copr-rpmbuild.yaml", + +class Config: + def __init__(self): + self.job_tag_to_mock_snippets = [] + + def load_config(self): + if not os.path.exists(CONFIG_PATH): + return + + with open(CONFIG_PATH, "r") as file: + config_data = yaml.safe_load(file) or {} + + self.job_tag_to_mock_snippets = config_data.get('job_tag_to_mock_snippets', []) diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index f18cba23f..a7e62cbcf 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -469,3 +469,15 @@ def package_version(name): return pkg_resources.require(name)[0].version except pkg_resources.DistributionNotFound: return "git" + + +def mock_snippets_for_tags(job_tag_to_mock_snippets, tags): + """ + Return a list of mock snippets for a given list of tags + """ + mock_snippets = [] + for tag in tags: + snippet = job_tag_to_mock_snippets.get(tag) + if snippet: + mock_snippets.append(snippet) + return mock_snippets diff --git a/rpmbuild/copr_rpmbuild/providers/base.py b/rpmbuild/copr_rpmbuild/providers/base.py index 8250e4036..b24c87ff0 100644 --- a/rpmbuild/copr_rpmbuild/providers/base.py +++ b/rpmbuild/copr_rpmbuild/providers/base.py @@ -8,7 +8,7 @@ from copr_common.request import SafeRequest from copr_rpmbuild.helpers import CONF_DIRS -from copr_rpmbuild.helpers import run_cmd +from copr_rpmbuild.helpers import run_cmd, mock_snippets_for_tags log = logging.getLogger("__main__") @@ -18,7 +18,7 @@ class Provider(object): # pylint: disable=too-many-instance-attributes _safe_resultdir = None - def __init__(self, source_dict, config, macros=None, task=None): + def __init__(self, source_dict, config, rpmbuild_config, macros=None, task=None): self.source_dict = source_dict self.config = config self.request = SafeRequest(log=log) @@ -51,6 +51,15 @@ def __init__(self, source_dict, config, macros=None, task=None): if e.errno != errno.EEXIST: raise + self.rpmbuild_config = rpmbuild_config + + # Mock snippets to render in the mock config + self.mock_snippets = None + if self.task is not None: + self.mock_snippets = mock_snippets_for_tags( + self.rpmbuild_config.job_tag_to_mock_snippets, self.task.get("tags") + ) or None, + # Change home directory to workdir and create .rpmmacros there os.environ["HOME"] = self.workdir self.create_rpmmacros() @@ -130,7 +139,7 @@ def render_mock_config_template(self, template_name): """ jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS)) template = jinja_env.get_template(template_name) - return template.render(macros=self.macros) + return template.render(macros=self.macros, mock_snippets=self.mock_snippets) def produce_srpm(self): """ diff --git a/rpmbuild/copr_rpmbuild/providers/custom.py b/rpmbuild/copr_rpmbuild/providers/custom.py index ce5469a62..e7335781b 100644 --- a/rpmbuild/copr_rpmbuild/providers/custom.py +++ b/rpmbuild/copr_rpmbuild/providers/custom.py @@ -51,6 +51,7 @@ def render_mock_config_template(self, *_args): chroot=self.chroot, repos=self.repos, macros=self.macros, + mock_snippets=self.mock_snippets, ) def produce_srpm(self): diff --git a/rpmbuild/main.py b/rpmbuild/main.py index ab4edb2d3..d48215a03 100755 --- a/rpmbuild/main.py +++ b/rpmbuild/main.py @@ -25,6 +25,7 @@ locate_srpm, package_version, ) +from copr_rpmbuild.config import Config from six.moves.urllib.parse import urlparse, urljoin, urlencode log = logging.getLogger(__name__) @@ -88,6 +89,7 @@ def main(): parser = get_parser() args = parser.parse_args() config = read_config(args.config) + rpmbuild_config = Config().load_config() if args.verbose: log.setLevel(logging.DEBUG) @@ -133,7 +135,7 @@ def main(): else: action = build_rpm - action(args, config) + action(args, config, rpmbuild_config=rpmbuild_config) except RequestError as ex: log.error("Network error: %s", ex) sys.exit(1) @@ -158,14 +160,16 @@ def init(args, config): os.makedirs(resultdir) -def produce_srpm(task, config): +def produce_srpm(task, config, rpmbuild_config): """ Use *Provider() classes to create source RPM in config.get("resultdir") """ try: macros = macros_for_task(task, config) clazz = providers.factory(task["source_type"]) - provider = clazz(task["source_json"], config, macros=macros, task=task) + provider = clazz( + task["source_json"], config, macros=macros, task=task, rpmbuild_config=rpmbuild_config + ) provider.produce_srpm() provider.copy_insecure_results() finally: @@ -204,7 +208,7 @@ def log_task(task): log.info("Task:\n"+pp.pformat(task)+'\n') -def build_srpm(args, config): +def build_srpm(args, config, rpmbuild_config): if args.chroot: raise RuntimeError("--chroot option is not supported with --srpm") @@ -218,7 +222,7 @@ def build_srpm(args, config): task = get_task(args, config, build_config_url_path) log_task(task) - produce_srpm(task, config) + produce_srpm(task, config, rpmbuild_config) resultdir = config.get("main", "resultdir") log.info("Output: {0}".format( @@ -233,7 +237,7 @@ def build_srpm(args, config): run_automation_tools(task, resultdir, None, log, config) -def build_rpm(args, config): +def build_rpm(args, config, rpmbuild_config): if not args.chroot: raise RuntimeError("Missing --chroot parameter") @@ -261,14 +265,16 @@ def build_rpm(args, config): "clone_url": task["git_repo"], "committish": task["git_hash"], } - distgit = providers.DistGitProvider(source_json, config) + distgit = providers.DistGitProvider(source_json, config, rpmbuild_config=rpmbuild_config) # Just clone and download sources, don't create source RPM (aka # produce_srpm). We want to create the source RPM using Mock # in the target chroot. distgit.produce_sources() resultdir = config.get("main", "resultdir") - builder = MockBuilder(task, distgit.clone_to, resultdir, config) + builder = MockBuilder( + task, distgit.clone_to, resultdir, config, rpmbuild_config=rpmbuild_config + ) builder.run() builder.touch_success_file() run_automation_tools( diff --git a/rpmbuild/mock-custom-build.cfg.j2 b/rpmbuild/mock-custom-build.cfg.j2 index c71e85609..15a8d998a 100644 --- a/rpmbuild/mock-custom-build.cfg.j2 +++ b/rpmbuild/mock-custom-build.cfg.j2 @@ -16,6 +16,12 @@ config_opts["root"] = "copr-custom-" + config_opts["root"] # /bin/mock calls (when tmpfs_enable is on). config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +{%- if mock_snippets %} +{% for snippet in mock_snippets %} +{{ snippet }} +{% endfor %} +{% endif %} + {%- for key, value in macros.items() %} config_opts['macros']['{{ key }}'] = '{{ value }}' {%- endfor %} diff --git a/rpmbuild/mock.cfg.j2 b/rpmbuild/mock.cfg.j2 index 25f3bbf19..3043b482b 100644 --- a/rpmbuild/mock.cfg.j2 +++ b/rpmbuild/mock.cfg.j2 @@ -4,6 +4,12 @@ config_opts.setdefault('plugin_conf', {}) config_opts['plugin_conf'].setdefault('tmpfs_opts', {}) config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +{%- if mock_snippets %} +{% for snippet in mock_snippets %} +{{ snippet }} +{% endfor %} +{% endif %} + {% if buildroot_pkgs %} config_opts['chroot_additional_packages'] = '{{ buildroot_pkgs| join(" ") }}' {% endif %} diff --git a/rpmbuild/tests/test_mock.py b/rpmbuild/tests/test_mock.py index fe43e0bda..7c5915176 100644 --- a/rpmbuild/tests/test_mock.py +++ b/rpmbuild/tests/test_mock.py @@ -146,6 +146,7 @@ def test_mock_config(self, call, f_mock_calls): config_opts.setdefault('plugin_conf', {}) config_opts['plugin_conf'].setdefault('tmpfs_opts', {}) config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '140g' config_opts['chroot_additional_packages'] = 'pkg1 pkg2 pkg3' @@ -164,6 +165,21 @@ def test_mock_config(self, call, f_mock_calls): """ # TODO: make the output nicer + @mock.patch("copr_rpmbuild.builders.mock.subprocess.call") + # pylint: disable-next=unused-argument, redefined-outer-name + def test_mock_config_hp_fs_size(self, call, f_mock_calls): + """ test that fs_size for performance builders is correctly set """ + self.task["tags"] = ["blabla_tag", "on_demand_powerful"] + MockBuilder(self.task, self.sourcedir, self.resultdir, + self.config).run() + + #config = open(self.child_config, 'r').readlines() + with open(self.child_config, "r", encoding="utf-8") as f: + config = f.readlines() + + config = ''.join(config) + assert "config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '280g'" in config + @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.prepare_configs") @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.archive_configs") def test_mock_options(self, archive_configs, prep_configs, f_mock_calls):