From 48fd69eb75bdf1ecfc653949e5619331355218d8 Mon Sep 17 00:00:00 2001 From: Jiri Kyjovsky Date: Tue, 17 Sep 2024 14:35:28 +0200 Subject: [PATCH] rpmbuild: specify snippets to mock config via copr-rpmbuild config file This allows us to specify tpm fs size to rpmbuild in order to be able to automatically generate its size for performance builders. See #3268 --- rpmbuild/copr-rpmbuild.spec | 4 ++++ rpmbuild/copr-rpmbuild.yml | 4 ++++ rpmbuild/copr_rpmbuild/builders/mock.py | 9 +++++++ rpmbuild/copr_rpmbuild/config.py | 28 ++++++++++++++++++++++ rpmbuild/copr_rpmbuild/helpers.py | 16 +++++++++++++ rpmbuild/copr_rpmbuild/providers/base.py | 15 ++++++++++-- rpmbuild/copr_rpmbuild/providers/custom.py | 1 + rpmbuild/mock-custom-build.cfg.j2 | 3 +++ rpmbuild/mock.cfg.j2 | 3 +++ rpmbuild/tests/test_mock.py | 24 +++++++++++++++++++ 10 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 rpmbuild/copr-rpmbuild.yml create mode 100644 rpmbuild/copr_rpmbuild/config.py diff --git a/rpmbuild/copr-rpmbuild.spec b/rpmbuild/copr-rpmbuild.spec index 605e90da9..f0e92d2a6 100644 --- a/rpmbuild/copr-rpmbuild.spec +++ b/rpmbuild/copr-rpmbuild.spec @@ -40,6 +40,7 @@ BuildRequires: %{python}-requests BuildRequires: %{python_pfx}-jinja2 BuildRequires: %{python_pfx}-specfile >= 0.21.0 BuildRequires: python3-backoff >= 1.9.0 +BuildRequires: python3-pyyaml BuildRequires: /usr/bin/argparse-manpage BuildRequires: python-rpm-macros @@ -57,6 +58,7 @@ Requires: %{python_pfx}-munch Requires: %{python}-requests Requires: %{python_pfx}-specfile >= 0.21.0 Requires: python3-backoff >= 1.9.0 +Requires: python3-pyyaml Requires: mock >= 5.0 Requires: git @@ -213,6 +215,7 @@ install -m 644 mock.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/mock.cfg.j2 install -m 644 rpkg.conf.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2 install -m 644 mock-source-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/ install -m 644 mock-custom-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/ +install -m 644 copr-rpmbuild.yml %{buildroot}%{_sysconfdir}/copr-rpmbuild/copr-rpmbuild.yml cat < %buildroot%mock_config_overrides/README Contents of this directory is used by %_bindir/copr-update-builder script. @@ -264,6 +267,7 @@ install -p -m 755 copr-update-builder %buildroot%_bindir %config(noreplace) %{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2 %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-source-build.cfg.j2 %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-custom-build.cfg.j2 +%config(noreplace) %{_sysconfdir}/copr-rpmbuild/copr-rpmbuild.yml %files -n copr-builder %license LICENSE diff --git a/rpmbuild/copr-rpmbuild.yml b/rpmbuild/copr-rpmbuild.yml new file mode 100644 index 000000000..cd3d2d236 --- /dev/null +++ b/rpmbuild/copr-rpmbuild.yml @@ -0,0 +1,4 @@ +--- +# Configure special mock configuration snippets per given set of tags. +# job_tag_to_mock_snippets: +# - on_demand_powerful: config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '280g' diff --git a/rpmbuild/copr_rpmbuild/builders/mock.py b/rpmbuild/copr_rpmbuild/builders/mock.py index b3cc31461..5ebefb262 100644 --- a/rpmbuild/copr_rpmbuild/builders/mock.py +++ b/rpmbuild/copr_rpmbuild/builders/mock.py @@ -12,7 +12,9 @@ get_mock_uniqueext, GentlyTimeoutedPopen, macros_for_task, + mock_snippet_for_tags, ) +from ..config import Config log = logging.getLogger("__main__") @@ -42,6 +44,10 @@ def __init__(self, task, sourcedir, resultdir, config): self.macros = macros_for_task(task, config) self.uniqueext = get_mock_uniqueext() self.allow_user_ssh = task.get("allow_user_ssh") + self.tags = task.get("tags", []) + + self.copr_rpmbuild_config = Config() + self.copr_rpmbuild_config.load_config() def run(self): open(self.logfile, 'w').close() # truncate logfile @@ -82,6 +88,9 @@ def render_config_template(self): copr_build_id=self.build_id, isolation=self.isolation, macros=self.macros, + mock_snippet=mock_snippet_for_tags( + self.copr_rpmbuild_config.job_tag_to_mock_snippets, self.tags + ), ) def produce_srpm(self, spec, sources, resultdir): diff --git a/rpmbuild/copr_rpmbuild/config.py b/rpmbuild/copr_rpmbuild/config.py new file mode 100644 index 000000000..40b6e15a4 --- /dev/null +++ b/rpmbuild/copr_rpmbuild/config.py @@ -0,0 +1,28 @@ +""" +Configuration class for copr-rpmbuild +""" + + +import os +import yaml + +CONFIG_PATH = "/etc/copr-rpmbuild/copr-rpmbuild.yaml" + +class Config: + """ + Configuration class for copr-rpmbuild + """ + def __init__(self): + self.job_tag_to_mock_snippets = {} + + def load_config(self): + """ + Load configuration from the config file + """ + if not os.path.exists(CONFIG_PATH): + return + + with open(CONFIG_PATH, "r", encoding="utf-8") as file: + config_data = yaml.safe_load(file) or {} + + self.job_tag_to_mock_snippets = config_data.get('job_tag_to_mock_snippets', {}) diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index f18cba23f..6126738c6 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -469,3 +469,19 @@ def package_version(name): return pkg_resources.require(name)[0].version except pkg_resources.DistributionNotFound: return "git" + + +def mock_snippet_for_tags(job_tag_to_mock_snippets, tags): + """ + Return mock snippets as string separated by newlines for a given + list of tags. + """ + if not tags or not job_tag_to_mock_snippets: + return "" + + mock_snippets = [] + for tag in tags: + snippet = job_tag_to_mock_snippets.get(tag) + if snippet: + mock_snippets.append(snippet) + return "\n".join(mock_snippets) diff --git a/rpmbuild/copr_rpmbuild/providers/base.py b/rpmbuild/copr_rpmbuild/providers/base.py index 8250e4036..aa50021ac 100644 --- a/rpmbuild/copr_rpmbuild/providers/base.py +++ b/rpmbuild/copr_rpmbuild/providers/base.py @@ -8,7 +8,8 @@ from copr_common.request import SafeRequest from copr_rpmbuild.helpers import CONF_DIRS -from copr_rpmbuild.helpers import run_cmd +from copr_rpmbuild.helpers import run_cmd, mock_snippet_for_tags +from copr_rpmbuild.config import Config log = logging.getLogger("__main__") @@ -51,6 +52,16 @@ def __init__(self, source_dict, config, macros=None, task=None): if e.errno != errno.EEXIST: raise + self.copr_rpmbuild_config = Config() + self.copr_rpmbuild_config.load_config() + + # Mock snippets to render in the mock config + self.mock_snippet = "" + if self.task is not None: + self.mock_snippet = mock_snippet_for_tags( + self.copr_rpmbuild_config.job_tag_to_mock_snippets, self.task.get("tags") + ) + # Change home directory to workdir and create .rpmmacros there os.environ["HOME"] = self.workdir self.create_rpmmacros() @@ -130,7 +141,7 @@ def render_mock_config_template(self, template_name): """ jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS)) template = jinja_env.get_template(template_name) - return template.render(macros=self.macros) + return template.render(macros=self.macros, mock_snippet=self.mock_snippet) def produce_srpm(self): """ diff --git a/rpmbuild/copr_rpmbuild/providers/custom.py b/rpmbuild/copr_rpmbuild/providers/custom.py index ce5469a62..2bf93cb9e 100644 --- a/rpmbuild/copr_rpmbuild/providers/custom.py +++ b/rpmbuild/copr_rpmbuild/providers/custom.py @@ -51,6 +51,7 @@ def render_mock_config_template(self, *_args): chroot=self.chroot, repos=self.repos, macros=self.macros, + mock_snippet=self.mock_snippet, ) def produce_srpm(self): diff --git a/rpmbuild/mock-custom-build.cfg.j2 b/rpmbuild/mock-custom-build.cfg.j2 index c71e85609..1c6d98615 100644 --- a/rpmbuild/mock-custom-build.cfg.j2 +++ b/rpmbuild/mock-custom-build.cfg.j2 @@ -16,6 +16,9 @@ config_opts["root"] = "copr-custom-" + config_opts["root"] # /bin/mock calls (when tmpfs_enable is on). config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +# Custom mock snippets configured in copr-crpmbuild config file - can be empty +{{ mock_snippet }} + {%- for key, value in macros.items() %} config_opts['macros']['{{ key }}'] = '{{ value }}' {%- endfor %} diff --git a/rpmbuild/mock.cfg.j2 b/rpmbuild/mock.cfg.j2 index 25f3bbf19..c0010056c 100644 --- a/rpmbuild/mock.cfg.j2 +++ b/rpmbuild/mock.cfg.j2 @@ -4,6 +4,9 @@ config_opts.setdefault('plugin_conf', {}) config_opts['plugin_conf'].setdefault('tmpfs_opts', {}) config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +# Custom mock snippets configured in copr-crpmbuild config file - can be empty +{{ mock_snippet }} + {% if buildroot_pkgs %} config_opts['chroot_additional_packages'] = '{{ buildroot_pkgs| join(" ") }}' {% endif %} diff --git a/rpmbuild/tests/test_mock.py b/rpmbuild/tests/test_mock.py index fe43e0bda..f97b7b4f2 100644 --- a/rpmbuild/tests/test_mock.py +++ b/rpmbuild/tests/test_mock.py @@ -147,6 +147,9 @@ def test_mock_config(self, call, f_mock_calls): config_opts['plugin_conf'].setdefault('tmpfs_opts', {}) config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True +# Custom mock snippets configured in copr-crpmbuild config file - can be empty + + config_opts['chroot_additional_packages'] = 'pkg1 pkg2 pkg3' @@ -164,6 +167,27 @@ def test_mock_config(self, call, f_mock_calls): """ # TODO: make the output nicer + @mock.patch("copr_rpmbuild.builders.mock.subprocess.call") + # pylint: disable-next=unused-argument, redefined-outer-name + def test_mock_config_hp_fs_size(self, call, f_mock_calls): + """ test that fs_size for performance builders is correctly set """ + self.task["tags"] = ["blabla_tag", "on_demand_powerful"] + builder = MockBuilder( + self.task, self.sourcedir, self.resultdir, self.config + ) + mock_snippet = "config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '280g'" + builder.copr_rpmbuild_config.job_tag_to_mock_snippets = { + "on_demand_powerful": mock_snippet + } + builder.run() + + with open(self.child_config, "r", encoding="utf-8") as f: + config = f.readlines() + + config = ''.join(config) + assert mock_snippet in config + + @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.prepare_configs") @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.archive_configs") def test_mock_options(self, archive_configs, prep_configs, f_mock_calls):