rpmbuilds: dynamically specify tmp_fs size

This moves specifying tmp fs size to rpmbuild from [1] in order to be
able to automatically generate its size for performance builders.

[1] - https://pagure.io/fedora-infra/ansible/blob/main/f/roles/copr/backend/files/provision/files/mock/site-defaults.cfg#_18

See fedora-copr#3268

rpmbuild/copr-rpmbuild.spec

@@ -40,6 +40,7 @@ BuildRequires: %{python}-requests
 BuildRequires: %{python_pfx}-jinja2
 BuildRequires: %{python_pfx}-specfile >= 0.21.0
 BuildRequires: python3-backoff >= 1.9.0
+BuildRequires: python3-pyyaml
 
 BuildRequires: /usr/bin/argparse-manpage
 BuildRequires: python-rpm-macros
@@ -57,6 +58,7 @@ Requires: %{python_pfx}-munch
 Requires: %{python}-requests
 Requires: %{python_pfx}-specfile >= 0.21.0
 Requires: python3-backoff >= 1.9.0
+Requires: python3-pyyaml
 
 Requires: mock >= 5.0
 Requires: git

rpmbuild/copr_rpmbuild/builders/mock.py

@@ -12,14 +12,15 @@
     get_mock_uniqueext,
     GentlyTimeoutedPopen,
     macros_for_task,
+    mock_snippets_for_tags,
 )
 
 log = logging.getLogger("__main__")
 
 MOCK_CALL = ['unbuffer', 'mock']
 
 class MockBuilder(object):
-    def __init__(self, task, sourcedir, resultdir, config):
+    def __init__(self, task, sourcedir, resultdir, config, rpmbuild_config):
         self.task_id = task.get("task_id")
         self.build_id = re.sub("-.*", "", self.task_id)
         self.chroot = task.get("chroot")
@@ -42,6 +43,9 @@ def __init__(self, task, sourcedir, resultdir, config):
         self.macros = macros_for_task(task, config)
         self.uniqueext = get_mock_uniqueext()
         self.allow_user_ssh = task.get("allow_user_ssh")
+        self.tags = task.get("tags", [])
+
+        self.rpmbuild_config = rpmbuild_config
 
     def run(self):
         open(self.logfile, 'w').close() # truncate logfile
@@ -82,6 +86,9 @@ def render_config_template(self):
             copr_build_id=self.build_id,
             isolation=self.isolation,
             macros=self.macros,
+            mock_snippets=mock_snippets_for_tags(
+                self.rpmbuild_config.job_tag_to_mock_snippets, self.tags
+            ) or None,
         )
 
     def produce_srpm(self, spec, sources, resultdir):

rpmbuild/copr_rpmbuild/config.py

@@ -0,0 +1,17 @@
+import os
+import yaml
+
+CONFIG_PATH = "/etc/copr-rpmbuild/copr-rpmbuild.yaml",
+
+class Config:
+    def __init__(self):
+        self.job_tag_to_mock_snippets = []
+
+    def load_config(self):
+        if not os.path.exists(CONFIG_PATH):
+            return
+
+        with open(CONFIG_PATH, "r") as file:
+            config_data = yaml.safe_load(file) or {}
+
+        self.job_tag_to_mock_snippets = config_data.get('job_tag_to_mock_snippets', [])

rpmbuild/copr_rpmbuild/helpers.py

@@ -469,3 +469,15 @@ def package_version(name):
             return pkg_resources.require(name)[0].version
         except pkg_resources.DistributionNotFound:
             return "git"
+
+
+def mock_snippets_for_tags(job_tag_to_mock_snippets, tags):
+    """
+    Return a list of mock snippets for a given list of tags
+    """
+    mock_snippets = []
+    for tag in tags:
+        snippet = job_tag_to_mock_snippets.get(tag)
+        if snippet:
+            mock_snippets.append(snippet)
+    return mock_snippets

rpmbuild/copr_rpmbuild/providers/base.py

@@ -8,7 +8,7 @@
 
 from copr_common.request import SafeRequest
 from copr_rpmbuild.helpers import CONF_DIRS
-from copr_rpmbuild.helpers import run_cmd
+from copr_rpmbuild.helpers import run_cmd, mock_snippets_for_tags
 
 
 log = logging.getLogger("__main__")
@@ -18,7 +18,7 @@ class Provider(object):
     # pylint: disable=too-many-instance-attributes
     _safe_resultdir = None
 
-    def __init__(self, source_dict, config, macros=None, task=None):
+    def __init__(self, source_dict, config, rpmbuild_config, macros=None, task=None):
         self.source_dict = source_dict
         self.config = config
         self.request = SafeRequest(log=log)
@@ -51,6 +51,15 @@ def __init__(self, source_dict, config, macros=None, task=None):
             if e.errno != errno.EEXIST:
                 raise
 
+        self.rpmbuild_config = rpmbuild_config
+
+        # Mock snippets to render in the mock config
+        self.mock_snippets = None
+        if self.task is not None:
+            self.mock_snippets = mock_snippets_for_tags(
+                self.rpmbuild_config.job_tag_to_mock_snippets, self.task.get("tags")
+            ) or None,
+
         # Change home directory to workdir and create .rpmmacros there
         os.environ["HOME"] = self.workdir
         self.create_rpmmacros()
@@ -130,7 +139,7 @@ def render_mock_config_template(self, template_name):
         """
         jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS))
         template = jinja_env.get_template(template_name)
-        return template.render(macros=self.macros)
+        return template.render(macros=self.macros, mock_snippets=self.mock_snippets)
 
     def produce_srpm(self):
         """

rpmbuild/copr_rpmbuild/providers/custom.py

@@ -51,6 +51,7 @@ def render_mock_config_template(self, *_args):
             chroot=self.chroot,
             repos=self.repos,
             macros=self.macros,
+            mock_snippets=self.mock_snippets,
         )
 
     def produce_srpm(self):

rpmbuild/main.py

@@ -25,6 +25,7 @@
     locate_srpm,
     package_version,
 )
+from copr_rpmbuild.config import Config
 from six.moves.urllib.parse import urlparse, urljoin, urlencode
 
 log = logging.getLogger(__name__)
@@ -88,6 +89,7 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
     config = read_config(args.config)
+    rpmbuild_config = Config().load_config()
 
     if args.verbose:
         log.setLevel(logging.DEBUG)
@@ -133,7 +135,7 @@ def main():
         else:
             action = build_rpm
 
-        action(args, config)
+        action(args, config, rpmbuild_config=rpmbuild_config)
     except RequestError as ex:
         log.error("Network error: %s", ex)
         sys.exit(1)
@@ -158,14 +160,16 @@ def init(args, config):
         os.makedirs(resultdir)
 
 
-def produce_srpm(task, config):
+def produce_srpm(task, config, rpmbuild_config):
     """
     Use *Provider() classes to create source RPM in config.get("resultdir")
     """
     try:
         macros = macros_for_task(task, config)
         clazz = providers.factory(task["source_type"])
-        provider = clazz(task["source_json"], config, macros=macros, task=task)
+        provider = clazz(
+            task["source_json"], config, macros=macros, task=task, rpmbuild_config=rpmbuild_config
+        )
         provider.produce_srpm()
         provider.copy_insecure_results()
     finally:
@@ -204,7 +208,7 @@ def log_task(task):
     log.info("Task:\n"+pp.pformat(task)+'\n')
 
 
-def build_srpm(args, config):
+def build_srpm(args, config, rpmbuild_config):
     if args.chroot:
         raise RuntimeError("--chroot option is not supported with --srpm")
 
@@ -218,7 +222,7 @@ def build_srpm(args, config):
     task = get_task(args, config, build_config_url_path)
     log_task(task)
 
-    produce_srpm(task, config)
+    produce_srpm(task, config, rpmbuild_config)
 
     resultdir = config.get("main", "resultdir")
     log.info("Output: {0}".format(
@@ -233,7 +237,7 @@ def build_srpm(args, config):
     run_automation_tools(task, resultdir, None, log, config)
 
 
-def build_rpm(args, config):
+def build_rpm(args, config, rpmbuild_config):
     if not args.chroot:
         raise RuntimeError("Missing --chroot parameter")
 
@@ -261,14 +265,16 @@ def build_rpm(args, config):
             "clone_url": task["git_repo"],
             "committish": task["git_hash"],
         }
-        distgit = providers.DistGitProvider(source_json, config)
+        distgit = providers.DistGitProvider(source_json, config, rpmbuild_config=rpmbuild_config)
 
         # Just clone and download sources, don't create source RPM (aka
         # produce_srpm).  We want to create the source RPM using Mock
         # in the target chroot.
         distgit.produce_sources()
         resultdir = config.get("main", "resultdir")
-        builder = MockBuilder(task, distgit.clone_to, resultdir, config)
+        builder = MockBuilder(
+            task, distgit.clone_to, resultdir, config, rpmbuild_config=rpmbuild_config
+        )
         builder.run()
         builder.touch_success_file()
         run_automation_tools(

rpmbuild/mock-custom-build.cfg.j2

@@ -16,6 +16,12 @@ config_opts["root"] = "copr-custom-" + config_opts["root"]
 # /bin/mock calls (when tmpfs_enable is on).
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 
+{%- if mock_snippets %}
+{% for snippet in mock_snippets %}
+{{ snippet }}
+{% endfor %}
+{% endif %}
+
 {%- for key, value in macros.items() %}
 config_opts['macros']['{{ key }}'] = '{{ value }}'
 {%- endfor %}

rpmbuild/mock.cfg.j2

@@ -4,6 +4,12 @@ config_opts.setdefault('plugin_conf', {})
 config_opts['plugin_conf'].setdefault('tmpfs_opts', {})
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 
+{%- if mock_snippets %}
+{% for snippet in mock_snippets %}
+{{ snippet }}
+{% endfor %}
+{% endif %}
+
 {% if buildroot_pkgs %}
 config_opts['chroot_additional_packages'] = '{{ buildroot_pkgs| join(" ") }}'
 {% endif %}

rpmbuild/tests/__init__.py

@@ -4,6 +4,7 @@
 import shutil
 
 from copr_rpmbuild import helpers
+from copr_rpmbuild.config import Config
 
 CONFIG = """
 [main]
@@ -45,6 +46,7 @@ def config_basic_dirs(self):
         self.workspace = os.path.join(self.workdir, "workspace")
         self.config.set("main", "resultdir", self.resultdir)
         self.config.set("main", "workspace", self.workspace)
+        self.rpmbuild_config = Config()
         os.makedirs(self.workspace)
         os.makedirs(self.resultdir)
 

rpmbuild/tests/test_base.py

@@ -49,7 +49,7 @@ def test_create_rpmmacros(self, mock_mkdir, mock_open):
     @mock.patch('copr_rpmbuild.providers.base.Provider.create_rpmmacros')
     def test_workdir_in_workspace(self, _mock_create_rpmmacros, _mock_mkdir):
         ws = self.config.get("main", "workspace")
-        provider = Provider(self.source_json, self.config)
+        provider = Provider(self.source_json, self.config, rpmbuild_config=self.rpmbuild_config)
         assert os.path.join(ws, "workdir-") in provider.workdir
 
     @staticmethod

rpmbuild/tests/test_distgit.py

@@ -89,7 +89,7 @@ def teardown_method(self, method):
 
     def test_distgit_method(self):
         source_dict = {"clone_url": self.origin}
-        dgp = DistGitProvider(source_dict, self.main_config)
+        dgp = DistGitProvider(source_dict, self.main_config, rpmbuild_config=self.rpmbuild_config)
         # this is normally created in main.py
         dgp.produce_srpm()
 

rpmbuild/tests/test_main.py

@@ -40,14 +40,14 @@ def auto_test_cleanup(self):
     @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.produce_srpm")
     def test_produce_srpm_cleanup_no(self, mock_produce_srpm, _cleanup):
         # Test that we cleanup after successful build
-        produce_srpm(self.task, self.config)
+        produce_srpm(self.task, self.config, rpmbuild_config=self.rpmbuild_config)
         # root + resultdir + workspace + not cleaned workdir
         directories = list(os.walk(self.workdir))
         assert len(directories) == 4
 
         mock_produce_srpm.side_effect = RuntimeError("Jeeez, something failed")
         with pytest.raises(RuntimeError):
-            produce_srpm(self.task, self.config)
+            produce_srpm(self.task, self.config, rpmbuild_config=self.rpmbuild_config)
 
         # .. and plus one more workdir
         directories = list(os.walk(self.workdir))
@@ -56,15 +56,15 @@ def test_produce_srpm_cleanup_no(self, mock_produce_srpm, _cleanup):
     @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.produce_srpm")
     def test_produce_srpm_cleanup_yes(self, mock_produce_srpm):
         # Test that we cleanup after successful build
-        produce_srpm(self.task, self.config)
+        produce_srpm(self.task, self.config, rpmbuild_config=self.rpmbuild_config)
 
         # root + resultdir + workspace (cleaned workdir)
         directories = list(os.walk(self.workdir))
         assert len(directories) == 3
 
         mock_produce_srpm.side_effect = RuntimeError("Jeeez, something failed")
         with pytest.raises(RuntimeError):
-            produce_srpm(self.task, self.config)
+            produce_srpm(self.task, self.config, rpmbuild_config=self.rpmbuild_config)
 
         # root + resultdir + workspace (cleaned workdir)
         directories = list(os.walk(self.workdir))