rpmbuilds: dynamically specify tmp_fs size

rpmbuild/copr-rpmbuild.spec

@@ -40,6 +40,7 @@ BuildRequires: %{python}-requests
 BuildRequires: %{python_pfx}-jinja2
 BuildRequires: %{python_pfx}-specfile >= 0.21.0
 BuildRequires: python3-backoff >= 1.9.0
+BuildRequires: python3-pyyaml
 
 BuildRequires: /usr/bin/argparse-manpage
 BuildRequires: python-rpm-macros
@@ -57,6 +58,7 @@ Requires: %{python_pfx}-munch
 Requires: %{python}-requests
 Requires: %{python_pfx}-specfile >= 0.21.0
 Requires: python3-backoff >= 1.9.0
+Requires: python3-pyyaml
 
 Requires: mock >= 5.0
 Requires: git
@@ -213,6 +215,7 @@ install -m 644 mock.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/mock.cfg.j2
 install -m 644 rpkg.conf.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2
 install -m 644 mock-source-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/
 install -m 644 mock-custom-build.cfg.j2 %{buildroot}%{_sysconfdir}/copr-rpmbuild/
+install -m 644 copr-rpmbuild.yml %{buildroot}%{_sysconfdir}/copr-rpmbuild/copr-rpmbuild.yml
 
 cat <<EOF > %buildroot%mock_config_overrides/README
 Contents of this directory is used by %_bindir/copr-update-builder script.
@@ -264,6 +267,7 @@ install -p -m 755 copr-update-builder %buildroot%_bindir
 %config(noreplace) %{_sysconfdir}/copr-rpmbuild/rpkg.conf.j2
 %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-source-build.cfg.j2
 %config(noreplace) %{_sysconfdir}/copr-rpmbuild/mock-custom-build.cfg.j2
+%config(noreplace) %{_sysconfdir}/copr-rpmbuild/copr-rpmbuild.yml
 
 %files -n copr-builder
 %license LICENSE

rpmbuild/copr-rpmbuild.yml

@@ -0,0 +1,14 @@
+---
+# Configure special mock configuration snippets per given set of tags.
+# tags_to_mock_snippet:
+#   - tagset:
+#       - on_demand_powerful
+#       - arch_x86_64
+#     snippet: config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '280g'
+#   - tagset:
+#       - on_demand_powerful
+#       - something
+#     snippet: |
+#       cute
+#       multiline
+#       snippet

rpmbuild/copr_rpmbuild/builders/mock.py

@@ -6,12 +6,14 @@
 import subprocess
 
 from jinja2 import Environment, FileSystemLoader
-from ..helpers import (
+from copr_rpmbuild.config import Config
+from copr_rpmbuild.helpers import (
     locate_spec,
     CONF_DIRS,
     get_mock_uniqueext,
     GentlyTimeoutedPopen,
     macros_for_task,
+    mock_snippet_for_tags,
 )
 
 log = logging.getLogger("__main__")
@@ -42,6 +44,10 @@ def __init__(self, task, sourcedir, resultdir, config):
         self.macros = macros_for_task(task, config)
         self.uniqueext = get_mock_uniqueext()
         self.allow_user_ssh = task.get("allow_user_ssh")
+        self.tags = task.get("tags", [])
+
+        self.copr_rpmbuild_config = Config()
+        self.copr_rpmbuild_config.load_config()
 
     def run(self):
         open(self.logfile, 'w').close() # truncate logfile
@@ -82,6 +88,9 @@ def render_config_template(self):
             copr_build_id=self.build_id,
             isolation=self.isolation,
             macros=self.macros,
+            mock_snippet=mock_snippet_for_tags(
+                self.copr_rpmbuild_config.tags_to_mock_snippet, self.tags
+            ),
         )
 
     def produce_srpm(self, spec, sources, resultdir):

rpmbuild/copr_rpmbuild/config.py

@@ -0,0 +1,29 @@
+"""
+Configuration class for copr-rpmbuild
+"""
+
+import yaml
+
+
+CONFIG_PATH = "/etc/copr-rpmbuild/copr-rpmbuild.yml"
+
+
+class Config:
+    """
+    Configuration class for copr-rpmbuild
+    """
+    def __init__(self):
+        self.tags_to_mock_snippet = []
+
+    def load_config(self):
+        """
+        Load configuration from the config file
+        """
+        config_data = {}
+        try:
+            with open(CONFIG_PATH, "r", encoding="utf-8") as file:
+                config_data = yaml.safe_load(file) or {}
+        except FileNotFoundError:
+            pass
+
+        self.tags_to_mock_snippet = config_data.get("tags_to_mock_snippet", [])

rpmbuild/copr_rpmbuild/helpers.py

@@ -469,3 +469,20 @@ def package_version(name):
             return pkg_resources.require(name)[0].version
         except pkg_resources.DistributionNotFound:
             return "git"
+
+
+def mock_snippet_for_tags(tags_to_mock_snippet, tags):
+    """
+    Return mock snippets as string separated by newlines for a given
+    list of tags.
+    """
+    if not tags or not tags_to_mock_snippet:
+        return ""
+
+    tags_set = set(tags)
+    snippets = []
+    for item in tags_to_mock_snippet:
+        if set(item["tagset"]).issubset(tags_set):
+            snippets.append(item["snippet"])
+
+    return "\n".join(snippets)

rpmbuild/copr_rpmbuild/providers/base.py

@@ -8,7 +8,8 @@
 
 from copr_common.request import SafeRequest
 from copr_rpmbuild.helpers import CONF_DIRS
-from copr_rpmbuild.helpers import run_cmd
+from copr_rpmbuild.helpers import run_cmd, mock_snippet_for_tags
+from copr_rpmbuild.config import Config
 
 
 log = logging.getLogger("__main__")
@@ -51,6 +52,16 @@ def __init__(self, source_dict, config, macros=None, task=None):
             if e.errno != errno.EEXIST:
                 raise
 
+        self.copr_rpmbuild_config = Config()
+        self.copr_rpmbuild_config.load_config()
+
+        # Mock snippets to render in the mock config
+        self.mock_snippet = ""
+        if self.task is not None:
+            self.mock_snippet = mock_snippet_for_tags(
+                self.copr_rpmbuild_config.tags_to_mock_snippet, self.task.get("tags")
+            )
+
         # Change home directory to workdir and create .rpmmacros there
         os.environ["HOME"] = self.workdir
         self.create_rpmmacros()
@@ -130,7 +141,7 @@ def render_mock_config_template(self, template_name):
         """
         jinja_env = Environment(loader=FileSystemLoader(CONF_DIRS))
         template = jinja_env.get_template(template_name)
-        return template.render(macros=self.macros)
+        return template.render(macros=self.macros, mock_snippet=self.mock_snippet)
 
     def produce_srpm(self):
         """

rpmbuild/copr_rpmbuild/providers/custom.py

@@ -51,6 +51,7 @@ def render_mock_config_template(self, *_args):
             chroot=self.chroot,
             repos=self.repos,
             macros=self.macros,
+            mock_snippet=self.mock_snippet,
         )
 
     def produce_srpm(self):

rpmbuild/mock-custom-build.cfg.j2

@@ -16,6 +16,9 @@ config_opts["root"] = "copr-custom-" + config_opts["root"]
 # /bin/mock calls (when tmpfs_enable is on).
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 
+# Custom mock snippets configured in copr-crpmbuild config file - can be empty
+{{ mock_snippet }}
+
 {%- for key, value in macros.items() %}
 config_opts['macros']['{{ key }}'] = '{{ value }}'
 {%- endfor %}

rpmbuild/mock.cfg.j2

@@ -4,6 +4,9 @@ config_opts.setdefault('plugin_conf', {})
 config_opts['plugin_conf'].setdefault('tmpfs_opts', {})
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 
+# Custom mock snippets configured in copr-crpmbuild config file - can be empty
+{{ mock_snippet }}
+
 {% if buildroot_pkgs %}
 config_opts['chroot_additional_packages'] = '{{ buildroot_pkgs| join(" ") }}'
 {% endif %}

rpmbuild/tests/test_base.py

@@ -18,7 +18,7 @@ def setUp(self):
         super(TestProvider, self).setUp()
         self.source_json = {}
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_create_rpmmacros(self, mock_mkdir, mock_open):
         task = {

rpmbuild/tests/test_mock.py

@@ -147,6 +147,9 @@ def test_mock_config(self, call, f_mock_calls):
 config_opts['plugin_conf'].setdefault('tmpfs_opts', {})
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True
 
+# Custom mock snippets configured in copr-crpmbuild config file - can be empty
+
+
 
 config_opts['chroot_additional_packages'] = 'pkg1 pkg2 pkg3'
 
@@ -164,6 +167,30 @@ def test_mock_config(self, call, f_mock_calls):
 
 """  # TODO: make the output nicer
 
+    @mock.patch("copr_rpmbuild.builders.mock.subprocess.call")
+    # pylint: disable-next=unused-argument, redefined-outer-name
+    def test_mock_config_hp_fs_size(self, call, f_mock_calls):
+        """ test that fs_size for performance builders is correctly set """
+        self.task["tags"] = ["blabla_tag", "on_demand_powerful", "aarch64"]
+        builder = MockBuilder(
+            self.task, self.sourcedir, self.resultdir, self.config
+        )
+        mock_snippet = "config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '280g'"
+        builder.copr_rpmbuild_config.tags_to_mock_snippet = [
+            {
+                'tagset': ['on_demand_powerful', "aarch64"],
+                'snippet': mock_snippet,
+            },
+        ]
+        builder.run()
+
+        with open(self.child_config, "r", encoding="utf-8") as f:
+            config = f.readlines()
+
+        config = ''.join(config)
+        assert mock_snippet in config
+
+
     @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.prepare_configs")
     @mock.patch("copr_rpmbuild.builders.mock.MockBuilder.archive_configs")
     def test_mock_options(self, archive_configs, prep_configs, f_mock_calls):
@@ -200,7 +227,7 @@ def test_produce_rpm(self, popen_mock, get_mock_uniqueext_mock,
              '--scrub', 'root-cache', '--quiet'])
         assert get_mock_uniqueext_mock.call_count == 1
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     def test_touch_success_file(self, mock_open):
         builder = MockBuilder(self.task, self.sourcedir, self.resultdir, self.config)
         builder.touch_success_file()

rpmbuild/tests/test_pypi.py

@@ -19,7 +19,7 @@ def setUp(self):
                             "python_versions": [2, 3]}
         self.resultdir = "/path/to/resultdir"
 
-    @mock.patch("{0}.open".format(builtins))
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_init(self, mock_mkdir, mock_open):
        provider = PyPIProvider(self.source_json, self.config)
@@ -29,7 +29,7 @@ def test_init(self, mock_mkdir, mock_open):
        self.assertEqual(provider.python_versions, [2, 3])
 
     @mock.patch("copr_rpmbuild.providers.pypi.run_cmd")
-    @mock.patch("{0}.open".format(builtins))
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_produce_srpm(self, mock_mkdir, mock_open, run_cmd):
         provider = PyPIProvider(self.source_json, self.config)

rpmbuild/tests/test_rubygems.py

@@ -18,14 +18,14 @@ def setUp(self):
         super(TestRubyGemsProvider, self).setUp()
         self.source_json = {"gem_name": "A_123"}
 
-    @mock.patch("{0}.open".format(builtins))
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_init(self, mock_mkdir, mock_open):
         provider = RubyGemsProvider(self.source_json, self.config)
         self.assertEqual(provider.gem_name, "A_123")
 
     @mock.patch("copr_rpmbuild.providers.rubygems.run_cmd")
-    @mock.patch("{0}.open".format(builtins))
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_produce_srpm(self, mock_mkdir, mock_open, run_cmd):
         provider = RubyGemsProvider(self.source_json, self.config)
@@ -35,7 +35,7 @@ def test_produce_srpm(self, mock_mkdir, mock_open, run_cmd):
         run_cmd.assert_called_with(assert_cmd)
 
     @mock.patch("copr_rpmbuild.providers.rubygems.run_cmd")
-    @mock.patch("{0}.open".format(builtins))
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_empty_license(self, mock_mkdir, mock_open, run_cmd):
         stderr = ("error: line 8: Empty tag: License:"

rpmbuild/tests/test_scm.py

@@ -34,7 +34,7 @@ def setUp(self):
             "srpm_build_method": "rpkg",
         }
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_init(self, mock_mkdir, mock_open):
         source_json = self.source_json.copy()
@@ -90,7 +90,7 @@ def test_generate_rpkg_config(self):
 
         shutil.rmtree(tmpdir)
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_get_rpkg_command(self, mock_mkdir, mock_open):
         provider = ScmProvider(self.source_json, self.config)
@@ -100,7 +100,7 @@ def test_get_rpkg_command(self, mock_mkdir, mock_open):
                       "--spec", provider.spec_path]
         self.assertEqual(provider.get_rpkg_command(), assert_cmd)
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_get_tito_command(self, mock_mkdir, mock_open):
         provider = ScmProvider(self.source_json, self.config)
@@ -110,7 +110,7 @@ def test_get_tito_command(self, mock_mkdir, mock_open):
 
 
     @mock.patch("copr_rpmbuild.helpers.run_cmd")
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_get_tito_test_command(self, mock_mkdir, mock_open, run_cmd_mock):
         provider = ScmProvider(self.source_json, self.config)

rpmbuild/tests/test_spec.py

@@ -24,15 +24,15 @@ class TestUrlProvider(TestCase):
     def auto_test_setup(self):
         self.source_json = {"url": u"http://foo.ex/somepackage.spec"}
 
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_init(self, mock_mkdir, mock_open):
         provider = UrlProvider(self.source_json, self.config)
         self.assertEqual(provider.url, "http://foo.ex/somepackage.spec")
 
     @mock.patch('copr_common.request.SafeRequest.get')
     @mock.patch("copr_rpmbuild.providers.base.run_cmd")
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.create_rpmmacros")
     @mock.patch("copr_rpmbuild.providers.spec.UrlProvider.generate_mock_config")
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
@@ -52,7 +52,7 @@ def test_produce_srpm(self, mock_mkdir, mock_generate_mock_config,
         run_cmd.assert_called_with(args, cwd=provider.workdir)
 
     @mock.patch('copr_common.request.SafeRequest.get')
-    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open())
+    @mock.patch('{0}.open'.format(builtins), new_callable=mock.mock_open)
     @mock.patch('copr_rpmbuild.providers.base.os.mkdir')
     def test_save_spec(self, mock_mkdir, mock_open, mock_get):
         provider = UrlProvider(self.source_json, self.config)