frontend: allow config-based build-chroot tags

This feature is to get "high performance" builders into Copr:
https://github.com/fedora-copr/debate/blob/main/2023-07-28-high-performance-builders.md
praiskup/resalloc#118

frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py

@@ -0,0 +1,25 @@
+"""
+BuildChroot Tags
+
+Revision ID: c6dd61c09256
+Create Date: 2023-08-04 12:37:23.509594
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c6dd61c09256'
+down_revision = '08dd42f4c304'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('build_chroot', sa.Column('tags_raw', sa.String(length=50),
+                                            nullable=True))
+
+
+def downgrade():
+    op.drop_column('build_chroot', 'tags_raw')

frontend/coprs_frontend/config/copr.conf

@@ -174,6 +174,16 @@ HIDE_IMPORT_LOG_AFTER_DAYS = 14
 # Whether to show a total packages count at homepage
 # PACKAGES_COUNT = False
 
+# Extend the set of "builder tags" (see `man 1 resalloc`).  This needs to be a
+# list like [{"pattern": r"regexp", "tags": ["tag1", "tag2", ...]}, ...].  The
+# regexp pattern is matched against BuildChroot fully qualified names in the
+# format "user/project:dirname/chroot_name/pkgname", so example rule might
+# look like
+#    "pattern": r"john/projectX.*/fedora-.*-x86_64/chromium",
+#    "tags": ["extra_powerful"],
+#EXTRA_BUILDCHROOT_TAGS = []
+
+
 #############################
 ##### DEBUGGING Section #####
 

frontend/coprs_frontend/coprs/config.py

@@ -177,6 +177,9 @@ class Config(object):
 
     PACKAGES_COUNT = False
 
+    EXTRA_BUILDCHROOT_TAGS = []
+
+
 class ProductionConfig(Config):
     DEBUG = False
     # SECRET_KEY = "put_some_secret_here"

frontend/coprs_frontend/coprs/logic/builds_logic.py

@@ -359,7 +359,7 @@ def get_pending_build_tasks(cls, background=None, data_type=None):
 
         if data_type in ["for_backend", "overview"]:
             query = query.options(
-                load_only("build_id"),
+                load_only("build_id", "tags_raw"),
                 joinedload('build').load_only("id", "is_background", "submitted_by", "batch_id")
                 .options(
                     # from copr project info we only need the project name
@@ -452,9 +452,9 @@ def create_new_from_other_build(cls, user, copr, source_build,
                 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.")
 
         build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names,
+                               package=source_build.package,
                                pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import,
                                srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options)
-        build.package_id = source_build.package_id
         build.pkg_version = source_build.pkg_version
         build.resubmitted_from_id = source_build.id
 
@@ -862,6 +862,8 @@ def add(cls, user, pkgs, copr, source_type=None, source_json=None,
             git_hashes=git_hashes,
             status=chroot_status,
         )
+        if skip_import and srpm_url:
+            build.backend_enqueue_buildchroots()
         return build
 
     @classmethod

frontend/coprs_frontend/coprs/models.py

@@ -11,6 +11,7 @@
 import base64
 import operator
 import os
+import re
 from urllib.parse import urljoin
 import uuid
 import time
@@ -1530,6 +1531,15 @@ def appstream(self):
         """Whether appstream metadata should be generated for a build."""
         return self.copr.appstream
 
+    def backend_enqueue_buildchroots(self):
+        """
+        When the sources are successfully imported into dist-git, and the set of
+        buildchroots is generated, do some last-minute BuildChroot preprations
+        before hand-it over to Backend (into pending-jobs queue).
+        """
+        for bch in self.build_chroots:
+            bch.backend_enqueue()
+
 
 class DistGitBranch(db.Model, helpers.Serializer):
     """
@@ -1540,7 +1550,33 @@ class DistGitBranch(db.Model, helpers.Serializer):
     name = db.Column(db.String(50), primary_key=True)
 
 
-class MockChroot(db.Model, helpers.Serializer):
+class TagMixin:
+    """
+    Work with tags assigned to BuildChroots or MockChroots uniformly.  Typically
+    used as additional tags for the Resalloc system to match appropriate
+    builder.
+    """
+    tags_raw = db.Column(db.String(50), nullable=True)
+
+    @property
+    def tags(self):
+        """
+        Return the list (of strings) of MockChroot tags.
+        """
+        return self.tags_raw.split() if self.tags_raw else []
+
+    def set_tags(self, tags):
+        """
+        Convert the given list of tags into a space separated string (the
+        internal "tags_raw" format).  Empty list NULLs the field.
+        """
+        if not tags:
+            self.tags_raw = None
+        else:
+            self.tags_raw = " ".join(tags)
+
+
+class MockChroot(db.Model, TagMixin, helpers.Serializer):
     """
     Representation of mock chroot
     """
@@ -1578,10 +1614,6 @@ class MockChroot(db.Model, helpers.Serializer):
         'x86_64': 'i386',
     }
 
-    # A space separated list of tags.  Typically used as additional tags for the
-    # Resalloc system to match appropriate builder.
-    tags_raw = db.Column(db.String(50), nullable=True)
-
     @classmethod
     def latest_fedora_branched_chroot(cls, arch='x86_64'):
         return (cls.query
@@ -1620,13 +1652,6 @@ def serializable_attributes(self):
         attr_list.extend(["name", "os"])
         return attr_list
 
-    @property
-    def tags(self):
-        """
-        Return the list (of strings) of MockChroot tags.
-        """
-        return self.tags_raw.split() if self.tags_raw else []
-
     @property
     def os_family(self):
         """
@@ -1878,7 +1903,7 @@ def isolation_setup(self):
         return settings
 
 
-class BuildChroot(db.Model, helpers.Serializer):
+class BuildChroot(db.Model, TagMixin, helpers.Serializer):
     """
     Representation of Build<->MockChroot relation
     """
@@ -2060,6 +2085,43 @@ def distgit_clone_url(self):
         package = self.build.package.name
         return "{}/{}/{}".format(app.config["DIST_GIT_CLONE_URL"], dirname, package)
 
+    def _compile_extra_buildchroot_tags(self):
+        """
+        Convert the EXTRA_BUILDCHROOT_TAGS to EXTRA_BUILDCHROOT_TAGS_COMPILED
+        array with items having the "compiled" field.  Just optimization to not
+        re-compile over again within the same process.
+        """
+        if "EXTRA_BUILDCHROOT_TAGS_COMPILED" in app.config:
+            return
+        new_array = app.config["_EXTRA_BUILDCHROOT_TAGS_COMPILED"] = []
+        for rule in app.config["EXTRA_BUILDCHROOT_TAGS"]:
+            try:
+                rule["compiled"] = re.compile(rule["pattern"])
+                new_array.append(rule)
+            except re.error:
+                # Don't stop the server if user makes a regexp error.
+                app.logger.exception("Invalid regexp: %s", rule["pattern"])
+
+    def backend_enqueue(self):
+        """
+        The sources are successfully prepared in copr-dist-git, it's time to
+        place this buildchroot task into the "pending-jobs" queue.
+        """
+        # now is the time to add tags...?
+        # now is the time to skip exclude arch...?
+        pkg_path = (
+            f"{self.build.copr_dir.full_name}/"
+            f"{self.mock_chroot.name}/"
+            f"{self.build.package.name}"
+        )
+
+        self._compile_extra_buildchroot_tags()
+        tags = []
+        for rule in app.config["_EXTRA_BUILDCHROOT_TAGS_COMPILED"]:
+            if rule["compiled"].match(pkg_path):
+                tags.extend(rule["tags"])
+        self.set_tags(tags)
+
 
 class BuildChrootResult(db.Model, helpers.Serializer):
     """

frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py

@@ -104,6 +104,9 @@ def dist_git_upload_completed():
             ch.status = StatusEnum("failed")
             db.session.add(ch)
 
+    if final_source_status == StatusEnum("succeeded"):
+        build.backend_enqueue_buildchroots()
+
     build.source_status = final_source_status
     db.session.add(build)
     db.session.commit()
@@ -143,7 +146,7 @@ def get_build_record(task, for_backend=False):
             "sandbox": task.build.sandbox,
             "background": bool(task.build.is_background),
             "chroot": task.mock_chroot.name,
-            "tags": task.mock_chroot.tags,
+            "tags": task.mock_chroot.tags + task.tags,
         }
 
         if for_backend:

frontend/coprs_frontend/tests/coprs_test_case.py

@@ -9,6 +9,7 @@
 import uuid
 
 import pytest
+from sqlalchemy import desc
 import decorator
 
 import coprs
@@ -882,6 +883,10 @@ def rebuild_package_and_finish(self, project_dirname, pkgname,
         build_id = out.json["id"]
         self.backend.finish_build(build_id, pkg_version=pkg_version)
 
+    @property
+    def last_build(self):
+        return models.Build.query.order_by(desc(models.Build.id)).first()
+
 
 class TransactionDecorator(object):
 

frontend/coprs_frontend/tests/request_test_api.py

@@ -206,6 +206,16 @@ def rebuild_all_packages(self, project_id, package_names=None):
         resp = self.client.post(route, data=form_data)
         return resp
 
+    def resubmit_build_id(self, build_id):
+        build = models.Build.query.get(build_id)
+        path = f"/coprs/{build.copr.full_name}/new_build_rebuild/{build_id}"
+        response = self.client.post(
+            path,
+            data={},
+        )
+        assert response.status_code == 302
+        return response
+
 
 class API3Requests(_RequestsInterface):
     """
@@ -424,9 +434,10 @@ def fail_source_build(self, build_id):
         assert self.update(form_data).status_code == 200
 
 
-    def finish_build(self, build_id, package_name=None, pkg_version="1"):
+    def finish_srpm_and_import(self, build_id, package_name=None, pkg_version="1"):
         """
-        Given the build_id, finish the build with succeeded state
+        Given the build_id, finish the source build, import it, and move the
+        corresponding BuildChroot instances to /pending-jobs/.
         """
         build = models.Build.query.get(build_id)
         if not package_name:
@@ -459,6 +470,12 @@ def finish_build(self, build_id, package_name=None, pkg_version="1"):
                "reponame": "some/repo"
         }))
 
+
+    def finish_build(self, build_id, package_name=None, pkg_version="1"):
+        """
+        Given the build_id, finish the build with succeeded state
+        """
+        self.finish_srpm_and_import(build_id, package_name, pkg_version)
         # finish rpms
         update_requests = []
         for build_chroot in models.BuildChroot.query.filter_by(build_id=build_id).all():