From cbb4e0e0861269b4788f710b958fcb3502f50467 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sat, 5 Aug 2023 07:14:27 +0200 Subject: [PATCH] frontend: allow config-based build-chroot tags This feature is to get "high performance" builders into Copr: https://github.com/fedora-copr/debate/blob/main/2023-07-28-high-performance-builders.md https://github.com/praiskup/resalloc/pull/118 --- .../versions/c6dd61c09256_buildchroot_tags.py | 25 ++++++ frontend/coprs_frontend/coprs/config.py | 12 +++ .../coprs/logic/builds_logic.py | 15 +++- frontend/coprs_frontend/coprs/models.py | 83 ++++++++++++++++--- .../coprs/views/backend_ns/backend_general.py | 5 +- 5 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py diff --git a/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py b/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py new file mode 100644 index 000000000..9ad73a658 --- /dev/null +++ b/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py @@ -0,0 +1,25 @@ +""" +BuildChroot Tags + +Revision ID: c6dd61c09256 +Create Date: 2023-08-04 12:37:23.509594 +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c6dd61c09256' +down_revision = 'daa62cd0743d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('build_chroot', sa.Column('tags_raw', sa.String(length=50), + nullable=True)) + + +def downgrade(): + op.drop_column('build_chroot', 'tags_raw') diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py index c135be6b2..bc89dc34a 100644 --- a/frontend/coprs_frontend/coprs/config.py +++ b/frontend/coprs_frontend/coprs/config.py @@ -177,6 +177,18 @@ class Config(object): PACKAGES_COUNT = False + EXTRA_BUILDCHROOT_TAGS = [{ + "pattern": ".*", + "tags": ["matched"], + }, { + "pattern": "*", + "tags": ["not-matched"], + }, { + "pattern": ".*", + "tags": ["matched2"], + }] + + class ProductionConfig(Config): DEBUG = False # SECRET_KEY = "put_some_secret_here" diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 030f3f17d..03031947c 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -359,7 +359,7 @@ def get_pending_build_tasks(cls, background=None, data_type=None): if data_type in ["for_backend", "overview"]: query = query.options( - load_only("build_id"), + load_only("build_id", "tags_raw"), joinedload('build').load_only("id", "is_background", "submitted_by", "batch_id") .options( # from copr project info we only need the project name @@ -452,9 +452,9 @@ def create_new_from_other_build(cls, user, copr, source_build, raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, + package=source_build.package, pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) - build.package_id = source_build.package_id build.pkg_version = source_build.pkg_version build.resubmitted_from_id = source_build.id @@ -862,6 +862,8 @@ def add(cls, user, pkgs, copr, source_type=None, source_json=None, git_hashes=git_hashes, status=chroot_status, ) + if skip_import and srpm_url: + build.backend_enqueue_buildchroots() return build @classmethod @@ -1383,12 +1385,19 @@ def new(cls, build, mock_chroot, **kwargs): copr_chroot = coprs_logic.CoprChrootsLogic.get_by_mock_chroot_id( build.copr, mock_chroot.id ).one() - return models.BuildChroot( + build_chroot = models.BuildChroot( mock_chroot=mock_chroot, copr_chroot=copr_chroot, build=build, **kwargs, ) + cls.set_tags(build_chroot) + return build_chroot + + @classmethod + def set_tags(cls, build_chroot): + # pattern = f"{build.copr.full_name}/{mock_chroot.name}/{" + pass @classmethod def get_by_build_id_and_name(cls, build_id, name): diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 34a40a875..0b91d049e 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -11,6 +11,7 @@ import base64 import operator import os +import re from urllib.parse import urljoin import uuid import time @@ -1530,6 +1531,15 @@ def appstream(self): """Whether appstream metadata should be generated for a build.""" return self.copr.appstream + def backend_enqueue_buildchroots(self): + """ + When the sources are successfully imported into dist-git, and the set of + buildchroots is generated, do some last-minute BuildChroot preprations + before hand-it over to Backend (into pending-jobs queue). + """ + for bch in self.build_chroots: + bch.backend_enqueue() + class DistGitBranch(db.Model, helpers.Serializer): """ @@ -1540,7 +1550,26 @@ class DistGitBranch(db.Model, helpers.Serializer): name = db.Column(db.String(50), primary_key=True) -class MockChroot(db.Model, helpers.Serializer): +class TagMixin: + # A space separated list of tags. Typically used as additional tags for the + # Resalloc system to match appropriate builder. + tags_raw = db.Column(db.String(50), nullable=True) + + @property + def tags(self): + """ + Return the list (of strings) of MockChroot tags. + """ + return self.tags_raw.split() if self.tags_raw else [] + + def set_tags(self, tags): + if not tags: + self.tags_raw = None + else: + self.tags_raw = " ".join(tags) + + +class MockChroot(db.Model, TagMixin, helpers.Serializer): """ Representation of mock chroot """ @@ -1578,10 +1607,6 @@ class MockChroot(db.Model, helpers.Serializer): 'x86_64': 'i386', } - # A space separated list of tags. Typically used as additional tags for the - # Resalloc system to match appropriate builder. - tags_raw = db.Column(db.String(50), nullable=True) - @classmethod def latest_fedora_branched_chroot(cls, arch='x86_64'): return (cls.query @@ -1620,13 +1645,6 @@ def serializable_attributes(self): attr_list.extend(["name", "os"]) return attr_list - @property - def tags(self): - """ - Return the list (of strings) of MockChroot tags. - """ - return self.tags_raw.split() if self.tags_raw else [] - @property def os_family(self): """ @@ -1878,7 +1896,7 @@ def isolation_setup(self): return settings -class BuildChroot(db.Model, helpers.Serializer): +class BuildChroot(db.Model, TagMixin, helpers.Serializer): """ Representation of Build<->MockChroot relation """ @@ -2060,6 +2078,45 @@ def distgit_clone_url(self): package = self.build.package.name return "{}/{}/{}".format(app.config["DIST_GIT_CLONE_URL"], dirname, package) + def _compile_extra_buildchroot_tags(self): + """ + Convert the EXTRA_BUILDCHROOT_TAGS "pattern" field to "compiled" field + (if possible, error out otherwise). Just optimization to not re-compile + over again within the same process. + """ + new_array = [] + for rule in app.config["EXTRA_BUILDCHROOT_TAGS"]: + if "compiled" in rule: + return # all the rules are already compiled + try: + rule["compiled"] = re.compile(rule["pattern"]) + except re.error: + # Don't stop the server if user makes a regexp error. + app.logger.exception("Invalid regexp: %s", rule["pattern"]) + continue + new_array.append(rule) + app.config["EXTRA_BUILDCHROOT_TAGS"] = new_array + + def backend_enqueue(self): + """ + The sources are successfully prepared in copr-dist-git, it's time to + place this buildchroot task into the "pending-jobs" queue. + """ + # now is the time to add tags...? + # now is the time to skip exclude arch...? + pkg_path = ( + f"{self.build.copr_dir.full_name}/" + f"{self.mock_chroot.name}/" + f"{self.build.package.name}" + ) + + self._compile_extra_buildchroot_tags() + tags = [] + for rule in app.config["EXTRA_BUILDCHROOT_TAGS"]: + if rule["compiled"].match(pkg_path): + tags.extend(rule["tags"]) + self.set_tags(tags) + class BuildChrootResult(db.Model, helpers.Serializer): """ diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index 59a11c877..14196b195 100755 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -104,6 +104,9 @@ def dist_git_upload_completed(): ch.status = StatusEnum("failed") db.session.add(ch) + if final_source_status == StatusEnum("succeeded"): + build.backend_enqueue_buildchroots() + build.source_status = final_source_status db.session.add(build) db.session.commit() @@ -143,7 +146,7 @@ def get_build_record(task, for_backend=False): "sandbox": task.build.sandbox, "background": bool(task.build.is_background), "chroot": task.mock_chroot.name, - "tags": task.mock_chroot.tags, + "tags": task.mock_chroot.tags + task.tags, } if for_backend: