From 9e71174fca5bd18feee1ebc3959cf6f36c4b0c28 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sat, 5 Aug 2023 07:14:27 +0200 Subject: [PATCH] frontend: allow config-based build-chroot tags This feature is to get "high performance" builders into Copr: https://github.com/fedora-copr/debate/blob/main/2023-07-28-high-performance-builders.md https://github.com/praiskup/resalloc/pull/118 --- .../versions/c6dd61c09256_buildchroot_tags.py | 25 ++++ frontend/coprs_frontend/config/copr.conf | 10 ++ frontend/coprs_frontend/coprs/config.py | 3 + .../coprs/logic/builds_logic.py | 6 +- frontend/coprs_frontend/coprs/models.py | 88 +++++++++++-- .../coprs/views/backend_ns/backend_general.py | 5 +- .../coprs_frontend/tests/coprs_test_case.py | 5 + .../coprs_frontend/tests/request_test_api.py | 21 ++- .../coprs_frontend/tests/test_resubmit.py | 124 ++++++++++++++++++ 9 files changed, 269 insertions(+), 18 deletions(-) create mode 100644 frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py create mode 100644 frontend/coprs_frontend/tests/test_resubmit.py diff --git a/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py b/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py new file mode 100644 index 000000000..ff2266c21 --- /dev/null +++ b/frontend/coprs_frontend/alembic/versions/c6dd61c09256_buildchroot_tags.py @@ -0,0 +1,25 @@ +""" +BuildChroot Tags + +Revision ID: c6dd61c09256 +Create Date: 2023-08-04 12:37:23.509594 +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c6dd61c09256' +down_revision = '08dd42f4c304' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('build_chroot', sa.Column('tags_raw', sa.String(length=50), + nullable=True)) + + +def downgrade(): + op.drop_column('build_chroot', 'tags_raw') diff --git a/frontend/coprs_frontend/config/copr.conf b/frontend/coprs_frontend/config/copr.conf index a1b417d35..0ddf6066b 100644 --- a/frontend/coprs_frontend/config/copr.conf +++ b/frontend/coprs_frontend/config/copr.conf @@ -174,6 +174,16 @@ HIDE_IMPORT_LOG_AFTER_DAYS = 14 # Whether to show a total packages count at homepage # PACKAGES_COUNT = False +# Extend the set of "builder tags" (see `man 1 resalloc`). This needs to be a +# list like [{"pattern": r"regexp", "tags": ["tag1", "tag2", ...]}, ...]. The +# regexp pattern is matched against BuildChroot fully qualified names in the +# format "user/project:dirname/chroot_name/pkgname", so example rule might +# look like +# "pattern": r"john/projectX.*/fedora-.*-x86_64/chromium", +# "tags": ["extra_powerful"], +#EXTRA_BUILDCHROOT_TAGS = [] + + ############################# ##### DEBUGGING Section ##### diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py index c135be6b2..71dbe40fb 100644 --- a/frontend/coprs_frontend/coprs/config.py +++ b/frontend/coprs_frontend/coprs/config.py @@ -177,6 +177,9 @@ class Config(object): PACKAGES_COUNT = False + EXTRA_BUILDCHROOT_TAGS = [] + + class ProductionConfig(Config): DEBUG = False # SECRET_KEY = "put_some_secret_here" diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 030f3f17d..99f6a6eea 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -359,7 +359,7 @@ def get_pending_build_tasks(cls, background=None, data_type=None): if data_type in ["for_backend", "overview"]: query = query.options( - load_only("build_id"), + load_only("build_id", "tags_raw"), joinedload('build').load_only("id", "is_background", "submitted_by", "batch_id") .options( # from copr project info we only need the project name @@ -452,9 +452,9 @@ def create_new_from_other_build(cls, user, copr, source_build, raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, + package=source_build.package, pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) - build.package_id = source_build.package_id build.pkg_version = source_build.pkg_version build.resubmitted_from_id = source_build.id @@ -862,6 +862,8 @@ def add(cls, user, pkgs, copr, source_type=None, source_json=None, git_hashes=git_hashes, status=chroot_status, ) + if skip_import and srpm_url: + build.backend_enqueue_buildchroots() return build @classmethod diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index 73e0781ae..7c85e5444 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -11,6 +11,7 @@ import base64 import operator import os +import re from urllib.parse import urljoin import uuid import time @@ -1530,6 +1531,15 @@ def appstream(self): """Whether appstream metadata should be generated for a build.""" return self.copr.appstream + def backend_enqueue_buildchroots(self): + """ + When the sources are successfully imported into dist-git, and the set of + buildchroots is generated, do some last-minute BuildChroot preprations + before hand-it over to Backend (into pending-jobs queue). + """ + for bch in self.build_chroots: + bch.backend_enqueue() + class DistGitBranch(db.Model, helpers.Serializer): """ @@ -1540,7 +1550,33 @@ class DistGitBranch(db.Model, helpers.Serializer): name = db.Column(db.String(50), primary_key=True) -class MockChroot(db.Model, helpers.Serializer): +class TagMixin: + """ + Work with tags assigned to BuildChroots or MockChroots uniformly. Typically + used as additional tags for the Resalloc system to match appropriate + builder. + """ + tags_raw = db.Column(db.String(50), nullable=True) + + @property + def tags(self): + """ + Return the list (of strings) of MockChroot tags. + """ + return self.tags_raw.split() if self.tags_raw else [] + + def set_tags(self, tags): + """ + Convert the given list of tags into a space separated string (the + internal "tags_raw" format). Empty list NULLs the field. + """ + if not tags: + self.tags_raw = None + else: + self.tags_raw = " ".join(tags) + + +class MockChroot(db.Model, TagMixin, helpers.Serializer): """ Representation of mock chroot """ @@ -1578,10 +1614,6 @@ class MockChroot(db.Model, helpers.Serializer): 'x86_64': 'i386', } - # A space separated list of tags. Typically used as additional tags for the - # Resalloc system to match appropriate builder. - tags_raw = db.Column(db.String(50), nullable=True) - @classmethod def latest_fedora_branched_chroot(cls, arch='x86_64'): return (cls.query @@ -1620,13 +1652,6 @@ def serializable_attributes(self): attr_list.extend(["name", "os"]) return attr_list - @property - def tags(self): - """ - Return the list (of strings) of MockChroot tags. - """ - return self.tags_raw.split() if self.tags_raw else [] - @property def os_family(self): """ @@ -1878,7 +1903,7 @@ def isolation_setup(self): return settings -class BuildChroot(db.Model, helpers.Serializer): +class BuildChroot(db.Model, TagMixin, helpers.Serializer): """ Representation of Build<->MockChroot relation """ @@ -2060,6 +2085,43 @@ def distgit_clone_url(self): package = self.build.package.name return "{}/{}/{}".format(app.config["DIST_GIT_CLONE_URL"], dirname, package) + def _compile_extra_buildchroot_tags(self): + """ + Convert the EXTRA_BUILDCHROOT_TAGS to EXTRA_BUILDCHROOT_TAGS_COMPILED + array with items having the "compiled" field. Just optimization to not + re-compile over again within the same process. + """ + if "EXTRA_BUILDCHROOT_TAGS_COMPILED" in app.config: + return + new_array = app.config["_EXTRA_BUILDCHROOT_TAGS_COMPILED"] = [] + for rule in app.config["EXTRA_BUILDCHROOT_TAGS"]: + try: + rule["compiled"] = re.compile(rule["pattern"]) + new_array.append(rule) + except re.error: + # Don't stop the server if user makes a regexp error. + app.logger.exception("Invalid regexp: %s", rule["pattern"]) + + def backend_enqueue(self): + """ + The sources are successfully prepared in copr-dist-git, it's time to + place this buildchroot task into the "pending-jobs" queue. + """ + # now is the time to add tags...? + # now is the time to skip exclude arch...? + pkg_path = ( + f"{self.build.copr_dir.full_name}/" + f"{self.mock_chroot.name}/" + f"{self.build.package.name}" + ) + + self._compile_extra_buildchroot_tags() + tags = [] + for rule in app.config["_EXTRA_BUILDCHROOT_TAGS_COMPILED"]: + if rule["compiled"].match(pkg_path): + tags.extend(rule["tags"]) + self.set_tags(tags) + class BuildChrootResult(db.Model, helpers.Serializer): """ diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index 8d180459b..8a7b4c34a 100755 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -104,6 +104,9 @@ def dist_git_upload_completed(): ch.status = StatusEnum("failed") db.session.add(ch) + if final_source_status == StatusEnum("succeeded"): + build.backend_enqueue_buildchroots() + build.source_status = final_source_status db.session.add(build) db.session.commit() @@ -143,7 +146,7 @@ def get_build_record(task, for_backend=False): "sandbox": task.build.sandbox, "background": bool(task.build.is_background), "chroot": task.mock_chroot.name, - "tags": task.mock_chroot.tags, + "tags": task.mock_chroot.tags + task.tags, } if for_backend: diff --git a/frontend/coprs_frontend/tests/coprs_test_case.py b/frontend/coprs_frontend/tests/coprs_test_case.py index 7eb93d345..41abcf98b 100644 --- a/frontend/coprs_frontend/tests/coprs_test_case.py +++ b/frontend/coprs_frontend/tests/coprs_test_case.py @@ -9,6 +9,7 @@ import uuid import pytest +from sqlalchemy import desc import decorator import coprs @@ -904,6 +905,10 @@ def rebuild_package_and_finish(self, project_dirname, pkgname, build_id = out.json["id"] self.backend.finish_build(build_id, pkg_version=pkg_version) + @property + def last_build(self): + return models.Build.query.order_by(desc(models.Build.id)).first() + class TransactionDecorator(object): diff --git a/frontend/coprs_frontend/tests/request_test_api.py b/frontend/coprs_frontend/tests/request_test_api.py index c0e8f1130..76acb343c 100644 --- a/frontend/coprs_frontend/tests/request_test_api.py +++ b/frontend/coprs_frontend/tests/request_test_api.py @@ -206,6 +206,16 @@ def rebuild_all_packages(self, project_id, package_names=None): resp = self.client.post(route, data=form_data) return resp + def resubmit_build_id(self, build_id): + build = models.Build.query.get(build_id) + path = f"/coprs/{build.copr.full_name}/new_build_rebuild/{build_id}" + response = self.client.post( + path, + data={}, + ) + assert response.status_code == 302 + return response + class API3Requests(_RequestsInterface): """ @@ -424,9 +434,10 @@ def fail_source_build(self, build_id): assert self.update(form_data).status_code == 200 - def finish_build(self, build_id, package_name=None, pkg_version="1"): + def finish_srpm_and_import(self, build_id, package_name=None, pkg_version="1"): """ - Given the build_id, finish the build with succeeded state + Given the build_id, finish the source build, import it, and move the + corresponding BuildChroot instances to /pending-jobs/. """ build = models.Build.query.get(build_id) if not package_name: @@ -459,6 +470,12 @@ def finish_build(self, build_id, package_name=None, pkg_version="1"): "reponame": "some/repo" })) + + def finish_build(self, build_id, package_name=None, pkg_version="1"): + """ + Given the build_id, finish the build with succeeded state + """ + self.finish_srpm_and_import(build_id, package_name, pkg_version) # finish rpms update_requests = [] for build_chroot in models.BuildChroot.query.filter_by(build_id=build_id).all(): diff --git a/frontend/coprs_frontend/tests/test_resubmit.py b/frontend/coprs_frontend/tests/test_resubmit.py new file mode 100644 index 000000000..9dad44dda --- /dev/null +++ b/frontend/coprs_frontend/tests/test_resubmit.py @@ -0,0 +1,124 @@ +import json + +import pytest + +from copr_common.enums import StatusEnum + +from tests.coprs_test_case import CoprsTestCase, TransactionDecorator + +class TestCoprResubmitBuild(CoprsTestCase): + # pylint: disable=attribute-defined-outside-init + + def _distgit_chromium_built(self): + self.api3.create_distgit_package("foocopr", "chromium") + self.api3.rebuild_package("foocopr", "chromium") + self.bdistgit = self.last_build + self.backend.finish_build(self.bdistgit.id) + + @pytest.fixture + def f_upload_processed(self, f_users, f_coprs, f_mock_chroots, f_builds): + """ + Uploaded build with one succeeded, one failed and one skipped + buildchroot. + """ + _fixtures = f_users, f_coprs, f_mock_chroots, f_builds + + version = "1.0-2" + data = { + "url": f"https://copr.example.com/tmp/tmpu6sl_mi2/hello-world-{version}.el9.src.rpm", + "pkg": f"hello-world-{version}.el9.src.rpm", + "tmp": "tmpu6sl_mi2" + } + + cc2 = self.models.CoprChroot() + cc2.mock_chroot = self.mc2 + cc3 = self.models.CoprChroot() + cc3.mock_chroot = self.mc3 + self.c1.copr_chroots.append(cc2) + self.c1.copr_chroots.append(cc3) + self.db.session.add_all([cc2, cc3]) + + self.bupload = self.models.Build( + copr=self.c1, + pkgs=data["url"], + pkg_version=version, + source_json=json.dumps(data), + source_type=2, + copr_dir=self.c1_dir, + user=self.u1, + submitted_on=9, + result_dir="06246100", + source_status=1, + srpm_url=data["url"], + package=self.p1, + ) + self.db.session.add_all([self.bupload]) + + for (mc, cc, status) in [ + (self.mc1, self.c1.copr_chroots[0], "failed"), + (self.mc2, self.c1.copr_chroots[1], "skipped"), + (self.mc3, self.c1.copr_chroots[2], "succeeded"), + ]: + bch = self.models.BuildChroot( + build=self.bupload, + mock_chroot=mc, + status=StatusEnum(status), + git_hash="2a7eeee353531828f306167730cca1cc300e35ae", + started_on=1674058555, + ended_on=1674058639, + result_dir='bar', + copr_chroot=cc, + ) + self.db.session.add(bch) + + @TransactionDecorator("u1") + @pytest.mark.usefixtures("f_users", "f_upload_processed", "f_db") + def test_copr_repeat_build_attributes_upload(self): + self.app.config["EXTRA_BUILDCHROOT_TAGS"] = [{ + "pattern": ".*", + "tags": ["every_build"], + }, { + "pattern": "*invalid_regexp", + "tags": ["invalid_not_attached"], + }, { + "pattern": "user1/foocopr/fedora-18-x86_64/hello-world", + "tags": ["specific"], + }, { + "pattern": ".*/.*/.*/chromium", + "tags": ["beefy"], + }] + + self._distgit_chromium_built() + + # resubmitting uploaded build skips import + self.web_ui.resubmit_build_id(self.bupload.id) + new_build = self.last_build + + old_build = self.bdistgit + for bch in old_build.build_chroots: + assert bch.tags == ["every_build", "beefy"] + + # this is waiting for source processing, and import + self.web_ui.resubmit_build_id(self.bdistgit.id) + new_build2 = self.last_build + + assert len(new_build.build_chroots) == 3 + assert len(new_build2.build_chroots) == 3 + + for bch in new_build.build_chroots: + if bch.name == "fedora-18-x86_64": + assert bch.tags == ["every_build", "specific"] + else: + assert bch.tags == ["every_build"] + + + for bch in new_build2.build_chroots: + # Nothing assigned till the package is imported + assert bch.tags == [] + + # Finish SRPM build by Backend, and import by DistGit. + self.backend.finish_srpm_and_import(new_build2.id) + for bch in new_build2.build_chroots: + # Nothing assigned till the package is imported + assert bch.state == "pending" + assert bch.tags == ["every_build", "beefy"]