backend, frontend: implement project and build deletion in Pulp

Fix fedora-copr#3318
Fix fedora-copr#3319

backend/copr_backend/actions.py

@@ -26,8 +26,7 @@
 from .exceptions import CreateRepoError, CoprSignError, FrontendClientException
 from .helpers import (get_redis_logger, silent_remove, ensure_dir_exists,
                       get_chroot_arch, format_filename,
-                      uses_devel_repo, call_copr_repo, build_chroot_log_name,
-                      copy2_but_hardlink_rpms)
+                      call_copr_repo, copy2_but_hardlink_rpms)
 from .sign import sign_rpms_in_dir, unsign_rpms_in_dir, get_pubkey
 
 
@@ -64,7 +63,6 @@ def get_action_class(cls, action):
             ActionTypeEnum("rawhide_to_release"): RawhideToRelease,
             ActionTypeEnum("fork"): Fork,
             ActionTypeEnum("build_module"): BuildModule,
-            ActionTypeEnum("delete"): Delete,
             ActionTypeEnum("remove_dirs"): RemoveDirs,
         }.get(action_type, None)
 
@@ -221,72 +219,21 @@ def run(self):
         return result
 
 
-class Delete(Action):
-    """
-    Abstract class for all other Delete* classes.
-    """
-    # pylint: disable=abstract-method
-    def _handle_delete_builds(self, ownername, projectname, project_dirname,
-                              chroot_builddirs, build_ids, appstream):
-        """ call /bin/copr-repo --delete """
-        devel = uses_devel_repo(self.front_url, ownername, projectname)
-        result = BackendResultEnum("success")
-        for chroot, subdirs in chroot_builddirs.items():
-            chroot_path = os.path.join(self.destdir, ownername, project_dirname,
-                                       chroot)
-            if not os.path.exists(chroot_path):
-                self.log.error("%s chroot path doesn't exist", chroot_path)
-                result = BackendResultEnum("failure")
-                continue
-
-            self.log.info("Deleting subdirs [%s] in %s",
-                          ", ".join(subdirs), chroot_path)
-
-            # Run createrepo first and then remove the files (to avoid old
-            # repodata temporarily pointing at non-existing files)!
-            if chroot != "srpm-builds":
-                # In srpm-builds we don't create repodata at all
-                if not call_copr_repo(chroot_path, delete=subdirs, devel=devel, appstream=appstream,
-                                      logger=self.log):
-                    result = BackendResultEnum("failure")
-
-            for build_id in build_ids or []:
-                log_paths = [
-                    os.path.join(chroot_path, build_chroot_log_name(build_id)),
-                    # we used to create those before
-                    os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
-                    os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
-                for log_path in log_paths:
-                    try:
-                        os.unlink(log_path)
-                    except OSError:
-                        self.log.debug("can't remove %s", log_path)
-        return result
-
-
-class DeleteProject(Delete):
+class DeleteProject(Action):
     def run(self):
         self.log.debug("Action delete copr")
-        result = BackendResultEnum("success")
-
-        ext_data = json.loads(self.data["data"])
-        ownername = ext_data["ownername"]
-        project_dirnames = ext_data["project_dirnames"]
+        project_dirnames = self.ext_data["project_dirnames"]
 
-        if not ownername:
+        if not self.storage.owner:
             self.log.error("Received empty ownername!")
-            result = BackendResultEnum("failure")
-            return result
+            return BackendResultEnum("failure")
 
         for dirname in project_dirnames:
             if not dirname:
                 self.log.warning("Received empty dirname!")
                 continue
-            path = os.path.join(self.destdir, ownername, dirname)
-            if os.path.exists(path):
-                self.log.info("Removing copr dir %s", path)
-                shutil.rmtree(path)
-        return result
+            self.storage.delete_project(dirname)
+        return BackendResultEnum("success")
 
 
 class CompsUpdate(Action):
@@ -322,7 +269,7 @@ def run(self):
         return result
 
 
-class DeleteMultipleBuilds(Delete):
+class DeleteMultipleBuilds(Action):
     def run(self):
         self.log.debug("Action delete multiple builds.")
 
@@ -334,25 +281,20 @@ def run(self):
         #     srpm-builds: [00849545, 00849546]
         #     fedora-30-x86_64: [00849545-example, 00849545-foo]
         #   [...]
-        ext_data = json.loads(self.data["data"])
 
-        ownername = ext_data["ownername"]
-        projectname = ext_data["projectname"]
-        project_dirnames = ext_data["project_dirnames"]
-        build_ids = ext_data["build_ids"]
-        appstream = ext_data["appstream"]
+        project_dirnames = self.ext_data["project_dirnames"]
+        build_ids = self.ext_data["build_ids"]
 
         result = BackendResultEnum("success")
         for project_dirname, chroot_builddirs in project_dirnames.items():
-            if BackendResultEnum("failure") == \
-               self._handle_delete_builds(ownername, projectname,
-                                          project_dirname, chroot_builddirs,
-                                          build_ids, appstream):
+            success = self.storage.delete_builds(
+                project_dirname, chroot_builddirs, build_ids)
+            if not success:
                 result = BackendResultEnum("failure")
         return result
 
 
-class DeleteBuild(Delete):
+class DeleteBuild(Action):
     def run(self):
         self.log.info("Action delete build.")
 
@@ -363,25 +305,27 @@ def run(self):
         # chroot_builddirs:
         #   srpm-builds: [00849545]
         #   fedora-30-x86_64: [00849545-example]
-        ext_data = json.loads(self.data["data"])
 
-        try:
-            ownername = ext_data["ownername"]
-            build_ids = [self.data['object_id']]
-            projectname = ext_data["projectname"]
-            project_dirname = ext_data["project_dirname"]
-            chroot_builddirs = ext_data["chroot_builddirs"]
-            appstream = ext_data["appstream"]
-        except KeyError:
+        valid = "object_id" in self.data
+        keys = {"ownername", "projectname", "project_dirname",
+                "chroot_builddirs", "appstream"}
+        for key in keys:
+            if key not in self.ext_data:
+                valid = False
+                break
+
+        if not valid:
             self.log.exception("Invalid action data")
             return BackendResultEnum("failure")
 
-        return self._handle_delete_builds(ownername, projectname,
-                                          project_dirname, chroot_builddirs,
-                                          build_ids, appstream)
+        success = self.storage.delete_builds(
+            self.ext_data["project_dirname"],
+            self.ext_data["chroot_builddirs"],
+            [self.data['object_id']])
+        return BackendResultEnum("success" if success else "failure")
 
 
-class DeleteChroot(Delete):
+class DeleteChroot(Action):
     def run(self):
         self.log.info("Action delete project chroot.")
         chroot = self.ext_data["chrootname"]

backend/copr_backend/pulp.py

@@ -164,6 +164,16 @@ def create_content(self, repository, path):
             return requests.post(
                 url, data=data, files=files, **self.request_params)
 
+    def delete_content(self, repository, artifacts):
+        """
+        Delete a list of artifacts from a repository
+        https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm/operation/repositories_rpm_rpm_modify
+        """
+        path = os.path.join(repository, "modify/")
+        url = self.config["base_url"] + path
+        data = {"remove_content_units": artifacts}
+        return requests.post(url, json=data, **self.request_params)
+
     def delete_repository(self, repository):
         """
         Delete an RPM repository
@@ -190,7 +200,16 @@ def wait_for_finished_task(self, task):
             response = self.get_task(task)
             if not response.ok:
                 break
-            if response.json()["state"] != "waiting":
+            if response.json()["state"] not in ["waiting", "running"]:
                 break
             time.sleep(5)
         return response
+
+    def list_distributions(self, prefix):
+        """
+        Get a list of distributions whose names match a given prefix
+        https://pulpproject.org/pulp_rpm/restapi/#tag/Distributions:-Rpm/operation/distributions_rpm_rpm_list
+        """
+        url = self.url("api/v3/distributions/rpm/rpm/?")
+        url += urlencode({"name__startswith": prefix})
+        return requests.get(url, **self.request_params)

backend/copr_backend/storage.py

@@ -3,9 +3,10 @@
 """
 
 import os
+import json
 import shutil
 from copr_common.enums import StorageEnum
-from copr_backend.helpers import call_copr_repo
+from copr_backend.helpers import call_copr_repo, build_chroot_log_name
 from copr_backend.pulp import PulpClient
 
 
@@ -71,6 +72,18 @@ def delete_repository(self, chroot):
         """
         raise NotImplementedError
 
+    def delete_project(self, dirname):
+        """
+        Delete the whole project and all of its repositories and builds
+        """
+        raise NotImplementedError
+
+    def delete_builds(self, dirname, chroot_builddirs, build_ids):
+        """
+        Delete multiple builds from the storage
+        """
+        raise NotImplementedError
+
 
 class BackendStorage(Storage):
     """
@@ -116,6 +129,48 @@ def delete_repository(self, chroot):
             return
         shutil.rmtree(chroot_path)
 
+    def delete_project(self, dirname):
+        path = os.path.join(self.opts.destdir, self.owner, dirname)
+        if os.path.exists(path):
+            self.log.info("Removing copr dir %s", path)
+            shutil.rmtree(path)
+
+    def delete_builds(self, dirname, chroot_builddirs, build_ids):
+        result = True
+        for chroot, subdirs in chroot_builddirs.items():
+            chroot_path = os.path.join(
+                self.opts.destdir, self.owner, dirname, chroot)
+            if not os.path.exists(chroot_path):
+                self.log.error("%s chroot path doesn't exist", chroot_path)
+                result = False
+                continue
+
+            self.log.info("Deleting subdirs [%s] in %s",
+                          ", ".join(subdirs), chroot_path)
+
+            # Run createrepo first and then remove the files (to avoid old
+            # repodata temporarily pointing at non-existing files)!
+            # In srpm-builds we don't create repodata at all
+            if chroot != "srpm-builds":
+                repo = call_copr_repo(
+                    chroot_path, delete=subdirs, devel=self.devel,
+                    appstream=self.appstream, logger=self.log)
+                if not repo:
+                    result = False
+
+            for build_id in build_ids or []:
+                log_paths = [
+                    os.path.join(chroot_path, build_chroot_log_name(build_id)),
+                    # we used to create those before
+                    os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
+                    os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
+                for log_path in log_paths:
+                    try:
+                        os.unlink(log_path)
+                    except OSError:
+                        self.log.debug("can't remove %s", log_path)
+        return result
+
 
 class PulpStorage(Storage):
     """
@@ -148,6 +203,7 @@ def init_project(self, dirname, chroot):
         return response.ok
 
     def upload_build_results(self, chroot, results_dir, target_dir_name):
+        resources = []
         for root, _, files in os.walk(results_dir):
             for name in files:
                 if os.path.basename(root) == "prev_build_backup":
@@ -169,8 +225,23 @@ def upload_build_results(self, chroot, results_dir, target_dir_name):
                                    path, response.text)
                     continue
 
+                # This involves a lot of unnecessary waiting until every
+                # RPM content is created. Once we can reliably label Pulp
+                # content with Copr build ID, we should drop this code and stop
+                # creating the `pulp.json` file
+                task = response.json()["task"]
+                response = self.client.wait_for_finished_task(task)
+                created = response.json()["created_resources"]
+                resources.extend(created)
+
                 self.log.info("Uploaded to Pulp: %s", path)
 
+        data = {"resources": resources}
+        path = os.path.join(results_dir, "pulp.json")
+        with open(path, "w+", encoding="utf8") as fp:
+            json.dump(data, fp)
+        self.log.info("Pulp resources: %s", resources)
+
     def publish_repository(self, chroot, **kwargs):
         repository = self._get_repository(chroot)
         response = self.client.create_publication(repository)
@@ -206,6 +277,62 @@ def delete_repository(self, chroot):
         self.client.delete_repository(repository)
         self.client.delete_distribution(distribution)
 
+    def delete_project(self, dirname):
+        prefix = "{0}/{1}".format(self.owner, dirname)
+        response = self.client.list_distributions(prefix)
+        distributions = response.json()["results"]
+        for distribution in distributions:
+            self.client.delete_distribution(distribution["pulp_href"])
+            if distribution["repository"]:
+                self.client.delete_repository(distribution["repository"])
+
+    def delete_builds(self, dirname, chroot_builddirs, build_ids):
+        # pylint: disable=too-many-locals
+        result = True
+        for chroot, subdirs in chroot_builddirs.items():
+            # We don't upload results of source builds to Pulp
+            if chroot == "srpm-builds":
+                continue
+
+            chroot_path = os.path.join(
+                self.opts.destdir, self.owner, dirname, chroot)
+            if not os.path.exists(chroot_path):
+                self.log.error("%s chroot path doesn't exist", chroot_path)
+                result = False
+                continue
+
+            repository = self._get_repository(chroot)
+            for subdir in subdirs:
+                # It is currently not possible to set labels for Pulp content.
+                # https://github.com/pulp/pulpcore/issues/3338
+                # Until it is implemented, we need read all Pulp resources that
+                # a copr build created from our `pulp.json` in the resultdir.
+                path = os.path.join(chroot_path, subdir, "pulp.json")
+                with open(path, "r", encoding="utf8") as fp:
+                    pulp = json.load(fp)
+
+                for resource in pulp["resources"]:
+                    is_package = resource.split("/api/v3/")[1].startswith(
+                        "content/rpm/packages")
+                    if not is_package:
+                        self.log.info("Not deleting %s", resource)
+                        continue
+
+                    # TODO We can make performance improvements here by deleting
+                    # all content at once
+                    response = self.client.delete_content(repository, [resource])
+                    if response.ok:
+                        self.log.info("Successfully deleted Pulp content %s", resource)
+                    else:
+                        result = False
+                        self.log.info("Failed to delete Pulp content %s", resource)
+
+            published = self.publish_repository(chroot)
+            if not published:
+                result = False
+
+        return result
+
     def _repository_name(self, chroot, dirname=None):
         return "/".join([
             self.owner,

backend/tests/test_action.py

@@ -488,7 +488,7 @@ def test_delete_build_acr_reflected(self, mc_devel, mc_time, devel):
             assert new_primary['names'] == set(['prunerepo'])
             assert len(new_primary_devel['names']) == 3
 
-    @mock.patch("copr_backend.actions.call_copr_repo")
+    @mock.patch("copr_backend.storage.call_copr_repo")
     @mock.patch("copr_backend.actions.uses_devel_repo")
     def test_delete_build_succeeded_createrepo_error(self, mc_devel,
                                                      mc_call_repo, mc_time):