frontend, backend, common, rpmbuild: allow user SSH to builders

Fix fedora-copr#2364
See fedora-copr/debate#6

backend/copr_backend/background_worker_build.py

@@ -10,7 +10,9 @@
 import statistics
 import time
 import json
+import shlex
 
+from datetime import datetime
 from packaging import version
 
 from copr_common.enums import StatusEnum
@@ -38,6 +40,11 @@
 MIN_BUILDER_VERSION = "0.68.dev"
 CANCEL_CHECK_PERIOD = 5
 
+# See also copr-builder script
+USER_SSH_DEFAULT_EXPIRATION = 60 * 60
+USER_SSH_MAX_EXPIRATION = 60 * 60 * 24 * 2
+USER_SSH_EXPIRATION_PATH = "/run/copr-builder-expiration"
+
 MESSAGES = {
     "give_up_repo":
         "Giving up waiting for copr_base repository, "
@@ -51,6 +58,9 @@
 
 COMMANDS = {
     "rpm_q_builder": "rpm -q copr-rpmbuild --qf \"%{VERSION}\n\"",
+    "echo_authorized_keys": "echo {0} >> /root/.ssh/authorized_keys",
+    "set_expiration": "echo -n {0} > " + USER_SSH_EXPIRATION_PATH,
+    "cat_expiration": "cat {0}".format(USER_SSH_EXPIRATION_PATH),
 }
 
 
@@ -139,6 +149,7 @@ def __init__(self):
         self.builder_livelog = os.path.join(self.builder_dir, "main.log")
         self.builder_results = os.path.join(self.builder_dir, "results")
         self.ssh = None
+        self.root_ssh = None
         self.job = None
         self.host = None
         self.canceled = False
@@ -589,12 +600,23 @@ def _download_results(self):
         """
         Retry rsync-download the results several times.
         """
+        include = None
+        exclude = None
+
+        if self.job.allow_user_ssh:
+            self.log.info("Builder allowed user SSH, not downloading the "
+                          "results for safety reasons.")
+            include = ["success", "*.spec"]
+            exclude = ["*"]
+
         self.log.info("Downloading results from builder")
         self.ssh.rsync_download(
             self.builder_results + "/",
             self.job.results_dir,
             logfile=self.job.rsync_log_name,
             max_retries=2,
+            include=include,
+            exclude=exclude,
         )
 
     def _check_build_success(self):
@@ -740,6 +762,113 @@ def _add_pubkey(self):
         self.log.info("Added pubkey for user %s project %s into: %s",
                       user, project, pubkey_path)
 
+    @skipped_for_source_build
+    def _setup_for_user_ssh(self):
+        """
+        Setup the builder for user SSH
+        https://github.com/fedora-copr/debate/tree/main/user-ssh-builders
+
+        If the builder setup for user SSH becomes more complicated than just
+        installing the public key, we might want to move the code to a script
+        within `copr-builder` and call it here or from `copr-rpmbuild`. There
+        is no requirement for it to be here.
+        """
+        if not self.job.allow_user_ssh:
+            return
+        self._alloc_root_ssh_connection()
+        self._deploy_user_ssh()
+        self._set_default_expiration()
+
+    def _alloc_root_ssh_connection(self):
+        self.log.info("Allocating root ssh connection to builder")
+        self.root_ssh = SSHConnection(
+            user="root",
+            host=self.host.hostname,
+            config_file=self.opts.ssh.builder_config,
+            log=self.log,
+        )
+
+    def _deploy_user_ssh(self):
+        """
+        Deploy user public key to the builder, so that they can connect via SSH.
+        """
+        pubkey = shlex.quote(self.job.ssh_public_key)
+        cmd = COMMANDS["echo_authorized_keys"].format(pubkey)
+        rc, _out, _err = self.root_ssh.run_expensive(cmd)
+        if rc != 0:
+            self.log.error("Failed to deploy user SSH key for %s",
+                           self.job.project_owner)
+            return
+        self.log.info("Deployed user SSH key for %s", self.job.project_owner)
+
+    def _set_default_expiration(self):
+        """
+        Set the default expiration time for the builder
+        """
+        default = self.job.started_on + USER_SSH_DEFAULT_EXPIRATION
+        cmd = COMMANDS["set_expiration"].format(shlex.quote(str(default)))
+        rc, _out, _err = self.root_ssh.run_expensive(cmd)
+        if rc != 0:
+            # This only affects the `copr-builder show` command to print unknown
+            # remaining time. It won't affect the backend in terminating the
+            # buidler when it is supposed to
+            self.log.error("Failed to set the default expiration time")
+            return
+        self.log.info("The expiration time was set to %s", default)
+
+    def _builder_expiration(self):
+        """
+        Find the user preference for the builder expiration.
+        """
+        rc, out, _err = self.root_ssh.run_expensive(COMMANDS["cat_expiration"])
+        if rc == 0:
+            try:
+                return datetime.fromtimestamp(float(out))
+            except ValueError:
+                pass
+        self.log.error("Unable to query builder expiration file")
+        return None
+
+    def _keep_alive_for_user_ssh(self):
+        """
+        Wait until user releases the VM or until it expires.
+        """
+        if not self.job.allow_user_ssh:
+            return
+
+        # We are calculating the limits from when the job started but we may
+        # want to consider starting the watch when job ends.
+        default = datetime.fromtimestamp(
+            self.job.started_on + USER_SSH_DEFAULT_EXPIRATION)
+        maxlimit = datetime.fromtimestamp(
+            self.job.started_on + USER_SSH_MAX_EXPIRATION)
+
+        self.log.info("Keeping builder alive for user SSH")
+
+        def _keep_alive():
+            while True:
+                if self.canceled:
+                    self.log.warning("Build canceled, VM will be shut-down soon")
+                    break
+                expiration = self._builder_expiration() or default
+                if datetime.now() > expiration:
+                    self.log.warning("VM expired, it will be shut-down soon")
+                    break
+                if datetime.now() > maxlimit:
+                    msg = "VM exceeded max limit, it will be shut-down soon"
+                    self.log.warning(msg)
+                    break
+                time.sleep(60)
+
+        CancellableThreadTask(
+            _keep_alive,
+            self._cancel_task_check_request,
+            self._cancel_running_worker,
+            check_period=CANCEL_CHECK_PERIOD,
+        ).run()
+        if self.canceled:
+            raise BuildCanceled
+
     def build(self, attempt):
         """
         Attempt to build.
@@ -754,6 +883,7 @@ def build(self, attempt):
         self._fill_build_info_file()
         self._cancel_if_requested()
         self._mark_running(attempt)
+        self._setup_for_user_ssh()
         self._start_remote_build()
         transfer_failure = CancellableThreadTask(
             self._transfer_log_file,
@@ -766,6 +896,8 @@ def build(self, attempt):
         if transfer_failure:
             raise BuildRetry("SSH problems when downloading live log: {}"
                              .format(transfer_failure))
+
+        self._keep_alive_for_user_ssh()
         self._download_results()
         self._drop_host()
 

backend/copr_backend/daemons/build_dispatcher.py

@@ -8,6 +8,7 @@
     ArchitectureWorkerLimit,
     ArchitectureUserWorkerLimit,
     BuildTagLimit,
+    UserSSHLimit,
     RPMBuildWorkerManager,
     BuildQueueTask,
 )
@@ -105,6 +106,14 @@ def __init__(self, backend_opts):
                 name=limit_type,
             ))
 
+        userssh = UserSSHLimit(
+            None,
+            # lambda x: getattr(x, "userssh"),
+            # "foo",
+            backend_opts.builds_limits["userssh"])
+        self.limits.append(userssh)
+
+
     def get_frontend_tasks(self):
         """
         Retrieve a list of build jobs to be done.

backend/copr_backend/helpers.py

@@ -240,6 +240,8 @@ def _get_limits_conf(parser):
         parser, "backend", "builds_max_workers_sandbox", 10, mode="int")
     limits['owner'] = _get_conf(
         parser, "backend", "builds_max_workers_owner", 20, mode="int")
+    limits['userssh'] = _get_conf(
+        parser, "backend", "builds_max_userssh", 2, mode="int")
     return limits
 
 

backend/copr_backend/job.py

@@ -27,6 +27,7 @@ def __init__(self, task_data, worker_opts):
                 - timeout: default worker timeout
 
         """
+        # pylint: disable=too-many-statements
 
         self.timeout = worker_opts.timeout
         self.frontend_base_url = worker_opts.frontend_base_url
@@ -72,6 +73,8 @@ def __init__(self, task_data, worker_opts):
 
         self.results = None
         self.appstream = None
+        self.allow_user_ssh = None
+        self.ssh_public_key = None
 
         # TODO: validate update data
         for key, val in task_data.items():

backend/copr_backend/rpm_builds.py

@@ -2,6 +2,7 @@
 Abstraction for RPM and SRPM builds on backend.
 """
 
+import uuid
 from copr_common.worker_manager import (
     HashWorkerLimit,
     WorkerManager,
@@ -150,6 +151,25 @@ def __init__(self, architecture, limit):
         )
 
 
+class UserSSHLimit(HashWorkerLimit):
+    """
+    Limit the number of builders that allow user SSH
+    """
+    def __init__(self, _hasher, limit):
+        def predicate(x):
+            # We don't allow user SSH for SRPM builds, return unique string
+            # making this unlimited
+            if x.source_build:
+                return str(uuid.uuid4())
+
+            # Not limit builds that doesn't allow user SSH
+            # pylint: disable=protected-access
+            if not x._task.get("allow_user_ssh"):
+                return str(uuid.uuid4())
+            return x.owner
+        super().__init__(predicate, limit, name="userssh")
+
+
 class BuildTagLimit(PredicateWorkerLimit):
     """
     Limit the amount of concurrently running builds per given build tag.

backend/copr_backend/sshcmd.py

@@ -213,7 +213,7 @@ def _full_source_path(self, src):
         return "{}@{}:{}".format(self.user, host, src)
 
     def rsync_download(self, src, dest, logfile=None, max_retries=0,
-                       subprocess_timeout=None):
+                       subprocess_timeout=None, include=None, exclude=None):
         """
         Run rsync over pre-allocated socket (by the config)
 
@@ -231,9 +231,10 @@ def rsync_download(self, src, dest, logfile=None, max_retries=0,
         directory needs to exist.
         """
         self._retry(self._rsync_download, max_retries, src, dest, logfile,
-                    subprocess_timeout)
+                    subprocess_timeout, include, exclude)
 
-    def _rsync_download(self, src, dest, logfile, subprocess_timeout):
+    def _rsync_download(self, src, dest, logfile, subprocess_timeout,
+                        include, exclude):
         ssh_opts = "ssh"
         if self.config_file:
             ssh_opts += " -F " + self.config_file
@@ -243,8 +244,23 @@ def _rsync_download(self, src, dest, logfile, subprocess_timeout):
         log_filepath = "/dev/null"
         if logfile:
             log_filepath = os.path.join(dest, logfile)
-        command = "/usr/bin/rsync -rltDvH --chmod=D755,F644 -e '{}' {} {}/ &> {}".format(
-            ssh_opts, full_source_path, dest, log_filepath)
+
+        command = [
+            "/usr/bin/rsync",
+            "-rltDvH",
+            "--chmod=D755,F644",
+            "-e", "'{}'".format(ssh_opts),
+        ]
+        for value in include or []:
+            command.extend(["--include", shlex.quote(value)])
+        for value in exclude or []:
+            command.extend(["--exclude", shlex.quote(value)])
+        command.extend([
+            full_source_path,
+            "{}/".format(dest),
+            "&>", log_filepath,
+        ])
+        command = " ".join(command)
 
         self.log.info("rsyncing of %s to %s started", full_source_path, dest)
         with self._popen_timeouted(command, shell=True) as cmd:

backend/tests/test_config_reader.py

@@ -38,7 +38,8 @@ def test_minimal_file_and_defaults(self):
         opts = BackendConfigReader(self.get_minimal_config_file()).read()
         assert opts.destdir == "/tmp"
         assert opts.builds_limits == {'arch': {}, 'tag': {}, 'owner': 20,
-                                      'sandbox': 10, 'arch_per_owner': {}}
+                                      'sandbox': 10, 'arch_per_owner': {},
+                                      'userssh': 2}
 
     def test_correct_build_limits(self):
         opts = BackendConfigReader(
@@ -65,6 +66,7 @@ def test_correct_build_limits(self):
                 'ppc64le': 11,
                 's390x': 5,
             },
+            'userssh': 2,
         }
 
     @pytest.mark.parametrize("broken_config", [

backend/tests/testlib/__init__.py

@@ -178,7 +178,8 @@ def _full_source_path(self, src):
         return src
 
     def rsync_download(self, src, dest, logfile=None, max_retries=0,
-                       subprocess_timeout=DEFAULT_SUBPROCESS_TIMEOUT):
+                       subprocess_timeout=DEFAULT_SUBPROCESS_TIMEOUT,
+                       include=None, exclude=None):
         data = os.environ["TEST_DATA_DIRECTORY"]
         trail_slash = src.endswith("/")
         src = os.path.join(data, "build_results", self.resultdir)

common/copr_common/helpers.py

@@ -41,3 +41,14 @@ def chroot_to_branch(chroot):
     elif name == "mageia":
         abbrev = "mga"
     return "{}{}".format(abbrev, version)
+
+
+def timedelta_to_dhms(delta):
+    """
+    By default the `datetime.timedelta` provides only days and seconds. Minutes,
+    hours, and the human friendly number of seconds, needs to be calculated.
+    """
+    days, remainder = divmod(delta.total_seconds(), 24 * 60 * 60)
+    hours, remainder = divmod(remainder, 60 * 60)
+    minutes, seconds = divmod(remainder, 60)
+    return int(days), int(hours), int(minutes), int(seconds)

common/python-copr-common.spec

@@ -16,7 +16,7 @@
 %endif
 
 Name:       python-copr-common
-Version:    0.21
+Version:    0.21.1.dev
 Release:    1%{?dist}
 Summary:    Python code used by Copr
 

common/setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name='copr-common',
-    version="0.21",
+    version="0.21.1.dev",
     description=__description__,
     long_description=long_description,
     author=__author__,