Get rid of debootstrap

Replace debootstrap with an apt-get based pre-download of
packages followed by a dpkg-deb extraction.
This Fixes #2599

kiwi/exceptions.py

@@ -208,10 +208,10 @@ class KiwiDataStructureError(KiwiError):
     """
 
 
-class KiwiDebootstrapError(KiwiError):
+class KiwiDebianBootstrapError(KiwiError):
     """
-    Exception raised if not enough user data to call debootstrap
-    were provided or the debootstrap has failed.
+    Exception raised if the bootstrap installation for Debian
+    based systems has failed
     """
 
 

kiwi/package_manager/apt.py

@@ -30,11 +30,12 @@
 from kiwi.package_manager.base import PackageManagerBase
 from kiwi.system.root_bind import RootBind
 from kiwi.repository.apt import RepositoryApt
+from kiwi.utils.temporary import Temporary
 
 import kiwi.defaults as defaults
 
 from kiwi.exceptions import (
-    KiwiDebootstrapError,
+    KiwiDebianBootstrapError,
     KiwiRequestError,
     KiwiFileNotFound
 )
@@ -131,12 +132,11 @@ def process_install_requests_bootstrap(
     ) -> CommandCallT:
         """
         Process package install requests for bootstrap phase (no chroot)
-        Either debootstrap or a prebuilt bootstrap package can be used
-        to bootstrap a new system.
+        Either a manual unpacking strategy or a prebuilt bootstrap
+        package can be used to bootstrap a new system.
 
         :param object root_bind:
-            instance of RootBind to manage kernel file systems before
-            debootstrap call
+            unused
         :param str bootstrap_package:
             package name of a bootstrap package
 
@@ -149,9 +149,7 @@ def process_install_requests_bootstrap(
                 bootstrap_package
             )
         else:
-            return self._process_install_requests_bootstrap_debootstrap(
-                root_bind
-            )
+            return self._process_install_requests_bootstrap()
 
     def _process_install_requests_bootstrap_prebuild_root(
         self, bootstrap_package: str
@@ -205,117 +203,78 @@ def _process_install_requests_bootstrap_prebuild_root(
         # Install eventual bootstrap packages as standard system install
         return self.process_install_requests()
 
-    def _process_install_requests_bootstrap_debootstrap(
-        self, root_bind: RootBind = None
-    ) -> CommandCallT:
+    def _process_install_requests_bootstrap(self) -> CommandCallT:
         """
-        Process package install requests for bootstrap phase (no chroot)
-        The debootstrap program is used to bootstrap a new system
+        Process package install requests for bootstrap phase (no chroot).
 
-        :param object root_bind: instance of RootBind to manage kernel
-            file systems before debootstrap call
-
-        :raises KiwiDebootstrapError: if no main distribution repository
-            is configured, if the debootstrap script is not found or if the
-            debootstrap script execution fails
+        :raises KiwiDebianBootstrapError
 
         :return: process results in command type
 
-        :rtype: namedtuple
+        :rtype: CommandCallT
         """
-        if not self.distribution:
-            raise KiwiDebootstrapError(
-                'No main distribution repository is configured'
+        # we invoke apt-get install to download all the essential packages.
+        # With DPkg::Pre-Install-Pkgs, we specify a shell command that will
+        # receive the list of packages that will be installed on stdin.
+        # By configuring Debug::pkgDpkgPm=1, apt-get install will not
+        # actually execute any dpkg commands, so all it does is download
+        # the essential debs and tell us their full in the apt cache without
+        # actually installing them.
+        try:
+            if 'apt' not in self.package_requests:
+                self.package_requests.append('apt')
+            update_cache = [
+                'apt-get'
+            ] + self.apt_get_args + self.custom_args + [
+                'update'
+            ]
+            result = Command.run(
+                update_cache, self.command_env
             )
-        bootstrap_script = '/usr/share/debootstrap/scripts/' + \
-            self.distribution
-        if not os.path.exists(bootstrap_script):
-            raise KiwiDebootstrapError(
-                'debootstrap script for %s distribution not found' %
-                self.distribution
+            log.debug(
+                'Apt update: {0} {1}'.format(result.output, result.error)
             )
-
-        # APT package manager does not support bootstrapping. To circumvent
-        # this limitation there is the debootstrap tool for APT based distros.
-        # Because of that there is a little overlap between KIWI and
-        # debootstrap. Debootstrap manages itself the kernel file systems for
-        # chroot environment, thus we need to umount the kernel file systems
-        # before calling debootstrap and remount them afterwards.
-        if root_bind:
-            root_bind.umount_kernel_file_systems()
-
-        # debootsrap will create its own dev/fd devices
-        debootstrap_device_node_conflicts = [
-            'dev/fd',
-            'dev/pts'
-        ]
-        for node in debootstrap_device_node_conflicts:
-            Path.wipe(os.path.normpath(os.sep.join([self.root_dir, node])))
-
-        if 'apt' in self.package_requests:
-            # debootstrap takes care to install apt
-            self.package_requests.remove('apt')
-        try:
-            cmd = ['debootstrap']
-            if self.repository.unauthenticated == 'false' and \
-               os.path.exists(self.repository.keyring):
-                cmd.append('--keyring={}'.format(self.repository.keyring))
-            else:
-                cmd.append('--no-check-gpg')
-            if self.deboostrap_minbase:
-                cmd.append('--variant=minbase')
-            if self.package_requests:
-                cmd.append(
-                    '--include={}'.format(','.join(self.package_requests))
-                )
-            if self.repository.components:
-                cmd.append(
-                    '--components={0}'.format(
-                        ','.join(self.repository.components)
+            package_names = Temporary(prefix='kiwi_debs_').new_file()
+            package_extract = Temporary(prefix='kiwi_bootstrap_').new_file()
+            download_bootstrap = [
+                'apt-get'
+            ] + self.apt_get_args + self.custom_args + [
+                'install',
+                '-oDebug::pkgDPkgPm=1',
+                f'-oDPkg::Pre-Install-Pkgs::=cat >{package_names.name}',
+                '?essential',
+                '?exact-name(usr-is-merged)'
+            ] + self.package_requests
+            result = Command.run(
+                download_bootstrap, self.command_env
+            )
+            log.debug(
+                'Apt download: {0} {1}'.format(result.output, result.error)
+            )
+            with open(package_extract.name, 'w') as install:
+                install.write(
+                    'while read -r deb;do echo "{0}";{1}|{2};done <{3}'.format(
+                        'Unpacking $deb',
+                        'dpkg-deb --fsys-tarfile $deb',
+                        f'tar -C {self.root_dir} -x',
+                        package_names.name
                     )
                 )
+            result = Command.run(
+                ['bash', package_extract.name], self.command_env
+            )
+            log.debug(
+                'Apt extract: {0} {1}'.format(result.output, result.error)
+            )
             self.cleanup_requests()
-            cmd.extend(
-                [self.distribution, self.root_dir, self.distribution_path]
+            return Command.call(
+                update_cache, self.command_env
             )
-
-            return Command.call(cmd, self.command_env)
         except Exception as e:
-            raise KiwiDebootstrapError(
+            raise KiwiDebianBootstrapError(
                 '%s: %s' % (type(e).__name__, format(e))
             )
 
-    def get_error_details(self) -> str:
-        """
-        Provide further error details
-
-        Read the debootstrap log if available
-
-        :rtype: str
-        """
-        debootstrap_log_file = os.path.join(
-            self.root_dir, 'debootstrap/debootstrap.log'
-        )
-        if os.path.exists(debootstrap_log_file):
-            with open(debootstrap_log_file) as log_fd:
-                return log_fd.read() or 'logfile is empty'
-        return f'logfile {debootstrap_log_file!r} does not exist'
-
-    def post_process_install_requests_bootstrap(
-        self, root_bind: RootBind = None, delta_root: bool = False
-    ) -> None:
-        """
-        Mounts the kernel file systems to the chroot environment is
-        ready after the bootstrap procedure
-
-        :param object root_bind:
-            instance of RootBind to manage kernel file systems
-        :param bool delta_root:
-            root is derived from a base system
-        """
-        if root_bind:
-            root_bind.mount_kernel_file_systems(delta_root)
-
     def process_install_requests(self) -> CommandCallT:
         """
         Process package install requests for image phase (chroot)

kiwi/repository/apt.py

@@ -72,7 +72,6 @@ def post_init(self, custom_args: List = []) -> None:
 
         self.distribution: str = ''
         self.distribution_path: str = ''
-        self.debootstrap_repo_set = False
         self.repo_names: List = []
         self.components: List = []
 
@@ -139,8 +138,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = None, pkg_gpgcheck: bool = None,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         """
         Add apt_get repository
@@ -157,8 +155,6 @@ def add_repo(
         :param bool repo_gpgcheck: enable repository signature validation
         :param bool pkg_gpgcheck: unused
         :param str sourcetype: unused
-        :param bool use_for_bootstrap: use this repository for the
-            debootstrap call
         :param str customization_script:
             custom script called after the repo file was created
         """
@@ -190,10 +186,8 @@ def add_repo(
             else:
                 # create a debian distributon repository setup for the
                 # specified distributon name and components
-                if not self.debootstrap_repo_set:
-                    self.distribution = dist
-                    self.distribution_path = uri
-                    self.debootstrap_repo_set = use_for_bootstrap
+                self.distribution = dist
+                self.distribution_path = uri
                 repo_details += 'Suites: ' + dist + os.linesep
                 repo_details += 'Components: ' + components + os.linesep
             if repo_gpgcheck is False:

kiwi/repository/base.py

@@ -77,7 +77,7 @@ def add_repo(
         self, name: str, uri: str, repo_type: str, prio: int, dist: str,
         components: str, user: str, secret: str, credentials_file: str,
         repo_gpgcheck: bool, pkg_gpgcheck: bool, sourcetype: str,
-        use_for_bootstrap: bool = False, customization_script: str = None
+        customization_script: str = None
     ) -> None:
         """
         Add repository
@@ -96,7 +96,6 @@ def add_repo(
         :param bool repo_gpgcheck: unused
         :param bool pkg_gpgcheck: unused
         :param str sourcetype: unused
-        :param bool use_for_bootstrap: unused
         :param str customization_script: unused
         """
         raise NotImplementedError

kiwi/repository/dnf.py

@@ -51,8 +51,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = False, pkg_gpgcheck: bool = False,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         pass  # pragma: no cover
 

kiwi/repository/dnf4.py

@@ -192,8 +192,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = False, pkg_gpgcheck: bool = False,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         """
         Add dnf repository
@@ -211,7 +210,6 @@ def add_repo(
         :param bool pkg_gpgcheck: enable package signature validation
         :param str sourcetype:
             source type, one of 'baseurl', 'metalink' or 'mirrorlist'
-        :param bool use_for_bootstrap: unused
         :param str customization_script:
             custom script called after the repo file was created
         """

kiwi/repository/dnf5.py

@@ -192,8 +192,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = False, pkg_gpgcheck: bool = False,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         """
         Add dnf repository
@@ -211,7 +210,6 @@ def add_repo(
         :param bool pkg_gpgcheck: enable package signature validation
         :param str sourcetype:
             source type, one of 'baseurl', 'metalink' or 'mirrorlist'
-        :param bool use_for_bootstrap: unused
         :param str customization_script:
             custom script called after the repo file was created
         """

kiwi/repository/pacman.py

@@ -115,8 +115,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = False, pkg_gpgcheck: bool = False,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         """
         Add pacman repository
@@ -133,7 +132,6 @@ def add_repo(
         :param bool repo_gpgcheck: enable database signature validation
         :param bool pkg_gpgcheck: enable package signature validation
         :param str sourcetype: unused
-        :param bool use_for_bootstrap: unused
         :param str customization_script:
             custom script called after the repo file was created
         """

kiwi/repository/template/apt.py

@@ -29,6 +29,7 @@ def __init__(self) -> None:
             # kiwi generated apt-get config file
             Dir "/";
             Dir::State "${apt_shared_base}/";
+            Dir::State::status "/var/lib/dpkg/status.kiwi";
             Dir::Cache "${apt_shared_base}/";
             Dir::Etc   "${apt_shared_base}/";
         ''').strip() + os.linesep

kiwi/repository/zypper.py

@@ -252,8 +252,7 @@ def add_repo(
         prio: int = None, dist: str = None, components: str = None,
         user: str = None, secret: str = None, credentials_file: str = None,
         repo_gpgcheck: bool = False, pkg_gpgcheck: bool = False,
-        sourcetype: str = None, use_for_bootstrap: bool = False,
-        customization_script: str = None
+        sourcetype: str = None, customization_script: str = None
     ) -> None:
         """
         Add zypper repository
@@ -270,7 +269,6 @@ def add_repo(
         :param bool repo_gpgcheck: enable repository signature validation
         :param bool pkg_gpgcheck: enable package signature validation
         :param str sourcetype: unused
-        :param boot use_for_bootstrap: unused
         :param str customization_script:
             custom script called after the repo file was created
         """