From 8064538ebf1032337a71fd2d2e522cf63de1a12a Mon Sep 17 00:00:00 2001 From: Fabian Vogt Date: Wed, 14 Feb 2024 12:51:58 +0100 Subject: [PATCH] Allow specifying the filesystem for live image, also direct squashfs Allow specifying the filesystem for live image, also direct squashfs By setting it's now possible to specify the filesystem used for live images. By using "squashfs", the rootfs container is skipped entirely. --- doc/source/building_images/build_live_iso.rst | 48 ++++--- dracut/modules.d/90kiwi-live/kiwi-live-lib.sh | 8 ++ kiwi/builder/live.py | 120 +++++++++++------- kiwi/schema/kiwi.rnc | 2 +- kiwi/schema/kiwi.rng | 2 +- test/unit/builder/live_test.py | 81 +++++++----- 6 files changed, 166 insertions(+), 95 deletions(-) diff --git a/doc/source/building_images/build_live_iso.rst b/doc/source/building_images/build_live_iso.rst index 03ee241b60a..650bc5ab9ee 100644 --- a/doc/source/building_images/build_live_iso.rst +++ b/doc/source/building_images/build_live_iso.rst @@ -32,16 +32,29 @@ To add a live ISO build to your appliance, create a `type` element with The following attributes of the `type` element are relevant when building live ISO images: -- `flags`: Specifies the live ISO technology and dracut module to use, can - be set to `overlay` or to `dmsquash`. - - If set to `overlay`, the kiwi-live dracut module will be used to support a - live ISO system based on squashfs and overlayfs. - If set to `dmsquash`, the dracut standard dmsquash-live module will be - used to support a live ISO system based on squashfs and the device - mapper. Note, both modules support a different set of live features. +- `flags`: Specifies the dracut module to use. + + If set to `overlay`, the kiwi supplied kiwi-live dracut module will used + for booting. + + If set to `dmsquash`, the dracut supplied dmsquash-live module will be + used for booting. + + Both modules support a different set of live features. For details see :ref:`live_features` +- `filesystem`: Specifies the root filesystem for the live system. + + If set to `squashfs`, the root filesystem is written into a squashfs image. + This option is not compatible with device-mapper specific features of the + dmsquash-live dracut module. In that case, using overayfs is required. + + If set to a value different from `squashfs`, the root filesystem is written + into a filesystem image of the specified type and that filesystem image + written into a squashfs image for compression. + + The default value for this option is `ext4`. + - `hybridpersistent`: Accepts `true` or `false`, if set to `true` then the resulting image will be created with a COW file to keep data persistent over a reboot @@ -78,13 +91,14 @@ deployment. Decision for a live ISO technology ---------------------------------- -The decision for the `overlay` vs. `dmsquash` dracut module depends on -the features one wants to use. From a design perspective the `overlay` -module is conceived for live ISO deployments on disk devices which -allows the creation of a write partition or cow file. The `dmsquash` -module is conceived as a generic mapping technology using device-mapper -snapshots. The following list describes important live ISO features and -their support status compared to the `overlay` and `dmsquash` modules. +The decision for the `overlay` vs. `dmsquash` dracut module depends on the +features one wants to use. The `overlay` module only supports overlayfs +based overlays, but with automatic creation of a writable layer for +persistence. The `dmsquash` module supports overlayfs as well as +device-mapper based overlays. + +The following list describes important live ISO features and their support +status compared to the `overlay` and `dmsquash` modules. ISO scan Usable in the same way with both dracut modules. This feature allows @@ -100,8 +114,8 @@ ISO in RAM completely to setup RAM only deployments in {kiwi} see: :ref:`ramdisk_deployment` Overlay based on overlayfs - Usable with the `overlay` module. A squashfs compressed readonly root - gets overlayed with a readwrite filesystem using the kernel overlayfs + Usable with both dracut modules. The readonly root filesystem gets + overlayed with a readwrite filesystem using the kernel overlayfs filesystem. Overlay based on device mapper snapshots diff --git a/dracut/modules.d/90kiwi-live/kiwi-live-lib.sh b/dracut/modules.d/90kiwi-live/kiwi-live-lib.sh index 8549ea6aba5..83d437e0b01 100755 --- a/dracut/modules.d/90kiwi-live/kiwi-live-lib.sh +++ b/dracut/modules.d/90kiwi-live/kiwi-live-lib.sh @@ -134,6 +134,14 @@ function mountReadOnlyRootImageFromContainer { local rootfs_image="${container_mount_point}/LiveOS/rootfs.img" local root_mount_point="${overlay_base}/rootfsbase" mkdir -m 0755 -p "${root_mount_point}" + + if ! [ -e "${rootfs_image}" ] && [ -d "${container_mount_point}/proc" ]; then + # It's the root filesystem directly, just do a bind mount + mount -n --bind "${container_mount_point}" "${root_mount_point}" + echo "${root_mount_point}" + return + fi + if ! mount -n "${rootfs_image}" "${root_mount_point}"; then die "Failed to mount live ISO root filesystem" fi diff --git a/kiwi/builder/live.py b/kiwi/builder/live.py index c72507a40bb..9165cd46003 100644 --- a/kiwi/builder/live.py +++ b/kiwi/builder/live.py @@ -230,7 +230,9 @@ def create(self) -> Result: self.live_type ) ) - root_filesystem = Defaults.get_default_live_iso_root_filesystem() + root_filesystem = self.xml_state.build_type.get_filesystem() + root_filesystem = root_filesystem if root_filesystem else \ + Defaults.get_default_live_iso_root_filesystem() filesystem_custom_parameters = { 'mount_options': self.xml_state.get_fs_mount_option_list(), 'create_options': self.xml_state.get_fs_create_option_list() @@ -238,57 +240,81 @@ def create(self) -> Result: filesystem_setup = FileSystemSetup( self.xml_state, self.root_dir ) - root_image = Temporary().new_file() - with LoopDevice( - root_image.name, - filesystem_setup.get_size_mbytes(root_filesystem), - self.xml_state.build_type.get_target_blocksize() - ) as loop_provider: - loop_provider.create() + if root_filesystem != 'squashfs': + # Create a filesystem image of the specified type + # and put it into a SquashFS container + root_image = Temporary().new_file() + with LoopDevice( + root_image.name, + filesystem_setup.get_size_mbytes(root_filesystem), + self.xml_state.build_type.get_target_blocksize() + ) as loop_provider: + loop_provider.create() + with FileSystem.new( + name=root_filesystem, + device_provider=loop_provider, + root_dir=self.root_dir + os.sep, + custom_args=filesystem_custom_parameters + ) as live_filesystem: + live_filesystem.create_on_device() + log.info( + '--> Syncing data to {0} root image'.format(root_filesystem) + ) + live_filesystem.sync_data( + Defaults. + get_exclude_list_for_root_data_sync() + Defaults. + get_exclude_list_from_custom_exclude_files(self.root_dir) + ) + + log.info('--> Creating squashfs container for root image') + self.live_container_dir = Temporary( + prefix='live-container.', path=self.target_dir + ).new_dir() + Path.create(self.live_container_dir.name + '/LiveOS') + shutil.copy( + root_image.name, self.live_container_dir.name + '/LiveOS/rootfs.img' + ) + with FileSystem.new( + name='squashfs', + device_provider=DeviceProvider(), + root_dir=self.live_container_dir.name, + custom_args={ + 'compression': + self.xml_state.build_type.get_squashfscompression() + } + ) as live_container_image: + container_image = Temporary().new_file() + live_container_image.create_on_file( + container_image.name + ) + Path.create(self.media_dir.name + '/LiveOS') + os.chmod(container_image.name, 0o644) + shutil.copy( + container_image.name, + self.media_dir.name + '/LiveOS/squashfs.img' + ) + else: + # Put the root filesystem into SquashFS directly with FileSystem.new( - name=root_filesystem, - device_provider=loop_provider, + name='squashfs', + device_provider=DeviceProvider(), root_dir=self.root_dir + os.sep, - custom_args=filesystem_custom_parameters - ) as live_filesystem: - live_filesystem.create_on_device() - log.info( - '--> Syncing data to {0} root image'.format(root_filesystem) + custom_args={ + 'compression': + self.xml_state.build_type.get_squashfscompression() + } + ) as live_container_image: + container_image = Temporary().new_file() + live_container_image.create_on_file( + container_image.name ) - live_filesystem.sync_data( - Defaults. - get_exclude_list_for_root_data_sync() + Defaults. - get_exclude_list_from_custom_exclude_files(self.root_dir) + Path.create(self.media_dir.name + '/LiveOS') + os.chmod(container_image.name, 0o644) + shutil.copy( + container_image.name, + self.media_dir.name + '/LiveOS/squashfs.img' ) - log.info('--> Creating squashfs container for root image') - self.live_container_dir = Temporary( - prefix='live-container.', path=self.target_dir - ).new_dir() - Path.create(self.live_container_dir.name + '/LiveOS') - shutil.copy( - root_image.name, self.live_container_dir.name + '/LiveOS/rootfs.img' - ) - with FileSystem.new( - name='squashfs', - device_provider=DeviceProvider(), - root_dir=self.live_container_dir.name, - custom_args={ - 'compression': - self.xml_state.build_type.get_squashfscompression() - } - ) as live_container_image: - container_image = Temporary().new_file() - live_container_image.create_on_file( - container_image.name - ) - Path.create(self.media_dir.name + '/LiveOS') - os.chmod(container_image.name, 0o644) - shutil.copy( - container_image.name, - self.media_dir.name + '/LiveOS/squashfs.img' - ) - # create iso filesystem from media_dir log.info('Creating live ISO image') with FileSystemIsoFs( diff --git a/kiwi/schema/kiwi.rnc b/kiwi/schema/kiwi.rnc index 775711276fe..00d0e80aa1e 100644 --- a/kiwi/schema/kiwi.rnc +++ b/kiwi/schema/kiwi.rnc @@ -1661,7 +1661,7 @@ div { } >> sch:pattern [ id = "filesystem" is-a = "image_type" sch:param [ name = "attr" value = "filesystem" ] - sch:param [ name = "types" value = "oem pxe kis" ] + sch:param [ name = "types" value = "oem pxe kis iso" ] ] >> sch:pattern [ id = "filesystem_mandatory" is-a = "image_type_requirement" diff --git a/kiwi/schema/kiwi.rng b/kiwi/schema/kiwi.rng index 4d20e5882b8..1499b74b15b 100644 --- a/kiwi/schema/kiwi.rng +++ b/kiwi/schema/kiwi.rng @@ -2403,7 +2403,7 @@ structure - + diff --git a/test/unit/builder/live_test.py b/test/unit/builder/live_test.py index 6128e526578..2004ff93393 100644 --- a/test/unit/builder/live_test.py +++ b/test/unit/builder/live_test.py @@ -1,7 +1,7 @@ from mock import ( MagicMock, patch, call, Mock ) -from pytest import raises +from pytest import raises, mark import sys from kiwi.bootloader.config.grub2 import BootLoaderConfigGrub2 import kiwi.builder.live @@ -171,6 +171,7 @@ def test_create_overlay_structure_boot_on_systemd_boot( ['mv', 'kiwi_used_initrd_name', 'root_dir/boot/dracut_initrd_name'] ) + @mark.parametrize('xml_filesystem', [None, 'squashfs']) @patch('kiwi.builder.live.create_boot_loader_config') @patch('kiwi.builder.live.LoopDevice') @patch('kiwi.builder.live.DeviceProvider') @@ -189,7 +190,7 @@ def test_create_overlay_structure_boot_on_grub( self, mock_chmod, mock_exists, mock_grub_dir, mock_size, mock_filesystem, mock_isofs, mock_Iso, mock_tag, mock_shutil, mock_Temporary, mock_setup_media_loader_directory, mock_DeviceProvider, - mock_LoopDevice, mock_create_boot_loader_config + mock_LoopDevice, mock_create_boot_loader_config, xml_filesystem ): bootloader_config = Mock() mock_create_boot_loader_config.return_value.__enter__.return_value = \ @@ -218,6 +219,10 @@ def side_effect(): self.live_image.live_type = 'overlay' + self.xml_state.build_type.get_filesystem = Mock( + return_value=xml_filesystem + ) + iso_image = Mock() iso_image.create_on_file.return_value = 'offset' mock_isofs.return_value.__enter__.return_value = iso_image @@ -237,34 +242,52 @@ def side_effect(): self.setup.import_cdroot_files.assert_called_once_with('temp_media_dir') - assert kiwi.builder.live.FileSystem.new.call_args_list == [ - call( - device_provider=loop_provider, name='ext4', - root_dir='root_dir/', - custom_args={ - 'mount_options': ['async'], - 'create_options': ['-O', 'option'] - } - ), - call( - device_provider=mock_DeviceProvider.return_value, - name='squashfs', - root_dir='temp-squashfs', - custom_args={'compression': 'lzo'} - ) - ] - - filesystem.create_on_device.assert_called_once_with() - filesystem.sync_data.assert_called_once_with([ - 'image', '.profile', '.kconfig', - 'run/*', 'tmp/*', '.buildenv', 'var/cache/kiwi' - ]) - filesystem.create_on_file.assert_called_once_with('kiwi-tmpfile') + if xml_filesystem == 'squashfs': + assert kiwi.builder.live.FileSystem.new.call_args_list == [ + call( + device_provider=mock_DeviceProvider.return_value, + name='squashfs', + root_dir='root_dir/', + custom_args={'compression': 'lzo'} + ) + ] + + filesystem.create_on_file.assert_called_once_with('kiwi-tmpfile') + + assert mock_shutil.copy.call_args_list == [ + call('kiwi-tmpfile', 'temp_media_dir/LiveOS/squashfs.img') + ] + else: + assert kiwi.builder.live.FileSystem.new.call_args_list == [ + call( + device_provider=loop_provider, name='ext4', + root_dir='root_dir/', + custom_args={ + 'mount_options': ['async'], + 'create_options': ['-O', 'option'] + } + ), + call( + device_provider=mock_DeviceProvider.return_value, + name='squashfs', + root_dir='temp-squashfs', + custom_args={'compression': 'lzo'} + ) + ] + + filesystem.create_on_device.assert_called_once_with() + filesystem.sync_data.assert_called_once_with([ + 'image', '.profile', '.kconfig', + 'run/*', 'tmp/*', '.buildenv', 'var/cache/kiwi' + ]) + + filesystem.create_on_file.assert_called_once_with('kiwi-tmpfile') + + assert mock_shutil.copy.call_args_list == [ + call('kiwi-tmpfile', 'temp-squashfs/LiveOS/rootfs.img'), + call('kiwi-tmpfile', 'temp_media_dir/LiveOS/squashfs.img') + ] - assert mock_shutil.copy.call_args_list == [ - call('kiwi-tmpfile', 'temp-squashfs/LiveOS/rootfs.img'), - call('kiwi-tmpfile', 'temp_media_dir/LiveOS/squashfs.img') - ] assert mock_chmod.call_args_list == [ call('initrd', 0o644), call('kiwi-tmpfile', 0o644) ]