From 11dab295276db31aa5b317ea49a061ce63f91a18 Mon Sep 17 00:00:00 2001 From: Martin Pohlack Date: Wed, 14 Oct 2026 12:59:16 +0000 Subject: [PATCH] Add optional sparse data sync to minimize image size Provide a runtime configuration key to let the data sync that installs the image payload turn sequences of null bytes in the source into holes on the target: sync: sparse: true The key is off by default. It adds the --sparse option to the rsync call. It is skipped with a warning rather than failing the build if it cannot be used, which is the case for rsync releases before 3.1.3 because they reject --sparse together with --inplace, and for a caller that asked for --preallocate because that allocates every file in full before writing it. The option alone is not enough on a filesystem that performs speculative preallocation. rsync only seeks forward across a null sequence, it does not punch a hole, and on xfs the skipped range falls inside the preallocation beyond the end of file and is recorded as an allocated but unwritten extent. Such an extent reads as zero and is reported as a hole by SEEK_HOLE, yet it still occupies space in the target image. A filesystem implementation can therefore declare mount options that its install time mount needs for a sparse sync to be effective, via the new sparse_install_mount_options attribute. These apply to the mount the payload is synced into and are not passed on to the fstab of the image, so they do not become mount options of the running system. An explicit setting of the same option in the fsmountoptions attribute of the image description takes precedence. The xfs implementation declares the minimum of allocsize=4k for maximal size savings. Measured with a payload of 64k aligned binaries carrying a zero tail per segment, as produced for architectures with a 64k page size. The filesystem accounts 244MB used and the image file occupies 581MB without the key. With the key it is 119MB and 457MB, with no unwritten extents left. The extent count drops at the same time, because the unwritten bridges are extents of their own. Signed-off-by: Martin Pohlack Assisted-by: Kiro:claude-opus-5 --- kiwi.yml | 24 ++++ kiwi/defaults.py | 15 +++ kiwi/filesystem/base.py | 47 +++++++- kiwi/filesystem/xfs.py | 9 ++ kiwi/runtime_config.py | 32 +++++ kiwi/utils/sync.py | 67 ++++++++++ .../kiwi_config/ok/.config/kiwi/config.yml | 3 + .../kiwi_config/other/.config/kiwi/config.yml | 3 + test/unit/defaults_test.py | 3 + test/unit/filesystem/base_test.py | 50 ++++++++ test/unit/runtime_config_test.py | 3 + test/unit/utils/sync_test.py | 114 +++++++++++++++++- 12 files changed, 366 insertions(+), 4 deletions(-) diff --git a/kiwi.yml b/kiwi.yml index 0ef035b03ba..6784090adf9 100644 --- a/kiwi.yml +++ b/kiwi.yml @@ -64,6 +64,30 @@ # options: '--threads=0' +# Setup behaviour of the data sync used to install the image payload +#sync: +# # Specify if sequences of null bytes in the source data should be +# # turned into holes on the target. This adds the --sparse option to +# # the rsync call. It costs sync time because the data has to be +# # scanned for null sequences, and the gain depends on the payload. +# # It is effective for binaries built for a 64k page size, which +# # carry a zero tail per segment. +# # Default: false +# # +# # Note: the option is skipped with a warning if it cannot be used, +# # which is the case for rsync releases before 3.1.3 because they +# # reject --sparse together with --inplace. +# # +# # Note: on filesystems that perform speculative preallocation the +# # option alone does not reclaim space, because the skipped range +# # becomes an allocated but unwritten extent rather than a hole. For +# # those filesystems KIWI adds the mount options needed to prevent +# # this to the install time mount only, so they do not appear in the +# # fstab of the image. An explicit setting of the same option in the +# # fsmountoptions attribute of the image description takes precedence +# sparse: true + + # Setup process parameters for container image creation #container: # # Specify compression for container images diff --git a/kiwi/defaults.py b/kiwi/defaults.py index 3e9ff5d9415..18ca06d8ac5 100644 --- a/kiwi/defaults.py +++ b/kiwi/defaults.py @@ -412,6 +412,21 @@ def get_sync_options(): '--one-file-system', '--inplace' ] + @staticmethod + def get_sync_sparse(): + """ + Provides default setting for sparse data sync + + If enabled, rsync is called with the --sparse option which + turns sequences of null bytes in the source data into holes + on the target. + + :return: True or False + + :rtype: bool + """ + return False + @staticmethod def get_removed_files_name(): """ diff --git a/kiwi/filesystem/base.py b/kiwi/filesystem/base.py index ea37021c9ec..4dd0fad5f82 100644 --- a/kiwi/filesystem/base.py +++ b/kiwi/filesystem/base.py @@ -29,6 +29,7 @@ import kiwi.defaults as defaults from kiwi.defaults import Defaults +from kiwi.runtime_config import RuntimeConfig from kiwi.utils.sync import DataSync from kiwi.mount_manager import MountManager from kiwi.command import Command @@ -53,6 +54,14 @@ class FileSystemBase: :param string root_dir: root directory path name :param dict custom_args: custom filesystem arguments """ + # Mount options applied only to the install time mount of a newly created + # filesystem. They are deliberately not part of the mount options from the + # image description and therefore never end up in the fstab of the + # image. A filesystem implementation declares options here when sparse + # sync cannot work without them, see the xfs implementation for an + # example. + sparse_install_mount_options: List[str] = [] + def __init__( self, device_provider: DeviceProvider, root_dir: str = '', custom_args: Dict = {} @@ -196,7 +205,7 @@ def sync_data(self, exclude: List[str] = []) -> MountManager: device=self.device_provider.get_device() ) self.filesystem_mount.mount( - self.custom_args['mount_options'] + self._get_install_mount_options() ) self._apply_attributes() data = DataSync( @@ -207,6 +216,42 @@ def sync_data(self, exclude: List[str] = []) -> MountManager: ) return self.filesystem_mount + def _get_install_mount_options(self) -> List[str]: + """ + Provide mount options for the install time mount of the + filesystem the image payload is synced into. + + These are the mount options from the image description, plus + any options the filesystem implementation declares as needed + for a sparse data sync. The latter apply to this mount only + and are not passed on to the fstab of the image. + + As everywhere else in kiwi the result is a list of at most one + element, holding the comma separated option string that is + passed along to the -o mount option. + + :return: max one element list with mount option string + + :rtype: list + """ + given_options = self.custom_args['mount_options'] + if not self.sparse_install_mount_options or \ + not RuntimeConfig().get_sync_sparse(): + return list(given_options) + options = ','.join(given_options).split(',') if given_options else [] + given_names = [option.split('=')[0] for option in options if option] + for option in self.sparse_install_mount_options: + if option.split('=')[0] in given_names: + # an explicit setting from the image description wins + continue + log.info( + 'Using install time mount option {0} for sparse sync'.format( + option + ) + ) + options.append(option) + return [','.join(options)] if options else [] + def create_verity_layer( self, blocks: Optional[int] = None, filename: str = None ): diff --git a/kiwi/filesystem/xfs.py b/kiwi/filesystem/xfs.py index ff253cfd13f..36e8acbbd25 100644 --- a/kiwi/filesystem/xfs.py +++ b/kiwi/filesystem/xfs.py @@ -26,6 +26,15 @@ class FileSystemXfs(FileSystemBase): """ **Implements creation of xfs filesystem** """ + # Minimize speculative preallocation when a sparse data sync is requested. + # + # rsync --sparse only seeks forward across a sequence of null bytes, it + # does not punch a hole. On xfs most small skips lead to speculative + # preallocation (optimized for a fast-changing runtime) instead of the + # desired holes (as desired during one-time image construction). + # rsync therefore fails to save most space without this setting. + sparse_install_mount_options = ['allocsize=4k'] + def create_on_device( self, label: str = None, size: int = 0, unit: str = defaults.UNIT.kb, uuid: str = None diff --git a/kiwi/runtime_config.py b/kiwi/runtime_config.py index 36476e6aaae..6a5e069c1cb 100644 --- a/kiwi/runtime_config.py +++ b/kiwi/runtime_config.py @@ -332,6 +332,38 @@ def get_xz_options(self) -> Optional[List[str]]: xz_options = self._get_attribute(element='xz', attribute='options') return xz_options.split() if xz_options else None + def get_sync_sparse(self) -> bool: + """ + Return if data sync operations should turn sequences of null bytes + into holes on the target. + + sync: + sparse: true|false + + If no configuration exists the default from the Defaults + class is returned. + + Enabling this adds the --sparse option to the rsync call used + to install the image payload. On filesystems that perform + speculative preallocation the option alone does not reclaim + the space, because the skipped range is recorded as an + allocated but unwritten extent rather than as a hole. Such + filesystems can declare additional install time mount options + via the sparse_install_mount_options attribute of their + FileSystemBase implementation. + + :return: True or False + + :rtype: bool + + """ + sync_sparse = self._get_attribute( + element='sync', attribute='sparse' + ) + if sync_sparse is None: + return Defaults.get_sync_sparse() + return bool(sync_sparse) + def get_container_compression(self) -> bool: """ Return compression for container images diff --git a/kiwi/utils/sync.py b/kiwi/utils/sync.py index b338ca8ba6e..8313f175300 100644 --- a/kiwi/utils/sync.py +++ b/kiwi/utils/sync.py @@ -16,6 +16,7 @@ # along with kiwi. If not, see # import os +import re import errno import logging from stat import ST_MODE @@ -23,6 +24,7 @@ # project from kiwi.command import Command +from kiwi.runtime_config import RuntimeConfig log = logging.getLogger('kiwi') @@ -87,6 +89,9 @@ def sync_data( rsync_options = [] if options: rsync_options = options + if RuntimeConfig().get_sync_sparse(): + rsync_options = \ + rsync_options + self._get_sparse_options(rsync_options) if not self.target_supports_extended_attributes(): warn_me = False if '--xattrs' in rsync_options: @@ -125,6 +130,68 @@ def sync_data( # not changed os.chmod(self.target_dir, target_entry_permissions) + def _get_sparse_options(self, rsync_options: List[str]) -> List[str]: + """ + Provide the --sparse option if it is both wanted and usable + + The option turns sequences of null bytes into holes on the + target. It is skipped if the caller already asked for it, and + it is skipped with a warning if it cannot be combined with the + options the caller provided: + + * rsync releases before 3.1.3 reject --sparse together with + --inplace. kiwi syncs with --inplace, so on such a host the + combination would abort the build. + * --preallocate defeats the purpose, because every file is + allocated in full before it is written. The result occupies + the space of a dense file while a hole detecting reader still + sees holes, which is worse than not asking for sparse at all. + + :param list rsync_options: options provided by the caller + + :return: list of options to add + + :rtype: list + """ + if '--sparse' in rsync_options: + return [] + if '--preallocate' in rsync_options: + log.warning( + 'Sparse data sync not used: incompatible with --preallocate' + ) + return [] + if '--inplace' in rsync_options and \ + not self._rsync_supports_sparse_inplace(): + log.warning( + 'Sparse data sync not used: rsync too old to combine ' + '--sparse with --inplace, 3.1.3 or later is required' + ) + return [] + return ['--sparse'] + + @staticmethod + def _rsync_supports_sparse_inplace() -> bool: + """ + Check if the host rsync allows --sparse together with --inplace + + :return: True or False + + :rtype: bool + """ + try: + version_text = Command.run(['rsync', '--version']).output + version = re.search(r'version (\d+)\.(\d+)\.(\d+)', version_text) + if not version: + return False + return tuple( + int(part) for part in version.groups() + ) >= (3, 1, 3) + except Exception as issue: + log.warning( + 'Could not detect rsync version: {0}'.format(issue) + ) + return False + def target_supports_extended_attributes(self) -> bool: """ Check if the target directory supports extended filesystem diff --git a/test/data/kiwi_config/ok/.config/kiwi/config.yml b/test/data/kiwi_config/ok/.config/kiwi/config.yml index 1e152424905..d60b83abc0f 100644 --- a/test/data/kiwi_config/ok/.config/kiwi/config.yml +++ b/test/data/kiwi_config/ok/.config/kiwi/config.yml @@ -1,6 +1,9 @@ xz: options: -a -b xxx +sync: + sparse: true + bundle: compress: true diff --git a/test/data/kiwi_config/other/.config/kiwi/config.yml b/test/data/kiwi_config/other/.config/kiwi/config.yml index 7ce6751bc9c..46fd3580dd1 100644 --- a/test/data/kiwi_config/other/.config/kiwi/config.yml +++ b/test/data/kiwi_config/other/.config/kiwi/config.yml @@ -1,6 +1,9 @@ bundle: has_package_changes: true +sync: + sparse: false + container: compress: true diff --git a/test/unit/defaults_test.py b/test/unit/defaults_test.py index d15552614dc..c89e1599c72 100644 --- a/test/unit/defaults_test.py +++ b/test/unit/defaults_test.py @@ -180,6 +180,9 @@ def test_get_exclude_list_for_root_data_sync(self): '.buildenv', 'var/cache/kiwi' ] + def test_get_sync_sparse(self): + assert Defaults.get_sync_sparse() is False + @patch('glob.iglob') def test_get_signed_grub_loader(self, mock_iglob): def iglob_no_matches(pattern): diff --git a/test/unit/filesystem/base_test.py b/test/unit/filesystem/base_test.py index 88a665ca6e1..6142ea6b9b2 100644 --- a/test/unit/filesystem/base_test.py +++ b/test/unit/filesystem/base_test.py @@ -90,6 +90,56 @@ def test_sync_data( filesystem_mount.mount.assert_called_once_with([]) assert self.fsbase.get_mountpoint() == 'tmpdir' + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_no_sparse_options( + self, mock_RuntimeConfig + ): + # a filesystem that declares nothing keeps the options as given + # and does not even consult the runtime configuration + self.fsbase.custom_args['mount_options'] = ['noatime'] + assert self.fsbase._get_install_mount_options() == ['noatime'] + assert not mock_RuntimeConfig.called + + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_sparse_not_requested( + self, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False + self.fsbase.sparse_install_mount_options = ['allocsize=4k'] + self.fsbase.custom_args['mount_options'] = ['noatime'] + assert self.fsbase._get_install_mount_options() == ['noatime'] + + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_sparse_requested( + self, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + self.fsbase.sparse_install_mount_options = ['allocsize=4k'] + + # appended to the given options as one comma separated element + self.fsbase.custom_args['mount_options'] = ['noatime'] + with self._caplog.at_level(logging.INFO): + assert self.fsbase._get_install_mount_options() == [ + 'noatime,allocsize=4k' + ] + assert 'allocsize=4k' in self._caplog.text + + # used on its own if the image description provides no options + self.fsbase.custom_args['mount_options'] = [] + assert self.fsbase._get_install_mount_options() == ['allocsize=4k'] + + # the mount options from the image description take precedence + self.fsbase.custom_args['mount_options'] = ['noatime,allocsize=1m'] + assert self.fsbase._get_install_mount_options() == [ + 'noatime,allocsize=1m' + ] + + # and the result stays independent of the custom args, which + # are the source for the fstab of the image + assert self.fsbase.custom_args['mount_options'] == [ + 'noatime,allocsize=1m' + ] + @patch('kiwi.filesystem.base.VeritySetup') def test_create_verity_layer(self, mock_VeritySetup): self.fsbase.create_verity_layer() diff --git a/test/unit/runtime_config_test.py b/test/unit/runtime_config_test.py index 197e2646b2a..0bf8cab3c4e 100644 --- a/test/unit/runtime_config_test.py +++ b/test/unit/runtime_config_test.py @@ -215,6 +215,7 @@ def test_config_sections_from_home_base_config( assert runtime_config.get_oci_archive_tool() == 'umoci' assert runtime_config.get_mapper_tool() == 'partx' assert runtime_config.get_package_changes() is True + assert runtime_config.get_sync_sparse() is True assert runtime_config.get_disabled_runtime_checks() == [ 'check_dracut_module_for_oem_install_in_package_list', 'check_container_tool_chain_installed' @@ -240,6 +241,7 @@ def test_config_sections_defaults( assert runtime_config.get_container_compression() is True assert runtime_config.get_iso_tool_category() == 'xorriso' assert runtime_config.get_iso_media_tag_tool() == 'checkmedia' + assert runtime_config.get_sync_sparse() == Defaults.get_sync_sparse() assert runtime_config.get_oci_archive_tool() == 'umoci' assert runtime_config.get_mapper_tool() == 'kpartx' assert runtime_config.get_package_changes() is False @@ -278,4 +280,5 @@ def test_config_sections_other_settings(self): assert runtime_config.get_container_compression() is True assert runtime_config.get_package_changes() is True + assert runtime_config.get_sync_sparse() is False assert runtime_config.get_iso_media_tag_tool() == "checkmedia" diff --git a/test/unit/utils/sync_test.py b/test/unit/utils/sync_test.py index e8fbf05bd27..f0e7d13f2e6 100644 --- a/test/unit/utils/sync_test.py +++ b/test/unit/utils/sync_test.py @@ -3,7 +3,9 @@ import logging from pytest import fixture from stat import ST_MODE -from unittest.mock import patch +from unittest.mock import ( + patch, Mock +) from kiwi.utils.sync import DataSync @@ -19,13 +21,16 @@ def setup(self): def setup_method(self, cls): self.setup() + @patch('kiwi.utils.sync.RuntimeConfig') @patch('kiwi.utils.sync.Command.run') @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') @patch('os.chmod') @patch('os.stat') def test_sync_data( - self, mock_stat, mock_chmod, mock_xattr_support, mock_command + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False mock_stat.return_value = os.stat('.') mock_xattr_support.return_value = False with self._caplog.at_level(logging.WARNING): @@ -46,12 +51,115 @@ def test_sync_data( 'target_dir', mock_stat.return_value[ST_MODE] ) + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.DataSync._rsync_supports_sparse_inplace') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_sparse_inplace, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_sparse_inplace.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + self.sync.sync_data(options=['--archive', '--inplace']) + mock_command.assert_called_once_with( + [ + 'rsync', '--archive', '--inplace', '--sparse', + 'source_dir', 'target_dir' + ] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_already_requested( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + self.sync.sync_data(options=['--archive', '--sparse']) + mock_command.assert_called_once_with( + ['rsync', '--archive', '--sparse', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_skipped_for_preallocate( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + with self._caplog.at_level(logging.WARNING): + self.sync.sync_data(options=['--archive', '--preallocate']) + assert 'incompatible with --preallocate' in self._caplog.text + mock_command.assert_called_once_with( + ['rsync', '--archive', '--preallocate', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.DataSync._rsync_supports_sparse_inplace') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_skipped_for_old_rsync( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_sparse_inplace, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_sparse_inplace.return_value = False + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + with self._caplog.at_level(logging.WARNING): + self.sync.sync_data(options=['--archive', '--inplace']) + assert 'rsync too old' in self._caplog.text + mock_command.assert_called_once_with( + ['rsync', '--archive', '--inplace', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.Command.run') + def test_rsync_supports_sparse_inplace(self, mock_command): + mock_command.return_value = Mock( + output='rsync version 3.4.0 protocol version 32' + ) + assert DataSync._rsync_supports_sparse_inplace() is True + mock_command.return_value = Mock( + output='rsync version 3.1.2 protocol version 31' + ) + assert DataSync._rsync_supports_sparse_inplace() is False + mock_command.return_value = Mock(output='no version information') + assert DataSync._rsync_supports_sparse_inplace() is False + + @patch('kiwi.utils.sync.Command.run') + def test_rsync_supports_sparse_inplace_detection_failed( + self, mock_command + ): + mock_command.side_effect = Exception('rsync not found') + with self._caplog.at_level(logging.WARNING): + assert DataSync._rsync_supports_sparse_inplace() is False + assert 'Could not detect rsync version' in self._caplog.text + + @patch('kiwi.utils.sync.RuntimeConfig') @patch('kiwi.utils.sync.Command.run') @patch('os.chmod') @patch('os.stat') def test_sync_data_force_trailing_slash( - self, mock_stat, mock_chmod, mock_command + self, mock_stat, mock_chmod, mock_command, mock_RuntimeConfig ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False mock_stat.return_value = os.stat('.') self.sync.sync_data(force_trailing_slash=True) mock_command.assert_called_once_with(