diff --git a/kiwi.yml b/kiwi.yml index 0ef035b03ba..6784090adf9 100644 --- a/kiwi.yml +++ b/kiwi.yml @@ -64,6 +64,30 @@ # options: '--threads=0' +# Setup behaviour of the data sync used to install the image payload +#sync: +# # Specify if sequences of null bytes in the source data should be +# # turned into holes on the target. This adds the --sparse option to +# # the rsync call. It costs sync time because the data has to be +# # scanned for null sequences, and the gain depends on the payload. +# # It is effective for binaries built for a 64k page size, which +# # carry a zero tail per segment. +# # Default: false +# # +# # Note: the option is skipped with a warning if it cannot be used, +# # which is the case for rsync releases before 3.1.3 because they +# # reject --sparse together with --inplace. +# # +# # Note: on filesystems that perform speculative preallocation the +# # option alone does not reclaim space, because the skipped range +# # becomes an allocated but unwritten extent rather than a hole. For +# # those filesystems KIWI adds the mount options needed to prevent +# # this to the install time mount only, so they do not appear in the +# # fstab of the image. An explicit setting of the same option in the +# # fsmountoptions attribute of the image description takes precedence +# sparse: true + + # Setup process parameters for container image creation #container: # # Specify compression for container images diff --git a/kiwi/defaults.py b/kiwi/defaults.py index 3e9ff5d9415..18ca06d8ac5 100644 --- a/kiwi/defaults.py +++ b/kiwi/defaults.py @@ -412,6 +412,21 @@ def get_sync_options(): '--one-file-system', '--inplace' ] + @staticmethod + def get_sync_sparse(): + """ + Provides default setting for sparse data sync + + If enabled, rsync is called with the --sparse option which + turns sequences of null bytes in the source data into holes + on the target. + + :return: True or False + + :rtype: bool + """ + return False + @staticmethod def get_removed_files_name(): """ diff --git a/kiwi/filesystem/base.py b/kiwi/filesystem/base.py index ea37021c9ec..4dd0fad5f82 100644 --- a/kiwi/filesystem/base.py +++ b/kiwi/filesystem/base.py @@ -29,6 +29,7 @@ import kiwi.defaults as defaults from kiwi.defaults import Defaults +from kiwi.runtime_config import RuntimeConfig from kiwi.utils.sync import DataSync from kiwi.mount_manager import MountManager from kiwi.command import Command @@ -53,6 +54,14 @@ class FileSystemBase: :param string root_dir: root directory path name :param dict custom_args: custom filesystem arguments """ + # Mount options applied only to the install time mount of a newly created + # filesystem. They are deliberately not part of the mount options from the + # image description and therefore never end up in the fstab of the + # image. A filesystem implementation declares options here when sparse + # sync cannot work without them, see the xfs implementation for an + # example. + sparse_install_mount_options: List[str] = [] + def __init__( self, device_provider: DeviceProvider, root_dir: str = '', custom_args: Dict = {} @@ -196,7 +205,7 @@ def sync_data(self, exclude: List[str] = []) -> MountManager: device=self.device_provider.get_device() ) self.filesystem_mount.mount( - self.custom_args['mount_options'] + self._get_install_mount_options() ) self._apply_attributes() data = DataSync( @@ -207,6 +216,42 @@ def sync_data(self, exclude: List[str] = []) -> MountManager: ) return self.filesystem_mount + def _get_install_mount_options(self) -> List[str]: + """ + Provide mount options for the install time mount of the + filesystem the image payload is synced into. + + These are the mount options from the image description, plus + any options the filesystem implementation declares as needed + for a sparse data sync. The latter apply to this mount only + and are not passed on to the fstab of the image. + + As everywhere else in kiwi the result is a list of at most one + element, holding the comma separated option string that is + passed along to the -o mount option. + + :return: max one element list with mount option string + + :rtype: list + """ + given_options = self.custom_args['mount_options'] + if not self.sparse_install_mount_options or \ + not RuntimeConfig().get_sync_sparse(): + return list(given_options) + options = ','.join(given_options).split(',') if given_options else [] + given_names = [option.split('=')[0] for option in options if option] + for option in self.sparse_install_mount_options: + if option.split('=')[0] in given_names: + # an explicit setting from the image description wins + continue + log.info( + 'Using install time mount option {0} for sparse sync'.format( + option + ) + ) + options.append(option) + return [','.join(options)] if options else [] + def create_verity_layer( self, blocks: Optional[int] = None, filename: str = None ): diff --git a/kiwi/filesystem/xfs.py b/kiwi/filesystem/xfs.py index ff253cfd13f..36e8acbbd25 100644 --- a/kiwi/filesystem/xfs.py +++ b/kiwi/filesystem/xfs.py @@ -26,6 +26,15 @@ class FileSystemXfs(FileSystemBase): """ **Implements creation of xfs filesystem** """ + # Minimize speculative preallocation when a sparse data sync is requested. + # + # rsync --sparse only seeks forward across a sequence of null bytes, it + # does not punch a hole. On xfs most small skips lead to speculative + # preallocation (optimized for a fast-changing runtime) instead of the + # desired holes (as desired during one-time image construction). + # rsync therefore fails to save most space without this setting. + sparse_install_mount_options = ['allocsize=4k'] + def create_on_device( self, label: str = None, size: int = 0, unit: str = defaults.UNIT.kb, uuid: str = None diff --git a/kiwi/runtime_config.py b/kiwi/runtime_config.py index 36476e6aaae..6a5e069c1cb 100644 --- a/kiwi/runtime_config.py +++ b/kiwi/runtime_config.py @@ -332,6 +332,38 @@ def get_xz_options(self) -> Optional[List[str]]: xz_options = self._get_attribute(element='xz', attribute='options') return xz_options.split() if xz_options else None + def get_sync_sparse(self) -> bool: + """ + Return if data sync operations should turn sequences of null bytes + into holes on the target. + + sync: + sparse: true|false + + If no configuration exists the default from the Defaults + class is returned. + + Enabling this adds the --sparse option to the rsync call used + to install the image payload. On filesystems that perform + speculative preallocation the option alone does not reclaim + the space, because the skipped range is recorded as an + allocated but unwritten extent rather than as a hole. Such + filesystems can declare additional install time mount options + via the sparse_install_mount_options attribute of their + FileSystemBase implementation. + + :return: True or False + + :rtype: bool + + """ + sync_sparse = self._get_attribute( + element='sync', attribute='sparse' + ) + if sync_sparse is None: + return Defaults.get_sync_sparse() + return bool(sync_sparse) + def get_container_compression(self) -> bool: """ Return compression for container images diff --git a/kiwi/utils/sync.py b/kiwi/utils/sync.py index b338ca8ba6e..8313f175300 100644 --- a/kiwi/utils/sync.py +++ b/kiwi/utils/sync.py @@ -16,6 +16,7 @@ # along with kiwi. If not, see # import os +import re import errno import logging from stat import ST_MODE @@ -23,6 +24,7 @@ # project from kiwi.command import Command +from kiwi.runtime_config import RuntimeConfig log = logging.getLogger('kiwi') @@ -87,6 +89,9 @@ def sync_data( rsync_options = [] if options: rsync_options = options + if RuntimeConfig().get_sync_sparse(): + rsync_options = \ + rsync_options + self._get_sparse_options(rsync_options) if not self.target_supports_extended_attributes(): warn_me = False if '--xattrs' in rsync_options: @@ -125,6 +130,68 @@ def sync_data( # not changed os.chmod(self.target_dir, target_entry_permissions) + def _get_sparse_options(self, rsync_options: List[str]) -> List[str]: + """ + Provide the --sparse option if it is both wanted and usable + + The option turns sequences of null bytes into holes on the + target. It is skipped if the caller already asked for it, and + it is skipped with a warning if it cannot be combined with the + options the caller provided: + + * rsync releases before 3.1.3 reject --sparse together with + --inplace. kiwi syncs with --inplace, so on such a host the + combination would abort the build. + * --preallocate defeats the purpose, because every file is + allocated in full before it is written. The result occupies + the space of a dense file while a hole detecting reader still + sees holes, which is worse than not asking for sparse at all. + + :param list rsync_options: options provided by the caller + + :return: list of options to add + + :rtype: list + """ + if '--sparse' in rsync_options: + return [] + if '--preallocate' in rsync_options: + log.warning( + 'Sparse data sync not used: incompatible with --preallocate' + ) + return [] + if '--inplace' in rsync_options and \ + not self._rsync_supports_sparse_inplace(): + log.warning( + 'Sparse data sync not used: rsync too old to combine ' + '--sparse with --inplace, 3.1.3 or later is required' + ) + return [] + return ['--sparse'] + + @staticmethod + def _rsync_supports_sparse_inplace() -> bool: + """ + Check if the host rsync allows --sparse together with --inplace + + :return: True or False + + :rtype: bool + """ + try: + version_text = Command.run(['rsync', '--version']).output + version = re.search(r'version (\d+)\.(\d+)\.(\d+)', version_text) + if not version: + return False + return tuple( + int(part) for part in version.groups() + ) >= (3, 1, 3) + except Exception as issue: + log.warning( + 'Could not detect rsync version: {0}'.format(issue) + ) + return False + def target_supports_extended_attributes(self) -> bool: """ Check if the target directory supports extended filesystem diff --git a/test/data/kiwi_config/ok/.config/kiwi/config.yml b/test/data/kiwi_config/ok/.config/kiwi/config.yml index 1e152424905..d60b83abc0f 100644 --- a/test/data/kiwi_config/ok/.config/kiwi/config.yml +++ b/test/data/kiwi_config/ok/.config/kiwi/config.yml @@ -1,6 +1,9 @@ xz: options: -a -b xxx +sync: + sparse: true + bundle: compress: true diff --git a/test/data/kiwi_config/other/.config/kiwi/config.yml b/test/data/kiwi_config/other/.config/kiwi/config.yml index 7ce6751bc9c..46fd3580dd1 100644 --- a/test/data/kiwi_config/other/.config/kiwi/config.yml +++ b/test/data/kiwi_config/other/.config/kiwi/config.yml @@ -1,6 +1,9 @@ bundle: has_package_changes: true +sync: + sparse: false + container: compress: true diff --git a/test/unit/defaults_test.py b/test/unit/defaults_test.py index d15552614dc..c89e1599c72 100644 --- a/test/unit/defaults_test.py +++ b/test/unit/defaults_test.py @@ -180,6 +180,9 @@ def test_get_exclude_list_for_root_data_sync(self): '.buildenv', 'var/cache/kiwi' ] + def test_get_sync_sparse(self): + assert Defaults.get_sync_sparse() is False + @patch('glob.iglob') def test_get_signed_grub_loader(self, mock_iglob): def iglob_no_matches(pattern): diff --git a/test/unit/filesystem/base_test.py b/test/unit/filesystem/base_test.py index 88a665ca6e1..6142ea6b9b2 100644 --- a/test/unit/filesystem/base_test.py +++ b/test/unit/filesystem/base_test.py @@ -90,6 +90,56 @@ def test_sync_data( filesystem_mount.mount.assert_called_once_with([]) assert self.fsbase.get_mountpoint() == 'tmpdir' + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_no_sparse_options( + self, mock_RuntimeConfig + ): + # a filesystem that declares nothing keeps the options as given + # and does not even consult the runtime configuration + self.fsbase.custom_args['mount_options'] = ['noatime'] + assert self.fsbase._get_install_mount_options() == ['noatime'] + assert not mock_RuntimeConfig.called + + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_sparse_not_requested( + self, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False + self.fsbase.sparse_install_mount_options = ['allocsize=4k'] + self.fsbase.custom_args['mount_options'] = ['noatime'] + assert self.fsbase._get_install_mount_options() == ['noatime'] + + @patch('kiwi.filesystem.base.RuntimeConfig') + def test_get_install_mount_options_sparse_requested( + self, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + self.fsbase.sparse_install_mount_options = ['allocsize=4k'] + + # appended to the given options as one comma separated element + self.fsbase.custom_args['mount_options'] = ['noatime'] + with self._caplog.at_level(logging.INFO): + assert self.fsbase._get_install_mount_options() == [ + 'noatime,allocsize=4k' + ] + assert 'allocsize=4k' in self._caplog.text + + # used on its own if the image description provides no options + self.fsbase.custom_args['mount_options'] = [] + assert self.fsbase._get_install_mount_options() == ['allocsize=4k'] + + # the mount options from the image description take precedence + self.fsbase.custom_args['mount_options'] = ['noatime,allocsize=1m'] + assert self.fsbase._get_install_mount_options() == [ + 'noatime,allocsize=1m' + ] + + # and the result stays independent of the custom args, which + # are the source for the fstab of the image + assert self.fsbase.custom_args['mount_options'] == [ + 'noatime,allocsize=1m' + ] + @patch('kiwi.filesystem.base.VeritySetup') def test_create_verity_layer(self, mock_VeritySetup): self.fsbase.create_verity_layer() diff --git a/test/unit/runtime_config_test.py b/test/unit/runtime_config_test.py index 197e2646b2a..0bf8cab3c4e 100644 --- a/test/unit/runtime_config_test.py +++ b/test/unit/runtime_config_test.py @@ -215,6 +215,7 @@ def test_config_sections_from_home_base_config( assert runtime_config.get_oci_archive_tool() == 'umoci' assert runtime_config.get_mapper_tool() == 'partx' assert runtime_config.get_package_changes() is True + assert runtime_config.get_sync_sparse() is True assert runtime_config.get_disabled_runtime_checks() == [ 'check_dracut_module_for_oem_install_in_package_list', 'check_container_tool_chain_installed' @@ -240,6 +241,7 @@ def test_config_sections_defaults( assert runtime_config.get_container_compression() is True assert runtime_config.get_iso_tool_category() == 'xorriso' assert runtime_config.get_iso_media_tag_tool() == 'checkmedia' + assert runtime_config.get_sync_sparse() == Defaults.get_sync_sparse() assert runtime_config.get_oci_archive_tool() == 'umoci' assert runtime_config.get_mapper_tool() == 'kpartx' assert runtime_config.get_package_changes() is False @@ -278,4 +280,5 @@ def test_config_sections_other_settings(self): assert runtime_config.get_container_compression() is True assert runtime_config.get_package_changes() is True + assert runtime_config.get_sync_sparse() is False assert runtime_config.get_iso_media_tag_tool() == "checkmedia" diff --git a/test/unit/utils/sync_test.py b/test/unit/utils/sync_test.py index e8fbf05bd27..f0e7d13f2e6 100644 --- a/test/unit/utils/sync_test.py +++ b/test/unit/utils/sync_test.py @@ -3,7 +3,9 @@ import logging from pytest import fixture from stat import ST_MODE -from unittest.mock import patch +from unittest.mock import ( + patch, Mock +) from kiwi.utils.sync import DataSync @@ -19,13 +21,16 @@ def setup(self): def setup_method(self, cls): self.setup() + @patch('kiwi.utils.sync.RuntimeConfig') @patch('kiwi.utils.sync.Command.run') @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') @patch('os.chmod') @patch('os.stat') def test_sync_data( - self, mock_stat, mock_chmod, mock_xattr_support, mock_command + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False mock_stat.return_value = os.stat('.') mock_xattr_support.return_value = False with self._caplog.at_level(logging.WARNING): @@ -46,12 +51,115 @@ def test_sync_data( 'target_dir', mock_stat.return_value[ST_MODE] ) + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.DataSync._rsync_supports_sparse_inplace') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_sparse_inplace, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_sparse_inplace.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + self.sync.sync_data(options=['--archive', '--inplace']) + mock_command.assert_called_once_with( + [ + 'rsync', '--archive', '--inplace', '--sparse', + 'source_dir', 'target_dir' + ] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_already_requested( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + self.sync.sync_data(options=['--archive', '--sparse']) + mock_command.assert_called_once_with( + ['rsync', '--archive', '--sparse', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_skipped_for_preallocate( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + with self._caplog.at_level(logging.WARNING): + self.sync.sync_data(options=['--archive', '--preallocate']) + assert 'incompatible with --preallocate' in self._caplog.text + mock_command.assert_called_once_with( + ['rsync', '--archive', '--preallocate', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.RuntimeConfig') + @patch('kiwi.utils.sync.DataSync._rsync_supports_sparse_inplace') + @patch('kiwi.utils.sync.Command.run') + @patch('kiwi.utils.sync.DataSync.target_supports_extended_attributes') + @patch('os.chmod') + @patch('os.stat') + def test_sync_data_sparse_skipped_for_old_rsync( + self, mock_stat, mock_chmod, mock_xattr_support, mock_command, + mock_sparse_inplace, mock_RuntimeConfig + ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True + mock_sparse_inplace.return_value = False + mock_xattr_support.return_value = True + mock_stat.return_value = os.stat('.') + with self._caplog.at_level(logging.WARNING): + self.sync.sync_data(options=['--archive', '--inplace']) + assert 'rsync too old' in self._caplog.text + mock_command.assert_called_once_with( + ['rsync', '--archive', '--inplace', 'source_dir', 'target_dir'] + ) + + @patch('kiwi.utils.sync.Command.run') + def test_rsync_supports_sparse_inplace(self, mock_command): + mock_command.return_value = Mock( + output='rsync version 3.4.0 protocol version 32' + ) + assert DataSync._rsync_supports_sparse_inplace() is True + mock_command.return_value = Mock( + output='rsync version 3.1.2 protocol version 31' + ) + assert DataSync._rsync_supports_sparse_inplace() is False + mock_command.return_value = Mock(output='no version information') + assert DataSync._rsync_supports_sparse_inplace() is False + + @patch('kiwi.utils.sync.Command.run') + def test_rsync_supports_sparse_inplace_detection_failed( + self, mock_command + ): + mock_command.side_effect = Exception('rsync not found') + with self._caplog.at_level(logging.WARNING): + assert DataSync._rsync_supports_sparse_inplace() is False + assert 'Could not detect rsync version' in self._caplog.text + + @patch('kiwi.utils.sync.RuntimeConfig') @patch('kiwi.utils.sync.Command.run') @patch('os.chmod') @patch('os.stat') def test_sync_data_force_trailing_slash( - self, mock_stat, mock_chmod, mock_command + self, mock_stat, mock_chmod, mock_command, mock_RuntimeConfig ): + mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False mock_stat.return_value = os.stat('.') self.sync.sync_data(force_trailing_slash=True) mock_command.assert_called_once_with(