Add optional sparse data sync to minimize image size

kiwi.yml

@@ -64,6 +64,30 @@
 #  options: '--threads=0'
 
 
+# Setup behaviour of the data sync used to install the image payload
+#sync:
+#  # Specify if sequences of null bytes in the source data should be
+#  # turned into holes on the target. This adds the --sparse option to
+#  # the rsync call. It costs sync time because the data has to be
+#  # scanned for null sequences, and the gain depends on the payload.
+#  # It is effective for binaries built for a 64k page size, which
+#  # carry a zero tail per segment.
+#  # Default: false
+#  #
+#  # Note: the option is skipped with a warning if it cannot be used,
+#  # which is the case for rsync releases before 3.1.3 because they
+#  # reject --sparse together with --inplace.
+#  #
+#  # Note: on filesystems that perform speculative preallocation the
+#  # option alone does not reclaim space, because the skipped range
+#  # becomes an allocated but unwritten extent rather than a hole. For
+#  # those filesystems KIWI adds the mount options needed to prevent
+#  # this to the install time mount only, so they do not appear in the
+#  # fstab of the image. An explicit setting of the same option in the
+#  # fsmountoptions attribute of the image description takes precedence
+#  sparse: true
+
+
 # Setup process parameters for container image creation
 #container:
 #  # Specify compression for container images

kiwi/defaults.py

@@ -412,6 +412,21 @@ def get_sync_options():
             '--one-file-system', '--inplace'
         ]
 
+    @staticmethod
+    def get_sync_sparse():
+        """
+        Provides default setting for sparse data sync
+
+        If enabled, rsync is called with the --sparse option which
+        turns sequences of null bytes in the source data into holes
+        on the target.
+
+        :return: True or False
+
+        :rtype: bool
+        """
+        return False
+
     @staticmethod
     def get_removed_files_name():
         """

kiwi/filesystem/base.py

@@ -29,6 +29,7 @@
 import kiwi.defaults as defaults
 
 from kiwi.defaults import Defaults
+from kiwi.runtime_config import RuntimeConfig
 from kiwi.utils.sync import DataSync
 from kiwi.mount_manager import MountManager
 from kiwi.command import Command
@@ -53,6 +54,14 @@ class FileSystemBase:
     :param string root_dir: root directory path name
     :param dict custom_args: custom filesystem arguments
     """
+    # Mount options applied only to the install time mount of a newly created
+    # filesystem. They are deliberately not part of the mount options from the
+    # image description and therefore never end up in the fstab of the
+    # image. A filesystem implementation declares options here when sparse
+    # sync cannot work without them, see the xfs implementation for an
+    # example.
+    sparse_install_mount_options: List[str] = []
+
     def __init__(
         self, device_provider: DeviceProvider,
         root_dir: str = '', custom_args: Dict = {}
@@ -196,7 +205,7 @@ def sync_data(self, exclude: List[str] = []) -> MountManager:
             device=self.device_provider.get_device()
         )
         self.filesystem_mount.mount(
-            self.custom_args['mount_options']
+            self._get_install_mount_options()
         )
         self._apply_attributes()
         data = DataSync(
@@ -207,6 +216,42 @@ def sync_data(self, exclude: List[str] = []) -> MountManager:
         )
         return self.filesystem_mount
 
+    def _get_install_mount_options(self) -> List[str]:
+        """
+        Provide mount options for the install time mount of the
+        filesystem the image payload is synced into.
+
+        These are the mount options from the image description, plus
+        any options the filesystem implementation declares as needed
+        for a sparse data sync. The latter apply to this mount only
+        and are not passed on to the fstab of the image.
+
+        As everywhere else in kiwi the result is a list of at most one
+        element, holding the comma separated option string that is
+        passed along to the -o mount option.
+
+        :return: max one element list with mount option string
+
+        :rtype: list
+        """
+        given_options = self.custom_args['mount_options']
+        if not self.sparse_install_mount_options or \
+           not RuntimeConfig().get_sync_sparse():
+            return list(given_options)
+        options = ','.join(given_options).split(',') if given_options else []
+        given_names = [option.split('=')[0] for option in options if option]
+        for option in self.sparse_install_mount_options:
+            if option.split('=')[0] in given_names:
+                # an explicit setting from the image description wins
+                continue
+            log.info(
+                'Using install time mount option {0} for sparse sync'.format(
+                    option
+                )
+            )
+            options.append(option)
+        return [','.join(options)] if options else []
+
     def create_verity_layer(
         self, blocks: Optional[int] = None, filename: str = None
     ):

kiwi/filesystem/xfs.py

@@ -26,6 +26,15 @@ class FileSystemXfs(FileSystemBase):
     """
     **Implements creation of xfs filesystem**
     """
+    # Minimize speculative preallocation when a sparse data sync is requested.
+    #
+    # rsync --sparse only seeks forward across a sequence of null bytes, it
+    # does not punch a hole. On xfs most small skips lead to speculative
+    # preallocation (optimized for a fast-changing runtime) instead of the
+    # desired holes (as desired during one-time image construction).
+    # rsync therefore fails to save most space without this setting.
+    sparse_install_mount_options = ['allocsize=4k']
+
     def create_on_device(
         self, label: str = None, size: int = 0,
         unit: str = defaults.UNIT.kb, uuid: str = None

kiwi/runtime_config.py

@@ -332,6 +332,38 @@ def get_xz_options(self) -> Optional[List[str]]:
         xz_options = self._get_attribute(element='xz', attribute='options')
         return xz_options.split() if xz_options else None
 
+    def get_sync_sparse(self) -> bool:
+        """
+        Return if data sync operations should turn sequences of null bytes
+        into holes on the target.
+
+        sync:
+          sparse: true|false
+
+        If no configuration exists the default from the Defaults
+        class is returned.
+
+        Enabling this adds the --sparse option to the rsync call used
+        to install the image payload. On filesystems that perform
+        speculative preallocation the option alone does not reclaim
+        the space, because the skipped range is recorded as an
+        allocated but unwritten extent rather than as a hole. Such
+        filesystems can declare additional install time mount options
+        via the sparse_install_mount_options attribute of their
+        FileSystemBase implementation.
+
+        :return: True or False
+
+        :rtype: bool
+
+        """
+        sync_sparse = self._get_attribute(
+            element='sync', attribute='sparse'
+        )
+        if sync_sparse is None:
+            return Defaults.get_sync_sparse()
+        return bool(sync_sparse)
+
     def get_container_compression(self) -> bool:
         """
         Return compression for container images

kiwi/utils/sync.py

@@ -16,13 +16,15 @@
 # along with kiwi.  If not, see <http://www.gnu.org/licenses/>
 #
 import os
+import re
 import errno
 import logging
 from stat import ST_MODE
 from typing import List
 
 # project
 from kiwi.command import Command
+from kiwi.runtime_config import RuntimeConfig
 
 log = logging.getLogger('kiwi')
 
@@ -87,6 +89,9 @@ def sync_data(
         rsync_options = []
         if options:
             rsync_options = options
+        if RuntimeConfig().get_sync_sparse():
+            rsync_options = \
+                rsync_options + self._get_sparse_options(rsync_options)
         if not self.target_supports_extended_attributes():
             warn_me = False
             if '--xattrs' in rsync_options:
@@ -125,6 +130,68 @@ def sync_data(
             # not changed
             os.chmod(self.target_dir, target_entry_permissions)
 
+    def _get_sparse_options(self, rsync_options: List[str]) -> List[str]:
+        """
+        Provide the --sparse option if it is both wanted and usable
+
+        The option turns sequences of null bytes into holes on the
+        target. It is skipped if the caller already asked for it, and
+        it is skipped with a warning if it cannot be combined with the
+        options the caller provided:
+
+        * rsync releases before 3.1.3 reject --sparse together with
+          --inplace. kiwi syncs with --inplace, so on such a host the
+          combination would abort the build.
+        * --preallocate defeats the purpose, because every file is
+          allocated in full before it is written. The result occupies
+          the space of a dense file while a hole detecting reader still
+          sees holes, which is worse than not asking for sparse at all.
+
+        :param list rsync_options: options provided by the caller
+
+        :return: list of options to add
+
+        :rtype: list
+        """
+        if '--sparse' in rsync_options:
+            return []
+        if '--preallocate' in rsync_options:
+            log.warning(
+                'Sparse data sync not used: incompatible with --preallocate'
+            )
+            return []
+        if '--inplace' in rsync_options and \
+           not self._rsync_supports_sparse_inplace():
+            log.warning(
+                'Sparse data sync not used: rsync too old to combine '
+                '--sparse with --inplace, 3.1.3 or later is required'
+            )
+            return []
+        return ['--sparse']
+
+    @staticmethod
+    def _rsync_supports_sparse_inplace() -> bool:
+        """
+        Check if the host rsync allows --sparse together with --inplace
+
+        :return: True or False
+
+        :rtype: bool
+        """
+        try:
+            version_text = Command.run(['rsync', '--version']).output
+            version = re.search(r'version (\d+)\.(\d+)\.(\d+)', version_text)
+            if not version:
+                return False
+            return tuple(
+                int(part) for part in version.groups()
+            ) >= (3, 1, 3)
+        except Exception as issue:
+            log.warning(
+                'Could not detect rsync version: {0}'.format(issue)
+            )
+            return False
+
     def target_supports_extended_attributes(self) -> bool:
         """
         Check if the target directory supports extended filesystem

test/data/kiwi_config/ok/.config/kiwi/config.yml

@@ -1,6 +1,9 @@
 xz:
   options: -a -b xxx
 
+sync:
+  sparse: true
+
 bundle:
   compress: true
 

test/data/kiwi_config/other/.config/kiwi/config.yml

@@ -1,6 +1,9 @@
 bundle:
   has_package_changes: true
 
+sync:
+  sparse: false
+
 container:
   compress: true
 

test/unit/defaults_test.py

@@ -180,6 +180,9 @@ def test_get_exclude_list_for_root_data_sync(self):
             '.buildenv', 'var/cache/kiwi'
         ]
 
+    def test_get_sync_sparse(self):
+        assert Defaults.get_sync_sparse() is False
+
     @patch('glob.iglob')
     def test_get_signed_grub_loader(self, mock_iglob):
         def iglob_no_matches(pattern):

test/unit/filesystem/base_test.py

@@ -90,6 +90,56 @@ def test_sync_data(
         filesystem_mount.mount.assert_called_once_with([])
         assert self.fsbase.get_mountpoint() == 'tmpdir'
 
+    @patch('kiwi.filesystem.base.RuntimeConfig')
+    def test_get_install_mount_options_no_sparse_options(
+        self, mock_RuntimeConfig
+    ):
+        # a filesystem that declares nothing keeps the options as given
+        # and does not even consult the runtime configuration
+        self.fsbase.custom_args['mount_options'] = ['noatime']
+        assert self.fsbase._get_install_mount_options() == ['noatime']
+        assert not mock_RuntimeConfig.called
+
+    @patch('kiwi.filesystem.base.RuntimeConfig')
+    def test_get_install_mount_options_sparse_not_requested(
+        self, mock_RuntimeConfig
+    ):
+        mock_RuntimeConfig.return_value.get_sync_sparse.return_value = False
+        self.fsbase.sparse_install_mount_options = ['allocsize=4k']
+        self.fsbase.custom_args['mount_options'] = ['noatime']
+        assert self.fsbase._get_install_mount_options() == ['noatime']
+
+    @patch('kiwi.filesystem.base.RuntimeConfig')
+    def test_get_install_mount_options_sparse_requested(
+        self, mock_RuntimeConfig
+    ):
+        mock_RuntimeConfig.return_value.get_sync_sparse.return_value = True
+        self.fsbase.sparse_install_mount_options = ['allocsize=4k']
+
+        # appended to the given options as one comma separated element
+        self.fsbase.custom_args['mount_options'] = ['noatime']
+        with self._caplog.at_level(logging.INFO):
+            assert self.fsbase._get_install_mount_options() == [
+                'noatime,allocsize=4k'
+            ]
+            assert 'allocsize=4k' in self._caplog.text
+
+        # used on its own if the image description provides no options
+        self.fsbase.custom_args['mount_options'] = []
+        assert self.fsbase._get_install_mount_options() == ['allocsize=4k']
+
+        # the mount options from the image description take precedence
+        self.fsbase.custom_args['mount_options'] = ['noatime,allocsize=1m']
+        assert self.fsbase._get_install_mount_options() == [
+            'noatime,allocsize=1m'
+        ]
+
+        # and the result stays independent of the custom args, which
+        # are the source for the fstab of the image
+        assert self.fsbase.custom_args['mount_options'] == [
+            'noatime,allocsize=1m'
+        ]
+
     @patch('kiwi.filesystem.base.VeritySetup')
     def test_create_verity_layer(self, mock_VeritySetup):
         self.fsbase.create_verity_layer()

test/unit/runtime_config_test.py

@@ -215,6 +215,7 @@ def test_config_sections_from_home_base_config(
         assert runtime_config.get_oci_archive_tool() == 'umoci'
         assert runtime_config.get_mapper_tool() == 'partx'
         assert runtime_config.get_package_changes() is True
+        assert runtime_config.get_sync_sparse() is True
         assert runtime_config.get_disabled_runtime_checks() == [
             'check_dracut_module_for_oem_install_in_package_list',
             'check_container_tool_chain_installed'
@@ -240,6 +241,7 @@ def test_config_sections_defaults(
         assert runtime_config.get_container_compression() is True
         assert runtime_config.get_iso_tool_category() == 'xorriso'
         assert runtime_config.get_iso_media_tag_tool() == 'checkmedia'
+        assert runtime_config.get_sync_sparse() == Defaults.get_sync_sparse()
         assert runtime_config.get_oci_archive_tool() == 'umoci'
         assert runtime_config.get_mapper_tool() == 'kpartx'
         assert runtime_config.get_package_changes() is False
@@ -278,4 +280,5 @@ def test_config_sections_other_settings(self):
 
         assert runtime_config.get_container_compression() is True
         assert runtime_config.get_package_changes() is True
+        assert runtime_config.get_sync_sparse() is False
         assert runtime_config.get_iso_media_tag_tool() == "checkmedia"