feat: media deduplication with hardlink on creation

synapse/media/filepath.py

@@ -134,6 +134,16 @@ def _wrapped(
 }
 
 
+def _content_hash_rel(directory: str, sha256: str) -> str:
+    """Relative path of a content-addressed blob inside `directory`."""
+    return os.path.join(
+        directory,
+        _validate_path_component(sha256[0:2]),
+        _validate_path_component(sha256[2:4]),
+        _validate_path_component(sha256[4:]),
+    )
+
+
 def _validate_path_component(name: str) -> str:
     """Checks that the given string can be safely used as a path component
 
@@ -184,6 +194,17 @@ def local_media_filepath_rel(self, media_id: str) -> str:
 
     local_media_filepath = _wrap_in_base_path(local_media_filepath_rel)
 
+    @_wrap_with_jail_check(relative=True)
+    def local_media_blob_rel(self, sha256: str) -> str:
+        """Relative path of the local content-hash blob.
+
+        Downloads keep using `local_media_filepath`. This path is an extra
+        hardlink so identical local uploads share one inode.
+        """
+        return _content_hash_rel("local_blobs", sha256)
+
+    local_media_blob_filepath = _wrap_in_base_path(local_media_blob_rel)
+
     @_wrap_with_jail_check(relative=True)
     def local_media_thumbnail_rel(
         self, media_id: str, width: int, height: int, content_type: str, method: str
@@ -230,6 +251,18 @@ def remote_media_filepath_rel(self, server_name: str, file_id: str) -> str:
 
     remote_media_filepath = _wrap_in_base_path(remote_media_filepath_rel)
 
+    @_wrap_with_jail_check(relative=True)
+    def remote_media_blob_rel(self, sha256: str) -> str:
+        """Relative path of the remote content-hash blob.
+
+        Remote downloads keep using `remote_media_filepath`. This path is a
+        separate tree from `local_blobs`, so local and remote media never
+        share an inode.
+        """
+        return _content_hash_rel("remote_blobs", sha256)
+
+    remote_media_blob_filepath = _wrap_in_base_path(remote_media_blob_rel)
+
     @_wrap_with_jail_check(relative=True)
     def remote_media_thumbnail_rel(
         self,

synapse/media/media_repository.py

@@ -439,6 +439,8 @@ async def create_or_update_content(
                 quarantined_by="system" if should_quarantine else None,
             )
 
+        await self.media_storage.link_id_path_to_blob(fname, sha256, remote=False)
+
         try:
             await self._generate_thumbnails(None, media_id, media_id, media_type)
         except Exception as e:
@@ -880,6 +882,8 @@ async def _store_remote_media_with_cleanup(
             # Re-raise so the caller can handle it
             raise
 
+        await self.media_storage.link_id_path_to_blob(fname, sha256, remote=True)
+
     async def _download_remote_file(
         self,
         server_name: str,

synapse/media/media_storage.py

@@ -332,6 +332,33 @@ async def fetch_media(self, file_info: FileInfo) -> Responder | None:
 
         return None
 
+    def _blob_path(self, sha256: str, remote: bool) -> str | None:
+        try:
+            if remote:
+                return self.filepaths.remote_media_blob_filepath(sha256)
+            return self.filepaths.local_media_blob_filepath(sha256)
+        except ValueError:
+            logger.warning("Ignoring invalid media sha256 %r", sha256)
+            return None
+
+    async def link_id_path_to_blob(
+        self, id_path: str, sha256: str, remote: bool
+    ) -> None:
+        """Hardlink `id_path` to the content-hash blob depending on the remote flag.
+        Args:
+            id_path: Absolute path of the media id file that was just stored.
+            sha256: Hex digest of that file.
+            remote: True when the file is remote media.
+        """
+        if not sha256 or self.local_media_directory is None:
+            return
+
+        blob_path = self._blob_path(sha256, remote)
+        if blob_path is None:
+            return
+
+        _link_id_path_to_blob(id_path, blob_path)
+
     @trace
     @contextlib.asynccontextmanager
     async def ensure_media_is_in_local_cache(
@@ -464,6 +491,46 @@ def _file_info_to_path(self, file_info: FileInfo) -> str:
         return self.filepaths.local_media_filepath_rel(file_info.file_id)
 
 
+def _link_id_path_to_blob(id_path: str, blob_path: str) -> None:
+    """Point `id_path` at the inode named by `blob_path`, creating the blob if needed.
+
+    A failed link leaves `id_path` in place.
+    """
+    if not os.path.isfile(id_path):
+        return
+
+    tmp_link = id_path + ".dedup-link"
+    try:
+        os.makedirs(os.path.dirname(blob_path), exist_ok=True)
+        if not os.path.exists(blob_path):
+            try:
+                os.link(id_path, blob_path)
+                return
+            except FileExistsError:
+                pass
+        if os.path.samefile(id_path, blob_path):
+            return
+
+        try:
+            os.remove(tmp_link)
+        except FileNotFoundError:
+            pass
+        # Link the existing blob to a temporary name first, replacing the id path only happens after that link succeeds.
+        os.link(blob_path, tmp_link)
+        os.replace(tmp_link, id_path)
+    except OSError:
+        logger.warning(
+            "Failed to hardlink media file %s to %s",
+            id_path,
+            blob_path,
+            exc_info=True,
+        )
+        try:
+            os.remove(tmp_link)
+        except OSError:
+            pass
+
+
 @trace
 def _write_file_synchronously(source: IO, dest: IO) -> None:
     """Write `source` to the file like `dest` synchronously. Should be called

tests/media/test_filepath.py

@@ -66,6 +66,26 @@ def test_local_media_thumbnail_dir(self) -> None:
             "/media_store/local_thumbnails/Ge/rZ/NDnDZVjsOtardLuwfIBg",
         )
 
+    def test_content_hash_blob_paths(self) -> None:
+        """Local and remote blobs are separate trees, sharded like media ids."""
+        sha256 = "ebf4f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a"
+        self.assertEqual(
+            self.filepaths.local_media_blob_rel(sha256),
+            "local_blobs/eb/f4/f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a",
+        )
+        self.assertEqual(
+            self.filepaths.local_media_blob_filepath(sha256),
+            "/media_store/local_blobs/eb/f4/f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a",
+        )
+        self.assertEqual(
+            self.filepaths.remote_media_blob_rel(sha256),
+            "remote_blobs/eb/f4/f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a",
+        )
+        self.assertEqual(
+            self.filepaths.remote_media_blob_filepath(sha256),
+            "/media_store/remote_blobs/eb/f4/f635a17d10d6eb46ba680b70142419aa3220f228001a036d311a22ee9d2a",
+        )
+
     def test_remote_media_filepath(self) -> None:
         """Test remote media paths"""
         self.assertEqual(