fix(importer): confirm deletes against the filesystem before they cascade

NEWS.md

@@ -18,6 +18,13 @@ border-radius: 128px;
     - Redesigned the Admin Tagging Status table to be more informative.
 
 - Fixes
+    - Comics are never deleted from the database while their files are still on
+      disk, so a misread filesystem event can no longer take a comic's bookmarks
+      and read progress with it.
+    - A comic replaced in place, by a tool that removes and rewrites the file,
+      is re-read instead of deleted and re-added as a new comic.
+    - Deleting a folder refreshes the series and publishers it emptied, which
+      kept listing comics that were gone.
     - Renaming follows a comic to its new path. Tagging a CBR converts it to CBZ
       without the rename failing, and a watched library no longer mistakes an
       unrelated new file for a renamed comic.

codex/librarian/fs/import_task.py

@@ -48,8 +48,28 @@ def _remove_paths(kwargs: dict[str, Any], deleted_key: str, moved_key: str) -> N
             del kwargs[moved_key][src_path]
 
 
+def _replaced_paths(kwargs: dict[str, Any], added_key: str, deleted_key: str) -> set:
+    """
+    Take paths reported both deleted and added; they were replaced in place.
+
+    An external tool that swaps a file by ``rm`` + ``mv``, or a watcher
+    backend that reports an atomic replace as a delete plus an add, leaves
+    both events in one batch. The recreated file carries a new inode, so
+    move detection can never pair them — and letting the delete win would
+    destroy the row, and its bookmarks, while a file sits at that very
+    path. It is the same path with new content: a modification.
+    """
+    replaced = kwargs[added_key] & kwargs[deleted_key]
+    kwargs[added_key] -= replaced
+    kwargs[deleted_key] -= replaced
+    return replaced
+
+
 def _deduplicate(kwargs: dict[str, Any]) -> None:
     """Prune conflicting events on the same paths."""
+    replaced_files = _replaced_paths(kwargs, "files_added", "files_deleted")
+    replaced_covers = _replaced_paths(kwargs, "covers_added", "covers_deleted")
+
     # deleted wins over moved-from-this-source
     _remove_paths(kwargs, "dirs_deleted", "dirs_moved")
     _remove_paths(kwargs, "files_deleted", "files_moved")
@@ -81,6 +101,11 @@ def _deduplicate(kwargs: dict[str, Any]) -> None:
     kwargs["covers_modified"] -= kwargs["covers_deleted"]
     kwargs["covers_modified"] -= kwargs["covers_added"]
 
+    # Added last: a replaced path is neither created nor deleted, and the
+    # subtractions above would have stripped it back out.
+    kwargs["files_modified"] |= replaced_files
+    kwargs["covers_modified"] |= replaced_covers
+
 
 def build_import_task(
     library_id: int,

codex/librarian/scribe/importer/delete/__init__.py

@@ -11,12 +11,20 @@ def delete(self) -> None:
         """Delete files and folders."""
         if self.abort_event.is_set():
             return
-        self.counts.folders_deleted += self.bulk_folders_deleted()
+        folders_deleted, comics_cascaded, folder_collections = (
+            self.bulk_folders_deleted()
+        )
+        self.counts.folders_deleted += folders_deleted
+        # Comics under a deleted folder die by cascade, not by path, so they
+        # never reach ``bulk_comics_deleted`` to be counted there.
+        self.counts.comics_deleted += comics_cascaded
         if self.abort_event.is_set():
             return
-        self.counts.comics_deleted, deleted_comic_collections = (
-            self.bulk_comics_deleted()
-        )
+        comics_deleted, deleted_comic_collections = self.bulk_comics_deleted()
+        self.counts.comics_deleted += comics_deleted
+        for model, pks in folder_collections.items():
+            if pks:
+                deleted_comic_collections.setdefault(model, set()).update(pks)
         if self.abort_event.is_set():
             return
         self.counts.covers_deleted = self.bulk_covers_deleted()

codex/librarian/scribe/importer/delete/comics.py

@@ -2,13 +2,20 @@
 
 from codex.librarian.scribe.importer.const import ALL_COMIC_COLLECTION_FIELD_NAMES
 from codex.librarian.scribe.importer.delete.covers import DeletedCoversImporter
+from codex.librarian.scribe.importer.delete.existence import confirm_deleted
 from codex.librarian.scribe.importer.statii.delete import ImporterRemoveComicsStatus
 from codex.models import Comic, Folder, StoryArc
 from codex.settings import (
     IMPORTER_DELETE_MAX_CHUNK_SIZE,
     IMPORTER_LINK_FK_BATCH_SIZE,
 )
 
+# A delete this large, and this much of the library, reads like a vanished
+# mount rather than a user tidying up. The floor keeps small libraries from
+# tripping it whenever a couple of comics are removed.
+_MASS_DELETE_FLOOR = 50
+_MASS_DELETE_FRACTION = 0.5
+
 
 class DeletedComicsImporter(DeletedCoversImporter):
     """Delete comics methods."""
@@ -54,6 +61,26 @@ def _populate_deleted_comic_collections(
         ):
             cls._populate_deleted_comic_collection(deleted_comic_collections, comic)
 
+    def _warn_on_mass_delete(self, num_deleted: int) -> None:
+        """
+        Flag a delete large enough to look like the library vanished.
+
+        An unmounted volume or dropped network share makes every path read
+        as missing, which the existence backstop cannot tell from a real
+        mass deletion. Nothing is blocked here — this only leaves a
+        breadcrumb in the log for a user asking where their comics went.
+        """
+        if num_deleted < _MASS_DELETE_FLOOR:
+            return
+        total = Comic.objects.filter(library=self.library).count()
+        if total and num_deleted >= total * _MASS_DELETE_FRACTION:
+            reason = (
+                f"Deleting {num_deleted} of {total} comics in"
+                f" {self.library.path}. If that library lives on a network"
+                f" share or removable volume, check that it is still mounted."
+            )
+            self.log.warning(reason)
+
     def bulk_comics_deleted(self, **kwargs) -> tuple[int, dict]:
         """Bulk delete comics found missing from the filesystem."""
         count = 0
@@ -64,8 +91,11 @@ def bulk_comics_deleted(self, **kwargs) -> tuple[int, dict]:
                 return count, deleted_comic_collections
             self.status_controller.start(status)
             # Batch path__in to stay under SQLite's variable limit.
-            paths = tuple(self.task.files_deleted)
+            paths = confirm_deleted(self.task.files_deleted, self.log, "comics")
             self.task.files_deleted = frozenset()
+            if not paths:
+                return count, deleted_comic_collections
+            self._warn_on_mass_delete(len(paths))
             delete_comic_pks: set[int] = set()
             for start in range(0, len(paths), IMPORTER_LINK_FK_BATCH_SIZE):
                 if self.abort_event.is_set():

codex/librarian/scribe/importer/delete/covers.py

@@ -1,6 +1,7 @@
 """Clean up covers from the db."""
 
 from codex.librarian.covers.tasks import CoverRemoveTask
+from codex.librarian.scribe.importer.delete.existence import confirm_deleted
 from codex.librarian.scribe.importer.search import SearchIndexImporter
 from codex.librarian.scribe.importer.statii.delete import ImporterRemoveCoversStatus
 from codex.models.paths import CustomCover
@@ -23,10 +24,11 @@ def bulk_covers_deleted(self, **kwargs) -> int:
             if not self.task.covers_deleted:
                 return 0
             self.status_controller.start(status)
-            covers = CustomCover.objects.filter(
-                library=self.library, path__in=self.task.covers_deleted
-            )
+            paths = confirm_deleted(self.task.covers_deleted, self.log, "covers")
             self.task.covers_deleted = frozenset()
+            if not paths:
+                return 0
+            covers = CustomCover.objects.filter(library=self.library, path__in=paths)
             delete_cover_pks = frozenset(covers.values_list("pk", flat=True))
             count, _ = covers.delete()
 

codex/librarian/scribe/importer/delete/existence.py

@@ -0,0 +1,49 @@
+"""
+Confirm a scanner's deletes against the filesystem before acting on them.
+
+Deleting a comic row cascades its bookmarks and read progress away, and
+nothing restores them — the next scan re-imports the file as a fresh,
+unread comic. So a delete is only safe when the file is really gone.
+
+Both scanners infer deletes rather than observing them, and every inference
+they make has failure modes: a watcher batch that reports a delete whose
+paired add lands in the *next* batch, a directory expansion that overmatched,
+an inode pair the compatibility checks refused. In each case the path is
+still on disk, and the delete is wrong.
+
+Probing the path is cheap next to what it protects, and a path that is
+genuinely gone answers immediately. A row skipped here is not stranded: it
+still points at a real file, so the next scan reconciles it normally — a
+stale row costs a re-read, a wrongly deleted one costs the user's place in
+the book.
+
+This cannot save a library whose whole mount disappeared, where every path
+reads as missing. ``DeletedComicsImporter`` logs that case instead.
+"""
+
+from collections.abc import Collection
+from pathlib import Path
+
+
+def split_extant(paths: Collection[str]) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    """Partition paths into (gone from disk, still on disk)."""
+    gone: list[str] = []
+    extant: list[str] = []
+    for path in paths:
+        if Path(path).exists():
+            extant.append(path)
+        else:
+            gone.append(path)
+    return tuple(gone), tuple(extant)
+
+
+def confirm_deleted(paths: Collection[str], log, kind: str) -> tuple[str, ...]:
+    """Return only the paths that are really gone, reporting any that aren't."""
+    gone, extant = split_extant(paths)
+    if extant:
+        reason = (
+            f"Not deleting {len(extant)} {kind} a scan reported missing that"
+            f" are still on disk. The next scan will reconcile them."
+        )
+        log.warning(reason)
+    return gone

codex/librarian/scribe/importer/delete/folders.py

@@ -1,6 +1,7 @@
 """Delete database folders methods."""
 
 from codex.librarian.scribe.importer.delete.comics import DeletedComicsImporter
+from codex.librarian.scribe.importer.delete.existence import confirm_deleted
 from codex.librarian.scribe.importer.statii.delete import ImporterRemoveFoldersStatus
 from codex.models.collections import Folder
 from codex.models.comic import Comic
@@ -9,26 +10,39 @@
 class DeletedFoldersImporter(DeletedComicsImporter):
     """Delete database folders methods."""
 
-    def bulk_folders_deleted(self, **kwargs) -> int:
-        """Bulk delete folders."""
+    def bulk_folders_deleted(self, **kwargs) -> tuple[int, int, dict]:
+        """
+        Bulk delete folders. Return (folders, cascaded comics, collections).
+
+        Comics under a deleted folder die by the ``parent_folder`` cascade
+        rather than through ``bulk_comics_deleted``, so their collections
+        are gathered here too. Without them a series or publisher emptied by
+        a folder delete is never re-stamped, and browsers viewing it keep
+        listing comics that are gone until some unrelated import moves the
+        timestamp.
+        """
         status = ImporterRemoveFoldersStatus(0, len(self.task.dirs_deleted))
+        deleted_comic_collections = self._init_deleted_comic_collections()
         try:
             if not self.task.dirs_deleted:
-                return 0
+                return 0, 0, deleted_comic_collections
             self.status_controller.start(status)
-            folders = Folder.objects.filter(
-                library=self.library, path__in=self.task.dirs_deleted
-            )
+            paths = confirm_deleted(self.task.dirs_deleted, self.log, "folders")
             self.task.dirs_deleted = frozenset()
-            delete_comic_pks = frozenset(
-                Comic.objects.filter(library=self.library, folders__in=folders)
-                .distinct()
-                .values_list("pk", flat=True)
+            if not paths:
+                return 0, 0, deleted_comic_collections
+            folders = Folder.objects.filter(library=self.library, path__in=paths)
+            folder_count = folders.count()
+            delete_comic_qs = Comic.objects.filter(
+                library=self.library, folders__in=folders
+            ).distinct()
+            self._populate_deleted_comic_collections(
+                delete_comic_qs, deleted_comic_collections
             )
+            delete_comic_pks = frozenset(delete_comic_qs.values_list("pk", flat=True))
             folders.delete()
-            count = len(delete_comic_pks)
 
             self.remove_covers(delete_comic_pks, custom=False)
         finally:
             self.status_controller.finish(status)
-        return count
+        return folder_count, len(delete_comic_pks), deleted_comic_collections