diff --git a/backend/modules/asset_renamerr.py b/backend/modules/asset_renamerr.py index 09e94db2..564c8900 100644 --- a/backend/modules/asset_renamerr.py +++ b/backend/modules/asset_renamerr.py @@ -658,9 +658,9 @@ def _apply_kometa( url: Optional[str], refusals: Optional[List[WriteRefused]] = None, kept_sources: Optional[List[OSError]] = None, - prev: Optional[dict] = None, + keep_source: bool = False, ) -> Tuple[bool, str]: - """Place one asset for Kometa by action_type and drop what ``prev`` placed elsewhere; ``refusals``/``kept_sources`` collect at DEBUG.""" + """Place one asset for Kometa by action_type, or by copy with ``keep_source``; ``refusals``/``kept_sources`` collect at DEBUG.""" # Kometa applies LOCAL files only — fanart art is never downloaded to # disk (it's Plex-only, streamed straight to Plex). fanart is already # filtered out of the Kometa source set in match_and_apply_assets, so @@ -674,12 +674,13 @@ def _apply_kometa( if new_path is None: return False, why + action = "copy" if keep_source else self.config.action_type # A move takes a link, never the file it points at; the other actions work on the final file. - src = resolve_folder(src) if self.config.action_type == "move" else os.path.realpath(src) + src = resolve_folder(src) if action == "move" else os.path.realpath(src) if not self.config.dry_run: try: os.makedirs(os.path.dirname(new_path), exist_ok=True) - kept = self._file_op(src, new_path, self.config.action_type) + kept = self._file_op(src, new_path, action) except OSError as exc: line = f"Asset file op failed for {src} -> {new_path}: {exc}" if refusals is not None and destination_refused(exc, new_path): @@ -690,10 +691,16 @@ def _apply_kometa( return False, failure_text("file op", exc) if kept is not None: note_kept_source(self.logger, kept, kept_sources) - if prev and prev.get("applied_method") == "kometa": - remove_superseded(prev.get("applied_path"), new_path, self.config.destination_dir, self.logger) return True, new_path + def _drop_superseded(self, db: ChubDB, prev: Optional[dict], new_path: str) -> None: + """Remove the file ``prev`` placed for Kometa now that ``new_path`` replaces it, unless a row still records it.""" + cfg = self.config + old = (prev or {}).get("applied_path") + if cfg.dry_run or cfg.apply_method == "plex" or not old or prev.get("applied_method") != "kometa": + return + remove_superseded(old, new_path, cfg.destination_dir, self.logger, db.records_placed_path) + def _kometa_path(self, media: dict, image_type: str, src: str) -> Tuple[Optional[str], str]: """Where a Kometa apply of ``src`` lands, resolved: ``(path, "")``, or ``(None, why not)``.""" dest_root = self.config.destination_dir @@ -763,13 +770,14 @@ def apply_chosen_asset( target_kind, target_id, image_type, True, matched_file=file ) applied_libs: Optional[List[str]] = None + prev = None if apply_method == "plex": applied, detail, applied_libs = self._apply_direct( db, media, image_type, file, None, is_collection ) else: prev = db.media_asset_matches.get_one(target_kind, target_id, image_type) if target_id is not None else None - applied, detail = self._apply_kometa(media, image_type, "local", file, None, prev=prev) + applied, detail = self._apply_kometa(media, image_type, "local", file, None) if target_id is not None: db.media_asset_matches.upsert( @@ -788,6 +796,8 @@ def apply_chosen_asset( match_status="applied" if applied else "failed", detail=detail, ) + if applied: + self._drop_superseded(db, prev, detail) return applied, detail def _placed_file(self, row: Optional[dict]) -> Optional[str]: @@ -1118,6 +1128,7 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]: continue applied_libs: Optional[List[str]] = None + carry = placed is not None and file == placed if apply_method == "plex": applied, detail, applied_libs = self._apply_direct_remaining( db, @@ -1132,8 +1143,14 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]: ) else: attempted += 1 + # A carry under move copies while another item's row records the file; _drop_superseded keeps it then. + shared = ( + carry + and self.config.action_type == "move" + and db.records_placed_path(placed, own=(target_kind, target_id, image_type)) + ) applied, detail = self._apply_kometa( - media, image_type, source, file, url, refusals, kept_sources, prev + media, image_type, source, file, url, refusals, kept_sources, keep_source=shared ) # On a dry run, NEVER persist match state: media_asset_matches @@ -1142,7 +1159,7 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]: # was never actually uploaded. We still report it in the output # (marked "would apply") so the dry-run preview is complete. # A failed carry leaves the row on the file still placed, so the next run retries it. - carry_failed = placed is not None and not applied and file == placed + carry_failed = carry and not applied if target_id is not None and not self.config.dry_run and not carry_failed: db.media_asset_matches.upsert( target_kind=target_kind, @@ -1163,6 +1180,8 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]: detail=detail, unless_locked=unlocked_at_start, ) + if applied: + self._drop_superseded(db, prev, detail) output[image_type].append( { diff --git a/backend/modules/poster_renamerr.py b/backend/modules/poster_renamerr.py index 32d32374..9e7d22c7 100755 --- a/backend/modules/poster_renamerr.py +++ b/backend/modules/poster_renamerr.py @@ -1086,12 +1086,16 @@ def _moved_source(self, row: dict) -> Optional[str]: src = row.get("original_file") or row.get("file") return None if src and os.path.lexists(src) else staged - def _remove_superseded(self, previous: Optional[str], new_file_path: str) -> None: + def _remove_superseded(self, previous: Optional[str], new_file_path: str, db: ChubDB) -> None: """Drop the file an asset was staged to before it moved (e.g. a folder rename): Kometa ignores it and the orphan pass spares its id.""" config = self.config if config.dry_run or config.run_border_replacerr: return - remove_superseded(previous, new_file_path, config.destination_dir, self.logger) + remove_superseded(previous, new_file_path, config.destination_dir, self.logger, db.records_placed_path) + + def _carry_action(self, moved: Optional[str], db: ChubDB) -> str: + """The file action, once _record_staged moved our row off ``moved``: a carry copies while another row records it.""" + return "copy" if moved and db.records_placed_path(moved) else self.config.action_type def _rename_failed(self, log: Callable[[str], None], message: str) -> None: """Log why one poster was not staged, and count it for the Run Summary.""" @@ -1212,7 +1216,7 @@ def rename_file(self, item: dict, db: ChubDB) -> Optional[dict]: if ( file_ops_enabled and not config.dry_run - and not self.process_file(file, new_file_path, config.action_type) + and not self.process_file(file, new_file_path, self._carry_action(moved, db)) ): self._rename_failed( self.logger.warning, @@ -1228,7 +1232,7 @@ def rename_file(self, item: dict, db: ChubDB) -> Optional[dict]: # Now that the asset is staged at its current path, drop any copy left # behind at the previous path (folder-rename self-cleanup). if file_ops_enabled and not config.dry_run: - self._remove_superseded(previous_renamed, new_file_path) + self._remove_superseded(previous_renamed, new_file_path, db) if messages or discord_message: return { diff --git a/backend/util/database/__init__.py b/backend/util/database/__init__.py index b7e7625e..94cea8c4 100644 --- a/backend/util/database/__init__.py +++ b/backend/util/database/__init__.py @@ -1,7 +1,8 @@ # util/database/__init__.py +import sqlite3 import threading -from typing import List, Optional +from typing import List, Optional, Tuple from backend.util.paths import get_db_path from backend.util.logger import Logger @@ -400,6 +401,27 @@ def rename_instance(self, old_name: str, new_name: str, service: str) -> int: ) return total + def records_placed_path(self, path: str, own: Optional[Tuple[str, int, str]] = None) -> bool: + """True when a poster row, or a live item's asset row other than ``own`` (kind, id, image type), records ``path``; True on a database error.""" + try: + self._ensure_initialized() + assets = ( + "SELECT 1 FROM media_asset_matches m WHERE m.applied_path=? AND (" + "(m.target_kind='media' AND m.target_id IN (SELECT id FROM media_cache)) OR " + "(m.target_kind='collection' AND m.target_id IN (SELECT id FROM collections_cache)))" + ) + if own: + assets += " AND NOT (m.target_kind=? AND m.target_id=? AND m.image_type=?)" + checks = ( + (self.media, "SELECT 1 FROM media_cache WHERE renamed_file=? LIMIT 1", (path,)), + (self.collection, "SELECT 1 FROM collections_cache WHERE renamed_file=? LIMIT 1", (path,)), + (self.media_asset_matches, assets + " LIMIT 1", (path, *(own or ()))), + ) + return any(table.execute_query(sql, args, fetch_one=True) for table, sql, args in checks) + except sqlite3.Error as e: + self.logger.get_adapter("DATABASE").warning(f"Keeping {path}: could not check whether a row records it: {e}") + return True + def close(self) -> None: """ Manually close the database context. diff --git a/backend/util/file_ops.py b/backend/util/file_ops.py index 4da5e651..5edbb9b9 100644 --- a/backend/util/file_ops.py +++ b/backend/util/file_ops.py @@ -127,14 +127,19 @@ def removing_takes(previous: str, current: str) -> bool: return previous.casefold() == current.casefold() and _same_inode(previous, current) -def remove_superseded(previous: Optional[str], current: str, root: str, logger: Any) -> None: - """Remove the file placed at ``previous`` now that ``current`` replaces it, then its emptied folder; only inside ``root``.""" +def remove_superseded( + previous: Optional[str], current: str, root: str, logger: Any, recorded: Callable[[str], bool] +) -> None: + """Remove the file placed at ``previous`` now that ``current`` replaces it, then its emptied folder; only inside ``root`` and unless ``recorded(previous)``.""" if not previous or removing_takes(previous, current): return try: real_base = os.path.realpath(root) if not resolve_folder(previous).startswith(real_base + os.sep): return # outside the destination: never ours to remove + # Call after the caller's own row records current: any row still holding previous keeps the file. + if recorded(previous): + return # A move has already taken it away; its emptied folder still goes. if os.path.lexists(previous): os.remove(previous) diff --git a/tests/test_asset_renamerr.py b/tests/test_asset_renamerr.py index 76b23eed..fe91f813 100644 --- a/tests/test_asset_renamerr.py +++ b/tests/test_asset_renamerr.py @@ -2398,6 +2398,26 @@ def test_a_case_only_folder_rename_keeps_the_logo_on_a_case_insensitive_mount(db assert [p.read_bytes() for p in dest.iterdir()] == [b"png"] +@pytest.mark.parametrize("action_type", ["copy", "move"]) +@pytest.mark.parametrize("live", [True, False], ids=["live", "deleted"]) +def test_a_logo_another_item_records_stays_after_a_rename(db, monkeypatch, tmp_path, live, action_type): + m, dest = _logo_run_then_rename(db, monkeypatch, tmp_path, action_type=action_type) + other = db.media.execute_query( + "INSERT INTO media_cache (identity_key, instance_name, folder, asset_type) VALUES (?,?,?,?)", + ("radarr-4k|Dune (2021)|None", "radarr-4k", "Dune (2021)", "movie"), + last_row_id=True, + ) + if not live: + db.media.execute_query("DELETE FROM media_cache WHERE id=?", (other,)) + old = db.media_asset_matches.get_one("media", 1, "logo")["applied_path"] + db.media_asset_matches.upsert( + target_kind="media", target_id=other, image_type="logo", source="local", matched_file=old, + applied_method="kometa", applied_path=old, match_status="applied", + ) + m.run() + assert (os.path.exists(old), (dest / "Dune (2021) [renamed]_logo.png").exists()) == (live, True) + + def test_a_dry_run_keeps_the_logo_placed_under_the_old_folder_name(db, monkeypatch, tmp_path): m, dest = _logo_run_then_rename(db, monkeypatch, tmp_path) m.config.dry_run = True diff --git a/tests/test_poster_renamerr.py b/tests/test_poster_renamerr.py index 274ebdb0..9584a72f 100644 --- a/tests/test_poster_renamerr.py +++ b/tests/test_poster_renamerr.py @@ -13,6 +13,7 @@ import errno import json import os +import sqlite3 import threading from types import SimpleNamespace @@ -21,6 +22,7 @@ from backend.modules.poster_renamerr import PosterRenamerr from backend.util.connector import SyncResult +from backend.util.database import ChubDB def make_module(): @@ -1345,7 +1347,7 @@ class _Iface: def update(self, **kwargs): pass - db = SimpleNamespace(collection=_Iface(), media=_Iface()) + db = SimpleNamespace(collection=_Iface(), media=_Iface(), records_placed_path=lambda path: False) item = { "asset_type": "collection", "title": "Phineas and Ferb Collection", @@ -1573,7 +1575,7 @@ def test_needs_staging_when_nothing_staged(): assert m._needs_staging(row) is True -def test_remove_superseded_drops_old_folder(tmp_path): +def test_remove_superseded_drops_old_folder(tmp_path, db): """Re-staging after a folder rename removes the stale file and its now-empty old folder — Kometa ignores it and the orphan pass spares its id, so it would otherwise linger forever.""" @@ -1583,12 +1585,12 @@ def test_remove_superseded_drops_old_folder(tmp_path): old = old_dir / "poster.jpg" old.write_bytes(b"x") new = tmp_path / "Dune Prophecy (2024) {tvdb-367118}" / "poster.jpg" - m._remove_superseded(str(old), str(new)) + m._remove_superseded(str(old), str(new), db) assert not old.exists() assert not old_dir.exists() # emptied folder is removed too -def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path): +def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path, db): """A season poster left in the old folder keeps the folder until the last file is gone.""" m = _staging_module(destination_dir=str(tmp_path)) @@ -1599,29 +1601,92 @@ def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path): m._remove_superseded( str(old_dir / "poster.jpg"), str(tmp_path / "Show X (2024) {tvdb-1}" / "poster.jpg"), + db, ) assert old_dir.exists() # Season01.jpg still there assert (old_dir / "Season01.jpg").exists() -def test_remove_superseded_keeps_the_poster_both_paths_name(tmp_path): +@pytest.fixture +def db(tmp_path, stub_logger): + with ChubDB(stub_logger, db_path=str(tmp_path / "chub.db"), quiet=True) as database: + yield database + + +@pytest.mark.parametrize("action_type", ["copy", "move"]) +@pytest.mark.parametrize("shared", [True, False], ids=["shared", "own"]) +def test_a_renamed_poster_leaves_the_old_file_only_while_another_row_records_it(tmp_path, db, shared, action_type): + src = tmp_path / "src" / "Dune (2021).jpg" + src.parent.mkdir() + if action_type == "copy": + src.write_bytes(b"poster") # under move the source is gone: the staged file is carried + kometa = tmp_path / "kometa" + old = kometa / "Dune (2021)" / "poster.jpg" + old.parent.mkdir(parents=True) + old.write_bytes(b"poster") + m = make_module() + m.config = SimpleNamespace( + destination_dir=str(kometa), asset_folders=True, dry_run=False, action_type=action_type, + run_border_replacerr=False, print_only_renames=False, + ) + for instance, folder in [("radarr-4k", "Dune (2021) {edition-4K}")] + [("radarr-hd", "Dune (2021)")] * shared: + db.media.execute_query( + "INSERT INTO media_cache (identity_key, instance_name, folder, asset_type, title, year, renamed_file)" + " VALUES (?,?,?,?,?,?,?)", + (f"{instance}|{folder}|None", instance, folder, "movie", "Dune", 2021, str(old)), + ) + row = db.media.execute_query("SELECT * FROM media_cache WHERE instance_name='radarr-4k'", fetch_one=True) + m.rename_file(dict(row, original_file=str(src)), db) + assert (old.exists(), (kometa / "Dune (2021) {edition-4K}" / "poster.jpg").read_bytes()) == (shared, b"poster") + + +@pytest.mark.parametrize("holder, recorded", [("collection", True), ("deleted item's asset", False)]) +def test_records_placed_path_reads_collections_and_only_live_items_assets(db, holder, recorded): + path = "/kometa/Marvel.jpg" + if holder == "collection": + db.collection.upsert({"title": "Marvel", "library_name": "Movies"}, "plex1") + db.collection.set_renamed_file(db.collection.get_by_title_and_instance("Marvel", "plex1", "Movies")["id"], path) + else: + db.media_asset_matches.upsert( + target_kind="media", target_id=99, image_type="logo", source="local", matched_file=path, + applied_method="kometa", applied_path=path, match_status="applied", + ) + found = db.records_placed_path(path) + assert found is recorded + + +@pytest.mark.parametrize("step", ["query", "schema"]) +def test_a_failed_placed_path_check_keeps_the_file(db, monkeypatch, stub_logger, step): + def locked(*a, **k): + raise sqlite3.OperationalError("database is locked") + + target, name = (db.media, "execute_query") if step == "query" else (db, "_ensure_initialized") + monkeypatch.setattr(target, name, locked) + kept = db.records_placed_path("/kometa/Dune (2021)/poster.jpg") + assert (kept, stub_logger.messages["warning"]) == ( + True, + ["Keeping /kometa/Dune (2021)/poster.jpg: could not check whether a row records it: database is locked"], + ) + + +def test_remove_superseded_keeps_the_poster_both_paths_name(tmp_path, db): real = tmp_path / "posters" real.mkdir() (tmp_path / "alias").symlink_to(real) poster = real / "Dune (2021).jpg" poster.write_bytes(b"x") m = _staging_module(destination_dir=str(real)) - m._remove_superseded(str(tmp_path / "alias" / "Dune (2021).jpg"), str(poster)) + m._remove_superseded(str(tmp_path / "alias" / "Dune (2021).jpg"), str(poster), db) assert poster.read_bytes() == b"x" -def test_remove_superseded_ignores_paths_outside_destination(tmp_path): +def test_remove_superseded_ignores_paths_outside_destination(tmp_path, db): """Never delete outside the staging tree, even if asked.""" m = _staging_module(destination_dir=str(tmp_path / "assets")) (tmp_path / "assets").mkdir() outside = tmp_path / "elsewhere.jpg" outside.write_bytes(b"x") - m._remove_superseded(str(outside), str(tmp_path / "assets" / "poster.jpg")) + m._remove_superseded(str(outside), str(tmp_path / "assets" / "poster.jpg"), db) assert outside.exists() # untouched