Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 28 additions & 9 deletions backend/modules/asset_renamerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,9 +658,9 @@ def _apply_kometa(
url: Optional[str],
refusals: Optional[List[WriteRefused]] = None,
kept_sources: Optional[List[OSError]] = None,
prev: Optional[dict] = None,
keep_source: bool = False,
) -> Tuple[bool, str]:
"""Place one asset for Kometa by action_type and drop what ``prev`` placed elsewhere; ``refusals``/``kept_sources`` collect at DEBUG."""
"""Place one asset for Kometa by action_type, or by copy with ``keep_source``; ``refusals``/``kept_sources`` collect at DEBUG."""
# Kometa applies LOCAL files only — fanart art is never downloaded to
# disk (it's Plex-only, streamed straight to Plex). fanart is already
# filtered out of the Kometa source set in match_and_apply_assets, so
Expand All @@ -674,12 +674,13 @@ def _apply_kometa(
if new_path is None:
return False, why

action = "copy" if keep_source else self.config.action_type
# A move takes a link, never the file it points at; the other actions work on the final file.
src = resolve_folder(src) if self.config.action_type == "move" else os.path.realpath(src)
src = resolve_folder(src) if action == "move" else os.path.realpath(src)
if not self.config.dry_run:
try:
os.makedirs(os.path.dirname(new_path), exist_ok=True)
kept = self._file_op(src, new_path, self.config.action_type)
kept = self._file_op(src, new_path, action)
except OSError as exc:
line = f"Asset file op failed for {src} -> {new_path}: {exc}"
if refusals is not None and destination_refused(exc, new_path):
Expand All @@ -690,10 +691,16 @@ def _apply_kometa(
return False, failure_text("file op", exc)
if kept is not None:
note_kept_source(self.logger, kept, kept_sources)
if prev and prev.get("applied_method") == "kometa":
remove_superseded(prev.get("applied_path"), new_path, self.config.destination_dir, self.logger)
return True, new_path

def _drop_superseded(self, db: ChubDB, prev: Optional[dict], new_path: str) -> None:
"""Remove the file ``prev`` placed for Kometa now that ``new_path`` replaces it, unless a row still records it."""
cfg = self.config
old = (prev or {}).get("applied_path")
if cfg.dry_run or cfg.apply_method == "plex" or not old or prev.get("applied_method") != "kometa":
return
remove_superseded(old, new_path, cfg.destination_dir, self.logger, db.records_placed_path)

def _kometa_path(self, media: dict, image_type: str, src: str) -> Tuple[Optional[str], str]:
"""Where a Kometa apply of ``src`` lands, resolved: ``(path, "")``, or ``(None, why not)``."""
dest_root = self.config.destination_dir
Expand Down Expand Up @@ -763,13 +770,14 @@ def apply_chosen_asset(
target_kind, target_id, image_type, True, matched_file=file
)
applied_libs: Optional[List[str]] = None
prev = None
if apply_method == "plex":
applied, detail, applied_libs = self._apply_direct(
db, media, image_type, file, None, is_collection
)
else:
prev = db.media_asset_matches.get_one(target_kind, target_id, image_type) if target_id is not None else None
applied, detail = self._apply_kometa(media, image_type, "local", file, None, prev=prev)
applied, detail = self._apply_kometa(media, image_type, "local", file, None)

if target_id is not None:
db.media_asset_matches.upsert(
Expand All @@ -788,6 +796,8 @@ def apply_chosen_asset(
match_status="applied" if applied else "failed",
detail=detail,
)
if applied:
self._drop_superseded(db, prev, detail)
return applied, detail

def _placed_file(self, row: Optional[dict]) -> Optional[str]:
Expand Down Expand Up @@ -1118,6 +1128,7 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]:
continue

applied_libs: Optional[List[str]] = None
carry = placed is not None and file == placed
if apply_method == "plex":
applied, detail, applied_libs = self._apply_direct_remaining(
db,
Expand All @@ -1132,8 +1143,14 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]:
)
else:
attempted += 1
# A carry under move copies while another item's row records the file; _drop_superseded keeps it then.
shared = (
carry
and self.config.action_type == "move"
and db.records_placed_path(placed, own=(target_kind, target_id, image_type))
)
applied, detail = self._apply_kometa(
media, image_type, source, file, url, refusals, kept_sources, prev
media, image_type, source, file, url, refusals, kept_sources, keep_source=shared
)

# On a dry run, NEVER persist match state: media_asset_matches
Expand All @@ -1142,7 +1159,7 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]:
# was never actually uploaded. We still report it in the output
# (marked "would apply") so the dry-run preview is complete.
# A failed carry leaves the row on the file still placed, so the next run retries it.
carry_failed = placed is not None and not applied and file == placed
carry_failed = carry and not applied
if target_id is not None and not self.config.dry_run and not carry_failed:
db.media_asset_matches.upsert(
target_kind=target_kind,
Expand All @@ -1163,6 +1180,8 @@ def match_and_apply_assets(self, db: ChubDB) -> Dict[str, List[dict]]:
detail=detail,
unless_locked=unlocked_at_start,
)
if applied:
self._drop_superseded(db, prev, detail)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

output[image_type].append(
{
Expand Down
12 changes: 8 additions & 4 deletions backend/modules/poster_renamerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -1086,12 +1086,16 @@ def _moved_source(self, row: dict) -> Optional[str]:
src = row.get("original_file") or row.get("file")
return None if src and os.path.lexists(src) else staged

def _remove_superseded(self, previous: Optional[str], new_file_path: str) -> None:
def _remove_superseded(self, previous: Optional[str], new_file_path: str, db: ChubDB) -> None:
"""Drop the file an asset was staged to before it moved (e.g. a folder rename): Kometa ignores it and the orphan pass spares its id."""
config = self.config
if config.dry_run or config.run_border_replacerr:
return
remove_superseded(previous, new_file_path, config.destination_dir, self.logger)
remove_superseded(previous, new_file_path, config.destination_dir, self.logger, db.records_placed_path)

def _carry_action(self, moved: Optional[str], db: ChubDB) -> str:
"""The file action, once _record_staged moved our row off ``moved``: a carry copies while another row records it."""
return "copy" if moved and db.records_placed_path(moved) else self.config.action_type

def _rename_failed(self, log: Callable[[str], None], message: str) -> None:
"""Log why one poster was not staged, and count it for the Run Summary."""
Expand Down Expand Up @@ -1212,7 +1216,7 @@ def rename_file(self, item: dict, db: ChubDB) -> Optional[dict]:
if (
file_ops_enabled
and not config.dry_run
and not self.process_file(file, new_file_path, config.action_type)
and not self.process_file(file, new_file_path, self._carry_action(moved, db))
):
self._rename_failed(
self.logger.warning,
Expand All @@ -1228,7 +1232,7 @@ def rename_file(self, item: dict, db: ChubDB) -> Optional[dict]:
# Now that the asset is staged at its current path, drop any copy left
# behind at the previous path (folder-rename self-cleanup).
if file_ops_enabled and not config.dry_run:
self._remove_superseded(previous_renamed, new_file_path)
self._remove_superseded(previous_renamed, new_file_path, db)

if messages or discord_message:
return {
Expand Down
24 changes: 23 additions & 1 deletion backend/util/database/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# util/database/__init__.py

import sqlite3
import threading
from typing import List, Optional
from typing import List, Optional, Tuple

from backend.util.paths import get_db_path
from backend.util.logger import Logger
Expand Down Expand Up @@ -400,6 +401,27 @@ def rename_instance(self, old_name: str, new_name: str, service: str) -> int:
)
return total

def records_placed_path(self, path: str, own: Optional[Tuple[str, int, str]] = None) -> bool:
"""True when a poster row, or a live item's asset row other than ``own`` (kind, id, image type), records ``path``; True on a database error."""
try:
self._ensure_initialized()
assets = (
"SELECT 1 FROM media_asset_matches m WHERE m.applied_path=? AND ("
"(m.target_kind='media' AND m.target_id IN (SELECT id FROM media_cache)) OR "
"(m.target_kind='collection' AND m.target_id IN (SELECT id FROM collections_cache)))"
)
if own:
assets += " AND NOT (m.target_kind=? AND m.target_id=? AND m.image_type=?)"
checks = (
(self.media, "SELECT 1 FROM media_cache WHERE renamed_file=? LIMIT 1", (path,)),
(self.collection, "SELECT 1 FROM collections_cache WHERE renamed_file=? LIMIT 1", (path,)),
(self.media_asset_matches, assets + " LIMIT 1", (path, *(own or ()))),
)
return any(table.execute_query(sql, args, fetch_one=True) for table, sql, args in checks)
except sqlite3.Error as e:
self.logger.get_adapter("DATABASE").warning(f"Keeping {path}: could not check whether a row records it: {e}")
return True

def close(self) -> None:
"""
Manually close the database context.
Expand Down
9 changes: 7 additions & 2 deletions backend/util/file_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,14 +127,19 @@ def removing_takes(previous: str, current: str) -> bool:
return previous.casefold() == current.casefold() and _same_inode(previous, current)


def remove_superseded(previous: Optional[str], current: str, root: str, logger: Any) -> None:
"""Remove the file placed at ``previous`` now that ``current`` replaces it, then its emptied folder; only inside ``root``."""
def remove_superseded(
previous: Optional[str], current: str, root: str, logger: Any, recorded: Callable[[str], bool]
) -> None:
"""Remove the file placed at ``previous`` now that ``current`` replaces it, then its emptied folder; only inside ``root`` and unless ``recorded(previous)``."""
if not previous or removing_takes(previous, current):
return
try:
real_base = os.path.realpath(root)
if not resolve_folder(previous).startswith(real_base + os.sep):
return # outside the destination: never ours to remove
# Call after the caller's own row records current: any row still holding previous keeps the file.
if recorded(previous):
return
# A move has already taken it away; its emptied folder still goes.
if os.path.lexists(previous):
os.remove(previous)
Expand Down
20 changes: 20 additions & 0 deletions tests/test_asset_renamerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -2398,6 +2398,26 @@ def test_a_case_only_folder_rename_keeps_the_logo_on_a_case_insensitive_mount(db
assert [p.read_bytes() for p in dest.iterdir()] == [b"png"]


@pytest.mark.parametrize("action_type", ["copy", "move"])
@pytest.mark.parametrize("live", [True, False], ids=["live", "deleted"])
def test_a_logo_another_item_records_stays_after_a_rename(db, monkeypatch, tmp_path, live, action_type):
m, dest = _logo_run_then_rename(db, monkeypatch, tmp_path, action_type=action_type)
other = db.media.execute_query(
"INSERT INTO media_cache (identity_key, instance_name, folder, asset_type) VALUES (?,?,?,?)",
("radarr-4k|Dune (2021)|None", "radarr-4k", "Dune (2021)", "movie"),
last_row_id=True,
)
if not live:
db.media.execute_query("DELETE FROM media_cache WHERE id=?", (other,))
old = db.media_asset_matches.get_one("media", 1, "logo")["applied_path"]
db.media_asset_matches.upsert(
target_kind="media", target_id=other, image_type="logo", source="local", matched_file=old,
applied_method="kometa", applied_path=old, match_status="applied",
)
m.run()
assert (os.path.exists(old), (dest / "Dune (2021) [renamed]_logo.png").exists()) == (live, True)


def test_a_dry_run_keeps_the_logo_placed_under_the_old_folder_name(db, monkeypatch, tmp_path):
m, dest = _logo_run_then_rename(db, monkeypatch, tmp_path)
m.config.dry_run = True
Expand Down
81 changes: 73 additions & 8 deletions tests/test_poster_renamerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import errno
import json
import os
import sqlite3
import threading
from types import SimpleNamespace

Expand All @@ -21,6 +22,7 @@

from backend.modules.poster_renamerr import PosterRenamerr
from backend.util.connector import SyncResult
from backend.util.database import ChubDB


def make_module():
Expand Down Expand Up @@ -1345,7 +1347,7 @@ class _Iface:
def update(self, **kwargs):
pass

db = SimpleNamespace(collection=_Iface(), media=_Iface())
db = SimpleNamespace(collection=_Iface(), media=_Iface(), records_placed_path=lambda path: False)
item = {
"asset_type": "collection",
"title": "Phineas and Ferb Collection",
Expand Down Expand Up @@ -1573,7 +1575,7 @@ def test_needs_staging_when_nothing_staged():
assert m._needs_staging(row) is True


def test_remove_superseded_drops_old_folder(tmp_path):
def test_remove_superseded_drops_old_folder(tmp_path, db):
"""Re-staging after a folder rename removes the stale file and its now-empty
old folder — Kometa ignores it and the orphan pass spares its id, so it
would otherwise linger forever."""
Expand All @@ -1583,12 +1585,12 @@ def test_remove_superseded_drops_old_folder(tmp_path):
old = old_dir / "poster.jpg"
old.write_bytes(b"x")
new = tmp_path / "Dune Prophecy (2024) {tvdb-367118}" / "poster.jpg"
m._remove_superseded(str(old), str(new))
m._remove_superseded(str(old), str(new), db)
assert not old.exists()
assert not old_dir.exists() # emptied folder is removed too


def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path):
def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path, db):
"""A season poster left in the old folder keeps the folder until the last
file is gone."""
m = _staging_module(destination_dir=str(tmp_path))
Expand All @@ -1599,29 +1601,92 @@ def test_remove_superseded_keeps_shared_folder_until_empty(tmp_path):
m._remove_superseded(
str(old_dir / "poster.jpg"),
str(tmp_path / "Show X (2024) {tvdb-1}" / "poster.jpg"),
db,
)
assert old_dir.exists() # Season01.jpg still there
assert (old_dir / "Season01.jpg").exists()


def test_remove_superseded_keeps_the_poster_both_paths_name(tmp_path):
@pytest.fixture
def db(tmp_path, stub_logger):
with ChubDB(stub_logger, db_path=str(tmp_path / "chub.db"), quiet=True) as database:
yield database


@pytest.mark.parametrize("action_type", ["copy", "move"])
@pytest.mark.parametrize("shared", [True, False], ids=["shared", "own"])
def test_a_renamed_poster_leaves_the_old_file_only_while_another_row_records_it(tmp_path, db, shared, action_type):
src = tmp_path / "src" / "Dune (2021).jpg"
src.parent.mkdir()
if action_type == "copy":
src.write_bytes(b"poster") # under move the source is gone: the staged file is carried
kometa = tmp_path / "kometa"
old = kometa / "Dune (2021)" / "poster.jpg"
old.parent.mkdir(parents=True)
old.write_bytes(b"poster")
m = make_module()
m.config = SimpleNamespace(
destination_dir=str(kometa), asset_folders=True, dry_run=False, action_type=action_type,
run_border_replacerr=False, print_only_renames=False,
)
for instance, folder in [("radarr-4k", "Dune (2021) {edition-4K}")] + [("radarr-hd", "Dune (2021)")] * shared:
db.media.execute_query(
"INSERT INTO media_cache (identity_key, instance_name, folder, asset_type, title, year, renamed_file)"
" VALUES (?,?,?,?,?,?,?)",
(f"{instance}|{folder}|None", instance, folder, "movie", "Dune", 2021, str(old)),
)
row = db.media.execute_query("SELECT * FROM media_cache WHERE instance_name='radarr-4k'", fetch_one=True)
m.rename_file(dict(row, original_file=str(src)), db)
assert (old.exists(), (kometa / "Dune (2021) {edition-4K}" / "poster.jpg").read_bytes()) == (shared, b"poster")


@pytest.mark.parametrize("holder, recorded", [("collection", True), ("deleted item's asset", False)])
def test_records_placed_path_reads_collections_and_only_live_items_assets(db, holder, recorded):
path = "/kometa/Marvel.jpg"
if holder == "collection":
db.collection.upsert({"title": "Marvel", "library_name": "Movies"}, "plex1")
db.collection.set_renamed_file(db.collection.get_by_title_and_instance("Marvel", "plex1", "Movies")["id"], path)
else:
db.media_asset_matches.upsert(
target_kind="media", target_id=99, image_type="logo", source="local", matched_file=path,
applied_method="kometa", applied_path=path, match_status="applied",
)
found = db.records_placed_path(path)
assert found is recorded


@pytest.mark.parametrize("step", ["query", "schema"])
def test_a_failed_placed_path_check_keeps_the_file(db, monkeypatch, stub_logger, step):
def locked(*a, **k):
raise sqlite3.OperationalError("database is locked")

target, name = (db.media, "execute_query") if step == "query" else (db, "_ensure_initialized")
monkeypatch.setattr(target, name, locked)
kept = db.records_placed_path("/kometa/Dune (2021)/poster.jpg")
assert (kept, stub_logger.messages["warning"]) == (
True,
["Keeping /kometa/Dune (2021)/poster.jpg: could not check whether a row records it: database is locked"],
)


def test_remove_superseded_keeps_the_poster_both_paths_name(tmp_path, db):
real = tmp_path / "posters"
real.mkdir()
(tmp_path / "alias").symlink_to(real)
poster = real / "Dune (2021).jpg"
poster.write_bytes(b"x")
m = _staging_module(destination_dir=str(real))
m._remove_superseded(str(tmp_path / "alias" / "Dune (2021).jpg"), str(poster))
m._remove_superseded(str(tmp_path / "alias" / "Dune (2021).jpg"), str(poster), db)
assert poster.read_bytes() == b"x"


def test_remove_superseded_ignores_paths_outside_destination(tmp_path):
def test_remove_superseded_ignores_paths_outside_destination(tmp_path, db):
"""Never delete outside the staging tree, even if asked."""
m = _staging_module(destination_dir=str(tmp_path / "assets"))
(tmp_path / "assets").mkdir()
outside = tmp_path / "elsewhere.jpg"
outside.write_bytes(b"x")
m._remove_superseded(str(outside), str(tmp_path / "assets" / "poster.jpg"))
m._remove_superseded(str(outside), str(tmp_path / "assets" / "poster.jpg"), db)
assert outside.exists() # untouched


Expand Down
Loading