diff --git a/backend/modules/poster_cleanarr.py b/backend/modules/poster_cleanarr.py index df36c788..aae38710 100644 --- a/backend/modules/poster_cleanarr.py +++ b/backend/modules/poster_cleanarr.py @@ -4,7 +4,9 @@ import glob import json import os +import re import shutil +import stat import threading import time import zipfile @@ -22,6 +24,7 @@ tvdb_id_regex, ) from backend.util.database import ChubDB +from backend.util.file_ops import removing_takes from backend.util.helper import create_table from backend.util.logger import Logger from backend.util.notification import NotificationManager @@ -94,6 +97,8 @@ def _bundle_root(path: str) -> Optional[str]: ASSET_IMAGE_EXTS = ASSET_IMAGE_EXTENSIONS RESTORE_DIR_NAME = "Poster Cleanarr Restore" ORPHAN_RESTORE_DIR_NAME = ".chub_orphan_restore" +# What a flat placement adds after its folder name: "", "_logo", "_Season01_logo"..., then the extension. +_FLAT_SUFFIX = re.compile(r"(?:_[A-Za-z0-9]+)*\.\w+$") PLEX_DB_NAME = "com.plexapp.plugins.library.db" DB_MAX_AGE_HOURS = 2 @@ -114,6 +119,80 @@ def _bundle_root(path: str) -> Optional[str]: } +def _live_id( + name: str, canonical_by_id: Dict[Tuple[str, int], Set[str]] +) -> Optional[Tuple[str, int]]: + """The {tvdb/tmdb} id in ``name`` naming a live media item, tvdb first.""" + for kind, regex in (("tvdb", tvdb_id_regex), ("tmdb", tmdb_id_regex)): + m = regex.search(name) + if m and (kind, int(m.group(1))) in canonical_by_id: + return kind, int(m.group(1)) + return None + + +def _ext_key(name: str) -> str: + """``name`` with an image extension lower-cased, so ``.PNG`` and ``.png`` meet.""" + stem, ext = os.path.splitext(name) + return stem + ext.lower() if ext.lower() in ASSET_IMAGE_EXTS else name + + +def _flat_folders(name: str, flat: Optional["re.Match[str]"]) -> List[str]: + """Every folder name ``name`` can be the placement of: itself, or a flat file's stem cut at each ``_`` of its suffix.""" + if flat is None: + return [name] + stem = name[: name.rindex(".")] + return [ + stem[:i] + for i in range(flat.start(), len(stem) + 1) + if i == len(stem) or stem[i] == "_" + ] + + +def _current_copies( + asset_dir: str, folder: str, suffix: str, names: Dict[str, List[str]] +) -> List[str]: + """Paths of ``folder``'s placement on disk: the folder, or its flat file under any image extension.""" + if suffix: + stem = folder + suffix.rsplit(".", 1)[0] + keys, is_copy = ( + [_ext_key(folder + suffix), *(stem + ext for ext in ASSET_IMAGE_EXTS)], + os.path.isfile, + ) + else: + keys, is_copy = [folder], os.path.isdir + # An unlisted key is still probed: a case-insensitive mount finds it under another case. + entries = dict.fromkeys(e for k in keys for e in names.get(k, [k])) + return [p for e in entries if is_copy(p := os.path.join(asset_dir, e))] + + +def _disk_size(path: str) -> int: + """Bytes in ``path``'s files, skipping links and hard-linked files, whose data another name may still hold.""" + files = ( + [os.path.join(r, f) for r, _d, fs in os.walk(path) for f in fs] + if os.path.isdir(path) and not os.path.islink(path) + else [path] + ) + size = 0 + for f in files: + try: + st = os.lstat(f) + except OSError: + continue # vanished mid-walk + if not stat.S_ISLNK(st.st_mode) and st.st_nlink == 1: + size += st.st_size + return size + + +def _unused_path(path: str, is_dir: bool) -> str: + """``path``, or the first free `` (n)`` beside it, so a move never replaces an earlier restore copy.""" + stem, ext = (path, "") if is_dir else os.path.splitext(path) + n = 0 + while os.path.lexists(path): + n += 1 + path = f"{stem} ({n}){ext}" + return path + + class PosterCleanarr(ChubModule): # Per-pass refusal tally behind _refuse/_report_refusals. Class level because # shim callers build the module with object.__new__, skipping __init__. @@ -1263,14 +1342,10 @@ def _build_library_id_sets( def _build_canonical_folder_map( self, db: ChubDB, instances: List[str] - ) -> Dict[Tuple[str, int], str]: - """Map ('tvdb'|'tmdb', id) -> the media item's canonical folder name - (media_cache.folder), scoped to `instances`. Only main (season-less) - rows define the folder, so the show-level folder isn't shadowed by a - season row. Used to tell a stale-duplicate asset folder (wrong name) - from the canonical one.""" + ) -> Dict[Tuple[str, int], Set[str]]: + """Map ('tvdb'|'tmdb', id) to every canonical folder name (media_cache.folder) its main rows in `instances` give it.""" wanted = set(instances) - out: Dict[Tuple[str, int], str] = {} + out: Dict[Tuple[str, int], Set[str]] = {} for row in db.media.get_all(): if row.get("instance_name") not in wanted: continue @@ -1285,7 +1360,7 @@ def _build_canonical_folder_map( ): try: if raw not in (None, "", 0): - out[(kind, int(raw))] = folder + out.setdefault((kind, int(raw)), set()).add(folder) except (ValueError, TypeError): continue return out @@ -1293,14 +1368,9 @@ def _build_canonical_folder_map( def _scan_stale_duplicates( self, asset_dirs: List[str], - canonical_by_id: Dict[Tuple[str, int], str], + canonical_by_id: Dict[Tuple[str, int], Set[str]], ) -> List[Dict[str, Any]]: - """Top-level asset folders whose {tvdb/tmdb} id matches a live media - item but whose name != that item's canonical folder. Spare-only on the - id: an id with no live match is left to the orphan pass, never flagged - here. `canonical_present` records whether the correctly-named folder is - already on disk — removal must keep the only copy (see - _execute_stale_mode).""" + """Stale-duplicate entries for the top-level folders and flat files in ``asset_dirs`` (see _stale_entry).""" out: List[Dict[str, Any]] = [] for asset_dir in asset_dirs: if not os.path.isdir(asset_dir): @@ -1309,52 +1379,71 @@ def _scan_stale_duplicates( entries = sorted(os.listdir(asset_dir)) except OSError: continue + names: Dict[str, List[str]] = {} + for n in entries: + names.setdefault(_ext_key(n), []).append(n) for name in entries: - full = os.path.join(asset_dir, name) - if not os.path.isdir(full): - continue - canonical = None - ident = None - mt = tmdb_id_regex.search(name) - mv = tvdb_id_regex.search(name) - if mv and ("tvdb", int(mv.group(1))) in canonical_by_id: - ident = ("tvdb", int(mv.group(1))) - elif mt and ("tmdb", int(mt.group(1))) in canonical_by_id: - ident = ("tmdb", int(mt.group(1))) - if ident is None: - continue # no live id match -> orphan-pass territory, not stale - canonical = canonical_by_id[ident] - if name == canonical: - continue # this IS the canonical folder - size = 0 - for r, _d, files in os.walk(full): - for f in files: - try: - size += os.path.getsize(os.path.join(r, f)) - except OSError: - pass # file vanished mid-walk — don't count it - out.append( - { - "folder": full, - "asset_dir": asset_dir, - "name": name, - "canonical": canonical, - "canonical_present": os.path.isdir( - os.path.join(asset_dir, canonical) - ), - "id": ident, - "size": size, - } - ) + entry = self._stale_entry(asset_dir, name, canonical_by_id, names) + if entry is not None: + out.append(entry) return out + @staticmethod + def _stale_entry( + asset_dir: str, + name: str, + canonical_by_id: Dict[Tuple[str, int], Set[str]], + names: Dict[str, List[str]], + ) -> Optional[Dict[str, Any]]: + """The stale-duplicate entry for one top-level name in ``asset_dir`` (listed in ``names`` by _ext_key), or None.""" + flat = _FLAT_SUFFIX.search(name) if name.lower().endswith(ASSET_IMAGE_EXTS) else None + ident = _live_id(name[: flat.start()] if flat else name, canonical_by_id) + if ident is None: + return None # no live id match -> orphan-pass territory, not stale + folders = canonical_by_id[ident] + full = os.path.join(asset_dir, name) + cuts = _flat_folders(name, flat) + if not folders.isdisjoint(cuts): + return None # a current placement, wherever its suffix starts + if not (os.path.isfile if flat else os.path.isdir)(full): + return None + # Each place the suffix can start names another asset: every one needs a current copy. + copies = { + cut: [ + (f, p) + for f in sorted(folders) + for p in _current_copies(asset_dir, f, name[len(cut) :], names) + ] + for cut in cuts + } + # A current name showing this entry (the same entry, or a link to or into it) keeps it. + shown = any( + removing_takes(full, p) for hits in copies.values() for _f, p in hits + ) + gap = next((c for c in cuts if not copies[c]), None) + present = not shown and gap is None + if present: + folder, path = copies[cuts[0]][0] + canonical = os.path.basename(path) + else: + folder = min(folders) + canonical = folder + name[len(gap or cuts[0]) :] + return { + "folder": full, + "asset_dir": asset_dir, + "name": name, + "canonical": canonical, + "canonical_folder": folder, + "canonical_present": present, + "is_dir": flat is None, + "id": ident, + "size": _disk_size(full), + } + def _execute_stale_mode( self, dupes: List[Dict[str, Any]], mode: str, logger: Logger ) -> Dict[str, Any]: - """report/move/remove stale-duplicate FOLDERS. move/remove are skipped - for an entry whose canonical folder is not yet on disk so the only - staged copy is never destroyed (the renamer recreates the canonical - folder on its next run, after which this dup is safe to drop).""" + """report/move/remove stale duplicates; move/remove keep one with no separate current copy on disk.""" count = 0 total_size = 0 touched: Set[str] = set() @@ -1368,8 +1457,8 @@ def _execute_stale_mode( continue if not d.get("canonical_present"): logger.info( - f" [STALE KEPT] {folder} — canonical '{d['canonical']}' " - "not staged yet; keeping the only copy" + f" [STALE KEPT] {folder} — no separate copy of " + f"'{d['canonical']}' on disk; keeping it" ) continue # Re-read per item, not once per batch: a root dropped from config @@ -1380,7 +1469,7 @@ def _execute_stale_mode( continue if mode == "move": dest_root = os.path.join(d["asset_dir"], ORPHAN_RESTORE_DIR_NAME) - dest = os.path.join(dest_root, d["name"]) + dest = _unused_path(os.path.join(dest_root, d["name"]), d["is_dir"]) if self._confined_target(folder, config, logger) is None: continue try: @@ -1395,7 +1484,8 @@ def _execute_stale_mode( total_size += size touched.add(d["asset_dir"]) elif mode == "remove": - if not self._rmtree_confined(folder, config, logger): + remove = self._rmtree_confined if d["is_dir"] else self._remove_confined + if not remove(folder, config, logger): continue logger.info(f" [STALE REMOVED] {folder}") count += 1 @@ -1460,7 +1550,7 @@ def _run_stale_pass( dupes = self._scan_stale_duplicates(valid_dirs, canonical) total = sum(d["size"] for d in dupes) logger.info( - f"Found {len(dupes)} stale-duplicate folder(s) ({format_bytes(total)})." + f"Found {len(dupes)} stale duplicate(s) ({format_bytes(total)})." ) return self._execute_stale_mode(dupes, mode, logger) @@ -1637,7 +1727,7 @@ def _execute_orphan_mode( if mode == "move": dest_root = os.path.join(item["asset_dir"], ORPHAN_RESTORE_DIR_NAME) rel = os.path.relpath(path, item["asset_dir"]) - dest = os.path.join(dest_root, rel) + dest = _unused_path(os.path.join(dest_root, rel), os.path.isdir(path)) if self._confined_target(path, config, logger) is None: continue try: @@ -1975,6 +2065,7 @@ def scan_kometa_assets( "plex_year": (match.get(d["id"]) or {}).get("year"), "name": d["name"], "canonical": d["canonical"], + "canonical_folder": d["canonical_folder"], "canonical_present": d["canonical_present"], "size": d["size"], "folder": d["folder"], diff --git a/backend/util/file_ops.py b/backend/util/file_ops.py index e88ffff2..4da5e651 100644 --- a/backend/util/file_ops.py +++ b/backend/util/file_ops.py @@ -112,16 +112,24 @@ def move_into_place(src: str, dest: str) -> Optional[OSError]: def _links_to(link: str, path: str) -> bool: - # A link resolving to the file at path (a carry under symlink): that file may be the only copy. - return os.path.islink(link) and os.path.realpath(link) == resolve_folder(path) + # A link resolving to path or into its tree (a carry under symlink): that file may be the only copy. + if not os.path.islink(link): + return False + target, root = os.path.realpath(link), resolve_folder(path) + return target == root or target.startswith(root + os.sep) + + +def removing_takes(previous: str, current: str) -> bool: + """Whether removing ``previous`` would also take what ``current`` shows: one entry, a link to or into it, or a case-only twin.""" + if _same_entry(previous, current) or _links_to(current, previous): + return True + # A case-insensitive mount (SMB, macOS) sees a case-only rename as one entry. + return previous.casefold() == current.casefold() and _same_inode(previous, current) def remove_superseded(previous: Optional[str], current: str, root: str, logger: Any) -> None: """Remove the file placed at ``previous`` now that ``current`` replaces it, then its emptied folder; only inside ``root``.""" - if not previous or _same_entry(previous, current) or _links_to(current, previous): - return - # A case-insensitive mount (SMB, macOS) sees a case-only rename as one entry: the file just placed. - if previous.casefold() == current.casefold() and _same_inode(previous, current): + if not previous or removing_takes(previous, current): return try: real_base = os.path.realpath(root) diff --git a/frontend/src/pages/poster/PosterCleanarrPage.jsx b/frontend/src/pages/poster/PosterCleanarrPage.jsx index 27fb9287..7997face 100644 --- a/frontend/src/pages/poster/PosterCleanarrPage.jsx +++ b/frontend/src/pages/poster/PosterCleanarrPage.jsx @@ -155,16 +155,15 @@ async function readKometaAssets(force, token) { // --------------------------------------------------------------------------- // Stale-duplicate → bundle matching helpers // --------------------------------------------------------------------------- -// A stale asset folder is identified by its {tvdb/tmdb} id; the scan resolves -// that to a Plex rating_key + title/year via plex_media_cache. The cached -// rating_key frequently drifts from the live bundle scan (a re-add/re-match -// mints a new key), so we match by rating_key first, then fall back to the -// Plex title+year, then the folder's own parsed title+year. +// Match by rating_key, then Plex title+year, then the current folder's title+year: +// the cached rating_key drifts when Plex re-matches an item. const normTitle = t => (t || '').toLowerCase().replace(/[^a-z0-9]/g, ''); const titleYearKey = (title, year) => `${normTitle(title)}|${year ?? ''}`; -// Parse "Show Name (2019) {tvdb-123}" → { title: 'Show Name', year: 2019 }. +// Parse "Show Name (2019) {tvdb-123} Remastered" → { title: 'Show Name', year: 2019 }. const parseFolderTitleYear = name => { - const noId = (name || '').replace(/\s*\{[^}]*\}\s*$/, ''); + const noId = (name || '') + .replace(/\s*[{[](?:tmdb|tvdb|imdb)[^}\]]*[}\]].*$/i, '') + .replace(/(?:\s*(?:\{[^}]*\}|\[[^\]]*\]))+\s*$/, ''); const ym = noId.match(/\((\d{4})\)\s*$/); const year = ym ? Number(ym[1]) : null; const title = noId.replace(/\s*\(\d{4}\)\s*$/, '').trim(); @@ -734,8 +733,8 @@ const PosterCleanarrPage = () => { return m; }, [bundles]); - // Attribute each stale folder to a bundle: rating_key first, then Plex - // title+year, then the folder's own parsed title+year. Folders that match + // Attribute each stale entry to a bundle: rating_key first, then Plex + // title+year, then its current folder's parsed title+year. Entries that match // no bundle (no poster row exists, or the title isn't in Plex) are surfaced // separately so they aren't silently dropped. const { staleByBundle, unmatchedStale } = useMemo(() => { @@ -754,7 +753,9 @@ const PosterCleanarrPage = () => { let b = s.rating_key != null ? byRk.get(String(s.rating_key)) : null; if (!b && s.plex_title) b = byTY.get(titleYearKey(s.plex_title, s.plex_year)); if (!b) { - const { title, year } = parseFolderTitleYear(s.canonical || s.name); + const { title, year } = parseFolderTitleYear( + s.canonical_folder || s.canonical || s.name + ); b = byTY.get(titleYearKey(title, year)); } if (b) counts.set(b.bundle_path, (counts.get(b.bundle_path) || 0) + 1); @@ -1626,8 +1627,9 @@ const PosterCleanarrPage = () => { {unmatchedStale.length} - Renamed asset folders whose title isn't in the Plex bundle - scan — still cleaned by the Stale pass + Renamed asset folders and files whose title isn't in the + Plex bundle scan — the Stale pass still moves or removes each + once its current copy is in place
@@ -1975,7 +1977,7 @@ const BundleTreeRow = ({ {bloatCount > 0 && ● {bloatCount}} {staleCount > 0 && ( - + ⧉ {staleCount} )} diff --git a/frontend/src/pages/poster/PosterCleanarrPage.test.jsx b/frontend/src/pages/poster/PosterCleanarrPage.test.jsx index d940dc68..39ad23af 100644 --- a/frontend/src/pages/poster/PosterCleanarrPage.test.jsx +++ b/frontend/src/pages/poster/PosterCleanarrPage.test.jsx @@ -191,7 +191,7 @@ describe('Poster Cleanarr — stale folder attribution', () => { render(); // The pill renders "⧉ 1" only on the row that owns the stale folder. - expect(await screen.findByTitle('Stale duplicate asset folder')).toHaveTextContent('1'); + expect(await screen.findByTitle('Stale duplicate asset')).toHaveTextContent('1'); }); it('does not attribute an unmatchable stale folder to an arbitrary unanchored bundle', async () => { @@ -218,8 +218,46 @@ describe('Poster Cleanarr — stale folder attribution', () => { render(); await screen.findAllByText('(unknown)'); - expect(screen.queryByTitle('Stale duplicate asset folder')).not.toBeInTheDocument(); + expect(screen.queryByTitle('Stale duplicate asset')).not.toBeInTheDocument(); }); + + it.each(['Real Movie (2019) {tmdb-1} [x]', 'Real Movie (2019) {tmdb-1} Remastered'])( + 'attributes a stale flat file with no Plex match by its current folder name (%s)', + async canonicalFolder => { + const ANCHORED = { + ...GHOST_A, + bundle_path: '/plex/Metadata/Movies/r/real.bundle', + rating_key: 4242, + title: 'Real Movie', + year: 2019, + variants: [variant('/plex/.../real/only_real', 'only_real')], + }; + mockPostersAPI.scanKometaAssets.mockResolvedValue({ + data: { + stale: [ + { + rating_key: null, + plex_title: '', + plex_year: null, + name: 'Real Film (2019) {tmdb-1}_logo.png', + canonical: 'Real Movie (2019) {tmdb-1}_logo.png', + canonical_folder: canonicalFolder, + canonical_present: true, + size: 10, + folder: '/assets/Real Film (2019) {tmdb-1}_logo.png', + }, + ], + orphans: [], + }, + }); + scanPayload = payload([ANCHORED, GHOST_A]); + render(); + + const row = (await screen.findByText('Real Movie (2019)')).closest('.flex-1'); + expect(await within(row).findByTitle('Stale duplicate asset')).toHaveTextContent('1'); + expect(screen.getAllByTitle('Stale duplicate asset')).toHaveLength(1); + } + ); }); describe('Poster Cleanarr — persisted tree state', () => { diff --git a/frontend/src/utils/constants/settings_schema.js b/frontend/src/utils/constants/settings_schema.js index 6befeeaf..ea1f434f 100755 --- a/frontend/src/utils/constants/settings_schema.js +++ b/frontend/src/utils/constants/settings_schema.js @@ -1493,7 +1493,7 @@ const CORE_SETTINGS_SCHEMA = [ kind: 'pass', enableField: 'stale_duplicates_enabled', modeField: 'stale_duplicates_mode', - note: 'The canonical folder is always kept — the pass only acts on the stale, renamed copy.', + note: 'Current copies are always kept — the pass only acts on stale, renamed copies.', }, ], fields: [ @@ -1615,9 +1615,9 @@ const CORE_SETTINGS_SCHEMA = [ type: 'dirlist_dragdrop', section: 'sources', checkWritable: true, - description: 'Scanned recursively by both asset passes.', + description: 'Orphan walks each recursively; Stale reads only its top level.', helpText: - "Directories scanned by BOTH asset passes below — Orphan and Stale duplicates. Typically poster_renamerr's destination_dir, but you can list any path you want explicitly cleaned — including source dirs or personal folders that the post-rename pass deliberately leaves alone. Each is walked recursively; the hidden .chub_orphan_restore subdir is skipped.", + "Directories scanned by BOTH asset passes below — Orphan and Stale duplicates. Typically poster_renamerr's destination_dir, but you can list any path you want explicitly cleaned — including source dirs or personal folders that the post-rename pass deliberately leaves alone. The Orphan pass walks each recursively; the Stale pass reads only its top level, where Kometa's asset folders and flat files sit. The hidden .chub_orphan_restore subdir is skipped.", }, { key: 'orphan_instances', @@ -1628,7 +1628,7 @@ const CORE_SETTINGS_SCHEMA = [ section: 'sources', description: 'Radarr / Sonarr libraries that define which assets are legitimate.', helpText: - "Radarr/Sonarr instances whose libraries define which assets are legitimate — used by BOTH passes below. Orphan flags an asset when it matches none of these libraries (by {tmdb-N}/{tvdb-N} id or title); Stale uses them to resolve each item's canonical folder. Leave empty to fall back to the Plex Bloat 'Plex Instance(s)' selection. The comparison set is read from CHUB's media cache, populated by poster_renamerr — run it first if the cache is stale.", + "Radarr/Sonarr instances whose libraries define which assets are legitimate — used by BOTH passes below. Orphan flags an asset when it matches none of these libraries (by {tmdb-N}/{tvdb-N} id or title); Stale uses them to resolve each item's current folder names. Leave empty to fall back to the Plex Bloat 'Plex Instance(s)' selection. The comparison set is read from CHUB's media cache, populated by poster_renamerr — run it first if the cache is stale.", }, { key: 'orphan_assets_enabled', @@ -1679,9 +1679,10 @@ const CORE_SETTINGS_SCHEMA = [ type: 'check_box', sectionToggle: true, section: 'stale', - description: 'Remove duplicate asset folders left behind after a library rename.', + description: + 'Find asset folders and files left behind under an old name after a library rename.', helpText: - 'Scans the Asset Directories for Kometa asset folders whose {tmdb-N}/{tvdb-N} id matches a live item but whose folder name no longer matches the media folder (e.g. after a Sonarr/Radarr folder rename), using the Library Instances to resolve each item’s canonical folder. The canonical folder is always kept; a non-canonical duplicate is reported/moved/removed. Skipped if the canonical folder is not yet staged (never deletes the only copy).', + 'Scans the Asset Directories for Kometa asset folders, and flat-layout files named after a folder, whose {tmdb-N}/{tvdb-N} id matches a live item but whose folder name matches none of the item’s current folder names (one per selected Library Instance; e.g. after a Sonarr/Radarr folder rename). Current copies are always kept. Report lists every non-canonical duplicate; Move and Remove skip one whose current folder or file is not on disk yet, or is only a link to it. A stale folder goes whole, with any file its current folder lacks; Move keeps those in .chub_orphan_restore.', }, { key: 'stale_duplicates_mode', diff --git a/tests/test_poster_cleanarr_duplicates.py b/tests/test_poster_cleanarr_duplicates.py index db816362..ad73d61a 100644 --- a/tests/test_poster_cleanarr_duplicates.py +++ b/tests/test_poster_cleanarr_duplicates.py @@ -77,11 +77,11 @@ def test_canonical_folder_map_keys_by_id(db): m = _make() _seed(db, "sonarr", "Dune Prophecy (2024) {tvdb-367118}", tvdb=367118) _seed(db, "radarr", "Inception (2010) {tmdb-27205}", tmdb=27205) - # A season row must NOT define the canonical (folder is the same, season set): - _seed(db, "sonarr", "Dune Prophecy (2024) {tvdb-367118}", tvdb=367118, season=1) + # A season row must NOT add a canonical name: + _seed(db, "sonarr", "Dune Prophecy Season (2024) {tvdb-367118}", tvdb=367118, season=1) cmap = m._build_canonical_folder_map(db, ["sonarr", "radarr"]) - assert cmap[("tvdb", 367118)] == "Dune Prophecy (2024) {tvdb-367118}" - assert cmap[("tmdb", 27205)] == "Inception (2010) {tmdb-27205}" + assert cmap[("tvdb", 367118)] == {"Dune Prophecy (2024) {tvdb-367118}"} + assert cmap[("tmdb", 27205)] == {"Inception (2010) {tmdb-27205}"} def test_canonical_folder_map_respects_instances(db): @@ -100,7 +100,7 @@ def test_scan_stale_flags_wrong_named_folder(tmp_path): old.mkdir(parents=True) (old / "poster.jpg").write_bytes(b"x") (old / "Season01.jpg").write_bytes(b"yy") - cmap = {("tvdb", 367118): "Dune Prophecy (2024) {tvdb-367118}"} + cmap = {("tvdb", 367118): {"Dune Prophecy (2024) {tvdb-367118}"}} stale = m._scan_stale_duplicates([str(root)], cmap) assert len(stale) == 1 e = stale[0] @@ -118,7 +118,7 @@ def test_scan_stale_skips_canonical_folder(tmp_path): good = root / "Dune Prophecy (2024) {tvdb-367118}" good.mkdir(parents=True) (good / "poster.jpg").write_bytes(b"x") - cmap = {("tvdb", 367118): "Dune Prophecy (2024) {tvdb-367118}"} + cmap = {("tvdb", 367118): {"Dune Prophecy (2024) {tvdb-367118}"}} assert m._scan_stale_duplicates([str(root)], cmap) == [] @@ -128,7 +128,7 @@ def test_scan_stale_ignores_unknown_id(tmp_path): folder = root / "Some Show (2020) {tvdb-999}" folder.mkdir(parents=True) (folder / "poster.jpg").write_bytes(b"x") - assert m._scan_stale_duplicates([str(root)], {("tvdb", 1): "X"}) == [] + assert m._scan_stale_duplicates([str(root)], {("tvdb", 1): {"X"}}) == [] def test_scan_stale_marks_canonical_present(tmp_path): @@ -138,7 +138,7 @@ def test_scan_stale_marks_canonical_present(tmp_path): old = root / "Dune - Prophecy (2024) {tvdb-367118}" old.mkdir(parents=True) (old / "poster.jpg").write_bytes(b"x") - cmap = {("tvdb", 367118): "Dune Prophecy (2024) {tvdb-367118}"} + cmap = {("tvdb", 367118): {"Dune Prophecy (2024) {tvdb-367118}"}} stale = m._scan_stale_duplicates([str(root)], cmap) assert len(stale) == 1 assert stale[0]["canonical_present"] is True @@ -159,6 +159,7 @@ def test_execute_stale_remove_deletes_old_folder(tmp_path, monkeypatch): "name": old.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -184,6 +185,7 @@ def test_execute_stale_remove_keeps_only_copy(tmp_path, monkeypatch): "name": old.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": False, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -207,6 +209,7 @@ def test_execute_stale_report_deletes_nothing(tmp_path, monkeypatch): "name": old.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -236,6 +239,7 @@ def test_execute_stale_move_moves_folder(tmp_path, monkeypatch): "name": old.name, "canonical": canonical_name, "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -255,8 +259,8 @@ def test_scan_stale_both_ids_resolves_via_tvdb(db, tmp_path): tvdb_canonical = "Show (2024) {tvdb-1}" tmdb_canonical = "Show (2024) {tmdb-2}" cmap = { - ("tvdb", 1): tvdb_canonical, - ("tmdb", 2): tmdb_canonical, + ("tvdb", 1): {tvdb_canonical}, + ("tmdb", 2): {tmdb_canonical}, } root = tmp_path / "assets" # Folder name carries BOTH ids but a different name (stale for both) @@ -303,6 +307,236 @@ def test_run_stale_pass_reports(db, tmp_path, monkeypatch): assert res["count"] == 1 +_OLD = "Dune - Prophecy (2024) {tvdb-367118}" +_CANON = "Dune Prophecy (2024) {tvdb-367118}" + + +@pytest.mark.parametrize("mode", ["move", "remove"]) +def test_stale_pass_clears_flat_files_under_an_old_folder_name(db, tmp_path, monkeypatch, mode): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + untouched = [f"{_OLD}.nfo", f"{_OLD}_Season01.jpg", "No Id (2020).jpg", "Other (2020) {tvdb-9}.jpg"] + for name in [f"{_CANON}.jpg", f"{_CANON}_logo.png", f"{_OLD}.jpg", f"{_OLD}_logo.png", *untouched]: + (root / name).write_bytes(b"x") + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode=mode, logger=_logger()) + restore = root / ORPHAN_RESTORE_DIR_NAME + left = sorted(p.name for p in root.iterdir() if p != restore) + moved = sorted(p.name for p in restore.iterdir()) if restore.exists() else [] + assert (res["count"], left) == (2, sorted([f"{_CANON}.jpg", f"{_CANON}_logo.png", *untouched])) + assert moved == ([f"{_OLD}.jpg", f"{_OLD}_logo.png"] if mode == "move" else []) + + +def test_scan_stale_names_a_flat_files_copy_after_the_canonical_id_tag(tmp_path): + m = _make() + root = tmp_path / "assets" + root.mkdir() + canonical = "Dune Prophecy (2024) {tvdb-1} [x]" + (root / f"{canonical}_logo.png").write_bytes(b"x") + (root / f"{canonical}.jpg").write_bytes(b"x") + (root / "Dune (2024) {tvdb-1}_logo.png").write_bytes(b"x") + stale = m._scan_stale_duplicates([str(root)], {("tvdb", 1): {canonical}}) + assert [(e["name"], e["canonical"], e["canonical_folder"], e["canonical_present"]) for e in stale] == [ + ("Dune (2024) {tvdb-1}_logo.png", f"{canonical}_logo.png", canonical, True) + ] + + +def test_scan_stale_reads_the_folder_name_before_the_placement_suffix(tmp_path): + m = _make() + root = tmp_path / "assets" + root.mkdir() + canon = "Dune Prophecy (2024) {tvdb-1}" + old = "Dune (2024) {tvdb-1} {imdb-tt9}" + for name in [f"{canon}.png", f"{canon}_logo.png", f"{canon}_Season01_logo.png", f"{old}.jpg", f"{old}_logo.png"]: + (root / name).write_bytes(b"x") + stale = m._scan_stale_duplicates([str(root)], {("tvdb", 1): {canon}}) + assert [(e["name"], e["canonical"], e["canonical_present"]) for e in stale] == [ + (f"{old}.jpg", f"{canon}.png", True), + (f"{old}_logo.png", f"{canon}_logo.png", True), + ] + + +def test_stale_pass_keeps_each_instances_folder_for_one_id(db, tmp_path, monkeypatch): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + _seed(db, "sonarr-4k", _OLD, tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + for folder in (_CANON, _OLD): + _place(root / folder, True) + _place(root / f"{folder}.jpg", False) + (root / "Dune Gone (2024) {tvdb-367118}.jpg").write_bytes(b"x") + res = m._run_stale_pass(db=db, instances=["sonarr", "sonarr-4k"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + left = sorted(p.name for p in root.iterdir()) + assert (res["count"], left) == (1, sorted([_CANON, f"{_CANON}.jpg", _OLD, f"{_OLD}.jpg"])) + + +def test_scan_stale_finds_the_canonical_copy_under_an_upper_case_extension(tmp_path): + m = _make() + root = tmp_path / "assets" + root.mkdir() + for name in ("Dune (2024) {tvdb-1}.jpg", "Dune Prophecy (2024) {tvdb-1}.PNG"): + (root / name).write_bytes(b"x") + stale = m._scan_stale_duplicates([str(root)], {("tvdb", 1): {"Dune Prophecy (2024) {tvdb-1}"}}) + assert [(e["canonical"], e["canonical_present"]) for e in stale] == [("Dune Prophecy (2024) {tvdb-1}.PNG", True)] + + +def test_scan_stale_keeps_a_file_named_for_a_current_folder_that_ends_like_a_suffix(tmp_path): + m = _make() + root = tmp_path / "assets" + root.mkdir() + for name in ("Show {tvdb-1}_logo.jpg", "Renamed Show {tvdb-1}_logo.jpg"): + (root / name).write_bytes(b"x") + folders = {"Show {tvdb-1}_logo", "Renamed Show {tvdb-1}"} + assert m._scan_stale_duplicates([str(root)], {("tvdb", 1): folders}) == [] + + +@pytest.mark.parametrize("current_poster", [False, True]) +def test_scan_stale_needs_a_current_copy_for_each_asset_an_ambiguous_name_can_be(tmp_path, current_poster): + m = _make() + root = tmp_path / "assets" + root.mkdir() + names = ["Old Show {tvdb-1}_logo.jpg", "New Show {tvdb-1}_logo.png"] + ["New Show {tvdb-1}.jpg"] * current_poster + for name in names: + (root / name).write_bytes(b"x") + stale = m._scan_stale_duplicates([str(root)], {("tvdb", 1): {"New Show {tvdb-1}"}}) + want = "New Show {tvdb-1}_logo.png" if current_poster else "New Show {tvdb-1}.jpg" + assert [(e["name"], e["canonical"], e["canonical_present"]) for e in stale] == [ + ("Old Show {tvdb-1}_logo.jpg", want, current_poster) + ] + + +def _place(path, is_dir, data=b"x"): + """A flat file, or a folder holding poster.jpg, at ``path`` with ``data``.""" + if is_dir: + path.mkdir() + path = path / "poster.jpg" + path.write_bytes(data) + + +def _bytes(path): + """What ``path`` holds: a flat file's bytes, or its folder's poster.jpg.""" + return (path / "poster.jpg" if path.is_dir() else path).read_bytes() + + +@pytest.mark.parametrize("is_dir", [False, True]) +def test_stale_pass_keeps_the_copy_a_canonical_link_points_at(db, tmp_path, monkeypatch, is_dir): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + ext = "" if is_dir else ".jpg" + _place(root / f"{_OLD}{ext}", is_dir) + (root / f"{_CANON}{ext}").symlink_to(root / f"{_OLD}{ext}") + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + assert (res["count"], _bytes(root / f"{_CANON}{ext}")) == (0, b"x") + + +def test_stale_pass_keeps_a_case_only_rename_on_a_case_insensitive_mount(db, tmp_path, monkeypatch): + (tmp_path / "probe").touch() + if not (tmp_path / "PROBE").exists(): + pytest.skip("needs a case-insensitive filesystem") + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON.replace("Prophecy", "PROPHECY"), tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + _place(root / _CANON, True) + _place(root / f"{_CANON}.jpg", False) + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + assert (res["count"], sorted(p.name for p in root.iterdir())) == (0, [_CANON, f"{_CANON}.jpg"]) + + +@pytest.mark.parametrize("is_dir", [False, True]) +def test_stale_move_keeps_an_earlier_restore_copy_of_the_same_name(db, tmp_path, monkeypatch, is_dir): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + restore = root / ORPHAN_RESTORE_DIR_NAME + restore.mkdir(parents=True) + ext = "" if is_dir else ".jpg" + _place(restore / f"{_OLD}{ext}", is_dir, b"earlier") + _place(root / f"{_CANON}{ext}", is_dir) + _place(root / f"{_OLD}{ext}", is_dir, b"later") + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="move", logger=_logger()) + kept = {p.name: _bytes(p) for p in restore.iterdir()} + assert (res["count"], kept) == (1, {f"{_OLD}{ext}": b"earlier", f"{_OLD} (1){ext}": b"later"}) + + +def test_stale_pass_finds_a_current_folder_listed_under_another_case(db, tmp_path, monkeypatch): + (tmp_path / "probe").touch() + if not (tmp_path / "PROBE").exists(): + pytest.skip("needs a case-insensitive filesystem") + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + upper = _CANON.replace("Prophecy", "PROPHECY") + _place(root / upper, True) + _place(root / _OLD, True) + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + assert (res["count"], sorted(p.name for p in root.iterdir())) == (1, [upper]) + + +def test_stale_pass_keeps_a_folder_a_canonical_link_points_into(db, tmp_path, monkeypatch): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + (root / _OLD).mkdir(parents=True) + _place(root / _OLD / "art", True) + (root / _CANON).symlink_to(root / _OLD / "art") + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + assert (res["count"], (root / _CANON / "poster.jpg").exists()) == (0, True) + + +def test_stale_pass_keeps_stale_files_that_current_names_link_to(db, tmp_path, monkeypatch): + m = _make(tmp_path) + _live(monkeypatch, tmp_path) + _seed(db, "sonarr", _CANON, tvdb=367118) + root = tmp_path / "assets" + root.mkdir() + a, b = root / f"{_OLD}.jpg", root / "Dune (2024) {tvdb-367118}.jpg" + _place(a, False, b"a") + _place(b, False, b"b") + (root / f"{_CANON}.jpg").symlink_to(a) + (root / f"{_CANON}.png").symlink_to(b) + res = m._run_stale_pass(db=db, instances=["sonarr"], asset_dirs=[str(root)], mode="remove", logger=_logger()) + current = [(root / f"{_CANON}{ext}").exists() for ext in (".jpg", ".png")] + assert (res["count"], current) == (0, [True, True]) + + +@pytest.mark.parametrize( + "kind, size", [("file", 4), ("symlink", 0), ("hardlink", 0), ("folder", 4)] +) +def test_scan_stale_counts_only_bytes_no_other_name_holds(tmp_path, kind, size): + m = _make() + root = tmp_path / "assets" + root.mkdir() + source = tmp_path / "source.jpg" + source.write_bytes(b"1234") + old = root / ("Old Show {tvdb-1}" if kind == "folder" else "Old Show {tvdb-1}.jpg") + if kind == "file": + old.write_bytes(b"1234") + elif kind == "symlink": + old.symlink_to(source) + elif kind == "hardlink": + os.link(source, old) + else: + old.mkdir() + (old / "logo.png").write_bytes(b"1234") + (old / "poster.jpg").symlink_to(source) + os.link(source, old / "background.jpg") + stale = m._scan_stale_duplicates([str(root)], {("tvdb", 1): {"New Show {tvdb-1}"}}) + assert [(e["name"], e["size"]) for e in stale] == [(old.name, size)] + + def _seed_stale_pair(root, canonical="Dune Prophecy (2024) {tvdb-367118}"): """A stale-named folder plus its canonical sibling, so remove mode is allowed.""" (root / canonical).mkdir(parents=True) @@ -403,6 +637,7 @@ def _swap_parent_for_link(allowed, outside): "name": scanned.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -497,6 +732,7 @@ def _swapping_open(path, *a, **kw): "name": stale.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -599,6 +835,7 @@ def _recording_rmtree(path, *a, **kw): "name": stale.name, "canonical": "Dune Prophecy (2024) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } @@ -618,7 +855,9 @@ def _stale_entry(folder, asset_dir, canonical): "asset_dir": str(asset_dir), "name": os.path.basename(str(folder)), "canonical": canonical, + "canonical_folder": canonical, "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, } diff --git a/tests/test_poster_cleanarr_orphans.py b/tests/test_poster_cleanarr_orphans.py index 710c20f3..27082d6e 100644 --- a/tests/test_poster_cleanarr_orphans.py +++ b/tests/test_poster_cleanarr_orphans.py @@ -788,6 +788,34 @@ def _recording_rename(src, dst, **kw): assert src_fd is not None and dst_fd is not None # ...against both fds +def test_execute_orphan_mode_move_keeps_an_earlier_restore_copy(tmp_path, monkeypatch): + allowed = tmp_path / "allowed" + restore = allowed / ORPHAN_RESTORE_DIR_NAME + restore.mkdir(parents=True) + (restore / "orphan.png").write_bytes(b"earlier") + f = allowed / "orphan.png" + f.write_bytes(b"later") + m = _make(allowed) + _live(monkeypatch, allowed) + res = m._execute_orphan_mode([_orphan_item(f, allowed)], "move", _logger()) + kept = {p.name: p.read_bytes() for p in restore.iterdir()} + assert (res["count"], kept) == (1, {"orphan.png": b"earlier", "orphan (1).png": b"later"}) + + +def test_the_kometa_scan_names_each_stale_files_current_folder(db, tmp_path, monkeypatch, no_kometa_cache): + assets, _ = _asset_dir(db, tmp_path, monkeypatch) + db.media.execute_query( + "INSERT INTO media_cache (identity_key, instance_name, folder, tmdb_id, asset_type) VALUES (?,?,?,?,?)", + ("d1", "radarr1", "Dune (2021) {tmdb-1}", 1, "movie"), + ) + for name in ("Dune (2021) {tmdb-1}_logo.png", "Dune Part One (2021) {tmdb-1}_logo.png"): + (assets / name).write_bytes(b"x") + stale = scan_kometa_assets(db, _logger(), force=True)["stale"] + assert [(s["name"], s["canonical_folder"]) for s in stale] == [ + ("Dune Part One (2021) {tmdb-1}_logo.png", "Dune (2021) {tmdb-1}") + ] + + def test_execute_orphan_mode_move_refuses_a_parent_swapped_after_the_check( tmp_path, monkeypatch ): @@ -1157,7 +1185,9 @@ def test_stale_removal_drops_the_cached_scan(tmp_path, monkeypatch, no_kometa_ca "asset_dir": str(root), "name": old.name, "canonical": "Gone Show (2018) {tvdb-1}", + "canonical_folder": "Gone Show (2018) {tvdb-1}", "canonical_present": True, + "is_dir": True, "id": ("tvdb", 1), "size": 1, }