From b436b07e7561e94a5b725f0072dc775fdf37f471 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:54:46 +0800 Subject: [PATCH 1/4] fix(borders): skip a poster only while its bordered output is still on disk The skip gate recorded the source's mtime and size only, so a poster restaged unbordered while borders were off stayed unbordered once they were back on. border_state now records the output's mtime and size too; a row from before these columns re-borders its poster once. --- backend/modules/border_replacerr.py | 81 ++++++++++------ backend/util/database/border_state.py | 29 +++--- backend/util/database/schema.py | 2 + tests/test_border_replacerr.py | 130 ++++++++++++++++++++++++-- 4 files changed, 193 insertions(+), 49 deletions(-) diff --git a/backend/modules/border_replacerr.py b/backend/modules/border_replacerr.py index a1734c5f1..4946921ce 100755 --- a/backend/modules/border_replacerr.py +++ b/backend/modules/border_replacerr.py @@ -193,6 +193,41 @@ def _file_digest(path: str) -> str: return "" return h.hexdigest() + def _unchanged_since( + self, prev: Optional[dict], sig: str, original_file: str, renamed_file: str + ) -> bool: + """True when the last pass wrote this output from this source with ``sig``, and neither changed.""" + # A link (a symlink run's) or a file restaged over the output is bordered again. + if not prev or prev["variant_sig"] != sig or os.path.islink(renamed_file): + return False + src, out = self._safe_stat(original_file), self._safe_stat(renamed_file) + if src is None or out is None: + return False + recorded = ( + prev["source_mtime"], + prev["source_size"], + prev["output_mtime"], + prev["output_size"], + ) + return recorded == (src.st_mtime, src.st_size, out.st_mtime, out.st_size) + + def _gate_state( + self, original_file: str, renamed_file: str, sig: str, now_iso: str + ) -> Optional[dict]: + """The border_state row that lets the next pass skip this poster, if both stat.""" + src, out = self._safe_stat(original_file), self._safe_stat(renamed_file) + if src is None or out is None: + return None + return { + "renamed_file": renamed_file, + "source_mtime": src.st_mtime, + "source_size": src.st_size, + "output_mtime": out.st_mtime, + "output_size": out.st_size, + "variant_sig": sig, + "updated_at": now_iso, + } + def _save_if_changed(self, out_img: "Image.Image", renamed_file: str) -> bool: """Write ``out_img`` to ``renamed_file`` if the bytes changed or a link is there; True if written.""" try: @@ -633,26 +668,16 @@ def _variant_sig(variant) -> str: variant = None variant_index += 1 sig = _variant_sig(variant) - original_file = asset["original_file"] renamed_file = asset["renamed_file"] - # Skip only a regular output whose source and settings match the last pass; - # a missing output or a link (a symlink run's) is bordered again. - if ( - gate_enabled - and os.path.isfile(renamed_file) - and not os.path.islink(renamed_file) + if gate_enabled and self._unchanged_since( + state_map.get(renamed_file), + sig, + asset["original_file"], + renamed_file, ): - prev = state_map.get(renamed_file) - if prev and prev["variant_sig"] == sig: - st = self._safe_stat(original_file) - if ( - st is not None - and prev["source_mtime"] == st.st_mtime - and prev["source_size"] == st.st_size - ): - gate_skipped += 1 - processed += 1 - continue + gate_skipped += 1 + processed += 1 + continue work.append((asset, variant, sig)) attempted = len(work) @@ -744,8 +769,8 @@ def _process_or_refuse(item: Tuple[dict, object, str]): ) workers = max(1, min(workers, len(work) or 1)) - # Refresh each processed asset's gate state (mtime/size of the SOURCE - # file + the signature it was made with) so the next run can skip it. + # Refresh each processed asset's gate state (mtime/size of the source and + # output files + the signature it was made with) so the next run can skip it. # Keyed by the output path (stable per-asset identity). Recorded for # written (True) and unchanged (False) results, not failures (None) — # failures should retry next run. Timestamp computed once. @@ -791,17 +816,11 @@ def _process_or_refuse(item: Tuple[dict, object, str]): elif result is None: failed += 1 if gate_enabled and result is not None: - st = self._safe_stat(original_file) - if st is not None: - new_states.append( - { - "renamed_file": renamed_file, - "source_mtime": st.st_mtime, - "source_size": st.st_size, - "variant_sig": sig, - "updated_at": now_iso, - } - ) + state = self._gate_state( + original_file, renamed_file, sig, now_iso + ) + if state: + new_states.append(state) processed += 1 done += 1 # Jobs-page bar; no-op without job context, and maps into diff --git a/backend/util/database/border_state.py b/backend/util/database/border_state.py index 4c42e91da..6e1511416 100644 --- a/backend/util/database/border_state.py +++ b/backend/util/database/border_state.py @@ -4,11 +4,11 @@ class BorderState(DatabaseBase): """Interface for the border_state table. - Tracks, per output poster, the (mtime, size) of the file border_replacerr - last wrote and a signature of the border settings used. On the next run - a poster is skipped entirely (no decode/resize/encode) when its output - file is unchanged AND the signature it would be processed with matches — - i.e. the result would be byte-identical to what's already on disk. + Tracks, per output poster, the (mtime, size) of its source and of the file + border_replacerr last wrote, and a signature of the border settings used. + On the next run a poster is skipped entirely (no decode/resize/encode) when + both files are unchanged AND the signature it would be processed with + matches — i.e. the result would be byte-identical to what's already on disk. Keying on the OUTPUT file (not the source) makes this robust whether the border is applied in place or to a separate path: any upstream re-upload @@ -16,11 +16,11 @@ class BorderState(DatabaseBase): """ def get_all_map(self) -> dict: - """Return {renamed_file: {source_mtime, source_size, variant_sig}}.""" + """Return {renamed_file: {source/output mtime and size, variant_sig}}.""" rows = ( self.execute_query( - "SELECT renamed_file, source_mtime, source_size, variant_sig " - "FROM border_state", + "SELECT renamed_file, source_mtime, source_size, output_mtime, " + "output_size, variant_sig FROM border_state", fetch_all=True, ) or [] @@ -29,6 +29,8 @@ def get_all_map(self) -> dict: r["renamed_file"]: { "source_mtime": r["source_mtime"], "source_size": r["source_size"], + "output_mtime": r["output_mtime"], + "output_size": r["output_size"], "variant_sig": r["variant_sig"], } for r in rows @@ -38,15 +40,18 @@ def bulk_record(self, records: list, chunk_size: int = 500) -> int: """Upsert many border-state rows, one transaction per chunk. records: dicts with renamed_file, source_mtime, source_size, - variant_sig, updated_at. + output_mtime, output_size, variant_sig, updated_at. """ sql = """ INSERT INTO border_state - (renamed_file, source_mtime, source_size, variant_sig, updated_at) - VALUES (?, ?, ?, ?, ?) + (renamed_file, source_mtime, source_size, output_mtime, + output_size, variant_sig, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(renamed_file) DO UPDATE SET source_mtime=excluded.source_mtime, source_size=excluded.source_size, + output_mtime=excluded.output_mtime, + output_size=excluded.output_size, variant_sig=excluded.variant_sig, updated_at=excluded.updated_at """ @@ -60,6 +65,8 @@ def bulk_record(self, records: list, chunk_size: int = 500) -> int: rec["renamed_file"], rec["source_mtime"], rec["source_size"], + rec["output_mtime"], + rec["output_size"], rec["variant_sig"], rec["updated_at"], ), diff --git a/backend/util/database/schema.py b/backend/util/database/schema.py index 7cdb84056..b3966400b 100755 --- a/backend/util/database/schema.py +++ b/backend/util/database/schema.py @@ -551,6 +551,8 @@ def _define_schema(self) -> None: ), ColumnDefinition("source_mtime", "REAL"), ColumnDefinition("source_size", "INTEGER"), + ColumnDefinition("output_mtime", "REAL"), + ColumnDefinition("output_size", "INTEGER"), ColumnDefinition("variant_sig", "TEXT"), ColumnDefinition("updated_at", "TEXT"), ], diff --git a/tests/test_border_replacerr.py b/tests/test_border_replacerr.py index ca965aa25..19980b123 100644 --- a/tests/test_border_replacerr.py +++ b/tests/test_border_replacerr.py @@ -198,6 +198,8 @@ def test_border_state_roundtrip_and_upsert(tmp_path): "renamed_file": "/m/a.jpg", "source_mtime": 1.5, "source_size": 10, + "output_mtime": 3.5, + "output_size": 30, "variant_sig": "color:8:(255, 0, 0)", "updated_at": "t0", }, @@ -205,6 +207,8 @@ def test_border_state_roundtrip_and_upsert(tmp_path): "renamed_file": "/m/b.jpg", "source_mtime": 2.0, "source_size": 20, + "output_mtime": 4.0, + "output_size": 40, "variant_sig": "remove:8", "updated_at": "t0", }, @@ -213,8 +217,13 @@ def test_border_state_roundtrip_and_upsert(tmp_path): assert n == 2 m = bs.get_all_map() assert set(m) == {"/m/a.jpg", "/m/b.jpg"} - assert m["/m/a.jpg"]["source_size"] == 10 - assert m["/m/a.jpg"]["variant_sig"] == "color:8:(255, 0, 0)" + assert m["/m/a.jpg"] == { + "source_mtime": 1.5, + "source_size": 10, + "output_mtime": 3.5, + "output_size": 30, + "variant_sig": "color:8:(255, 0, 0)", + } # PK conflict updates in place. bs.bulk_record( @@ -223,14 +232,21 @@ def test_border_state_roundtrip_and_upsert(tmp_path): "renamed_file": "/m/a.jpg", "source_mtime": 9.0, "source_size": 99, + "output_mtime": 8.0, + "output_size": 88, "variant_sig": "x", "updated_at": "t1", } ] ) m2 = bs.get_all_map() - assert m2["/m/a.jpg"]["source_size"] == 99 - assert m2["/m/a.jpg"]["variant_sig"] == "x" + assert m2["/m/a.jpg"] == { + "source_mtime": 9.0, + "source_size": 99, + "output_mtime": 8.0, + "output_size": 88, + "variant_sig": "x", + } bs.clear() assert bs.get_all_map() == {} @@ -763,15 +779,19 @@ def test_a_bordered_poster_a_link_replaced_is_bordered_again(tmp_path, monkeypat assert (dest.is_symlink(), dest.read_bytes() == bordered) == (False, True) -def test_a_link_to_the_bordered_bytes_is_replaced_by_a_file(tmp_path, monkeypatch): +@pytest.mark.parametrize("moved", [False, True], ids=["copied", "moved"]) +def test_a_link_to_the_bordered_bytes_is_replaced_by_a_file(tmp_path, monkeypatch, moved): br, dest_dir, by_id = _subset_harness(tmp_path, monkeypatch) manifest = {"media_cache": [1], "collections_cache": []} br.run(manifest, process_all=False) dest = dest_dir / "inman.jpg" twin = tmp_path / "twin.jpg" bordered = dest.read_bytes() - twin.write_bytes(bordered) - dest.unlink() + if moved: + dest.rename(twin) + else: + twin.write_bytes(bordered) + dest.unlink() dest.symlink_to(twin) br.run(manifest, process_all=False) assert (dest.is_symlink(), dest.read_bytes(), twin.read_bytes()) == ( @@ -1123,3 +1143,99 @@ def test_dry_run_ignores_a_destination_it_cannot_write(tmp_path, monkeypatch): "Border replacerr completed: 1 processed, 1 replaced, 0 skipped." in br.logger.messages["info"] ) + + +@pytest.mark.parametrize("restage", ["copy_into_place", "link_into_place"]) +def test_a_poster_restaged_unbordered_is_bordered_again(tmp_path, monkeypatch, restage): + """Borders off restages the raw source over the output; back on, it is bordered again.""" + from backend.util import file_ops + + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + br.run(process_all=True) + bordered = dest.read_bytes() + getattr(file_ops, restage)(str(tmp_path / "src" / "0.jpg"), str(dest)) + encodes.clear() + br.run(process_all=True) + assert (len(encodes), dest.read_bytes() == bordered) == (1, True) + br.run(process_all=True) + assert len(encodes) == 1, "the re-bordered poster was encoded again" + + +def test_a_row_recorded_before_output_tracking_borders_its_poster_again( + tmp_path, monkeypatch +): + """A border_state row from before the output columns cannot vouch for the output.""" + import sqlite3 + + from backend.util.file_ops import copy_into_place + + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + source = tmp_path / "src" / "0.jpg" + dest.parent.mkdir() + copy_into_place(str(source), str(dest)) + st = source.stat() + with sqlite3.connect(tmp_path / "b.db") as conn: + conn.execute( + "CREATE TABLE border_state (renamed_file TEXT PRIMARY KEY NOT NULL, " + "source_mtime REAL, source_size INTEGER, variant_sig TEXT, updated_at TEXT)" + ) + conn.execute( + "INSERT INTO border_state VALUES (?, ?, ?, ?, ?)", + (str(dest), st.st_mtime, st.st_size, "color:26:(255, 0, 0)", "t0"), + ) + conn.close() + br.run(process_all=True) + assert (len(encodes), dest.read_bytes() == source.read_bytes()) == (1, False) + + +def _pixel(path, xy) -> tuple: + with Image.open(path) as im: + return im.convert("RGB").getpixel(xy) + + +def test_a_border_colour_change_borders_the_poster_again(tmp_path, monkeypatch): + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + br.run(process_all=True) + br.config.border_colors = ["#00FF00"] + br.run(process_all=True) + r, g, _b = _pixel(dest, (5, 750)) + assert (len(encodes), g > 200 and r < 60) == (2, True) + + +@pytest.mark.parametrize( + "size, keep_mtime", + [((1000, 1500), False), ((900, 1350), True)], + ids=["same-size", "same-mtime"], +) +def test_a_replaced_source_is_bordered_again(tmp_path, monkeypatch, size, keep_mtime): + """A new source is bordered again when only its size or only its mtime differs.""" + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + source = tmp_path / "src" / "0.jpg" + Image.new("RGB", (1000, 1500), (200, 0, 0)).save(source, "BMP") + br.run(process_all=True) + old = source.stat() + Image.new("RGB", size, (0, 0, 200)).save(source, "BMP") + mtime = old.st_mtime_ns if keep_mtime else old.st_mtime_ns + 10**10 + os.utime(source, ns=(old.st_atime_ns, mtime)) + assert (source.stat().st_size == old.st_size) == (not keep_mtime) + br.run(process_all=True) + r, _g, b = _pixel(dest, (500, 750)) + assert (len(encodes), b > 150 and r < 60) == (2, True) + + +@pytest.mark.parametrize("same_size", [True, False], ids=["same-size", "same-mtime"]) +def test_an_output_changed_since_is_bordered_again(tmp_path, monkeypatch, same_size): + """A changed output is bordered again when only its size or only its mtime differs.""" + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + br.run(process_all=True) + bordered, old = dest.read_bytes(), dest.stat() + dest.write_bytes(bordered[:-1] + b"\0" if same_size else bordered + b"\0") + mtime = old.st_mtime_ns + 10**10 if same_size else old.st_mtime_ns + os.utime(dest, ns=(old.st_atime_ns, mtime)) + br.run(process_all=True) + assert (len(encodes), dest.read_bytes() == bordered) == (2, True) From 2a1a7a1bdf2bf58c1152cff128e3de29acf82da3 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:02:04 +0800 Subject: [PATCH 2/4] fix(borders): record the source stat taken before the encode; lstat the output A source replaced while its poster was being bordered was recorded with the old output, so the next pass skipped it. The output is compared by lstat, so a link is judged as itself; the islink check goes. --- backend/modules/border_replacerr.py | 71 +++++++++++++++-------------- tests/test_border_replacerr.py | 21 +++++++++ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/backend/modules/border_replacerr.py b/backend/modules/border_replacerr.py index 4946921ce..aca956e28 100755 --- a/backend/modules/border_replacerr.py +++ b/backend/modules/border_replacerr.py @@ -174,10 +174,10 @@ def convert_to_rgb(self, hex_color: str) -> Tuple[int, int, int]: return color_code @staticmethod - def _safe_stat(path: str): + def _safe_stat(path: str, follow_symlinks: bool = True): """os.stat(path) or None if it can't be stat'd.""" try: - return os.stat(path) + return os.stat(path, follow_symlinks=follow_symlinks) except OSError: return None @@ -193,37 +193,38 @@ def _file_digest(path: str) -> str: return "" return h.hexdigest() + @staticmethod + def _file_stats(src: os.stat_result, out: os.stat_result) -> dict: + """The source and output (mtime, size) a border_state row records.""" + return { + "source_mtime": src.st_mtime, + "source_size": src.st_size, + "output_mtime": out.st_mtime, + "output_size": out.st_size, + } + def _unchanged_since( - self, prev: Optional[dict], sig: str, original_file: str, renamed_file: str + self, prev: Optional[dict], sig: str, src: os.stat_result, renamed_file: str ) -> bool: - """True when the last pass wrote this output from this source with ``sig``, and neither changed.""" - # A link (a symlink run's) or a file restaged over the output is bordered again. - if not prev or prev["variant_sig"] != sig or os.path.islink(renamed_file): + """True when neither file changed since the last pass bordered them with ``sig``.""" + if not prev or prev["variant_sig"] != sig: return False - src, out = self._safe_stat(original_file), self._safe_stat(renamed_file) - if src is None or out is None: + # lstat: a link (a symlink run's) is compared as itself, not as its target. + out = self._safe_stat(renamed_file, follow_symlinks=False) + if out is None: return False - recorded = ( - prev["source_mtime"], - prev["source_size"], - prev["output_mtime"], - prev["output_size"], - ) - return recorded == (src.st_mtime, src.st_size, out.st_mtime, out.st_size) + return all(prev[k] == v for k, v in self._file_stats(src, out).items()) def _gate_state( - self, original_file: str, renamed_file: str, sig: str, now_iso: str + self, src: os.stat_result, renamed_file: str, sig: str, now_iso: str ) -> Optional[dict]: - """The border_state row that lets the next pass skip this poster, if both stat.""" - src, out = self._safe_stat(original_file), self._safe_stat(renamed_file) - if src is None or out is None: + """The row that lets the next pass skip this poster; None if the output is gone.""" + out = self._safe_stat(renamed_file, follow_symlinks=False) + if out is None: return None return { "renamed_file": renamed_file, - "source_mtime": src.st_mtime, - "source_size": src.st_size, - "output_mtime": out.st_mtime, - "output_size": out.st_size, + **self._file_stats(src, out), "variant_sig": sig, "updated_at": now_iso, } @@ -612,10 +613,10 @@ def run( # and the list would just be every input. changes: List[Dict[str, str]] = [] - # Skip-gate state: the (mtime, size, settings-signature) of each - # output we last wrote. A poster whose output file is unchanged and - # whose signature still matches would re-encode to byte-identical - # bytes, so it's skipped entirely (no decode/resize/encode). + # Skip-gate state: the source and output (mtime, size) and settings + # signature of each poster we last bordered. A poster whose files are + # unchanged and whose signature still matches would re-encode to + # byte-identical bytes, so it's skipped entirely (no decode/resize/encode). # Disabled on dry-run (we want to report what WOULD happen). Empty # on first run, so the first pass processes everything as before. gate_enabled = not dry_run @@ -641,6 +642,9 @@ def _variant_sig(variant) -> str: # means the worker holds no shared mutable index — the prerequisite # for concurrency. work: List[Tuple[dict, object, str]] = [] + # Stat'd before the encode reads it: a source replaced mid-pass is + # bordered again next run. + source_stats: Dict[str, os.stat_result] = {} variant_index = 0 for asset in assets: if not asset.get("original_file") or not asset.get("renamed_file"): @@ -649,7 +653,8 @@ def _variant_sig(variant) -> str: ) skipped += 1 continue - if not os.path.exists(asset["original_file"]): + src = self._safe_stat(asset["original_file"]) + if src is None: # Full-library sweeps re-encounter posters whose source was # since removed (gdrive cleanup, one-time drops). Skip them # quietly instead of letting the encoder fail and tallying a @@ -670,14 +675,12 @@ def _variant_sig(variant) -> str: sig = _variant_sig(variant) renamed_file = asset["renamed_file"] if gate_enabled and self._unchanged_since( - state_map.get(renamed_file), - sig, - asset["original_file"], - renamed_file, + state_map.get(renamed_file), sig, src, renamed_file ): gate_skipped += 1 processed += 1 continue + source_stats[renamed_file] = src work.append((asset, variant, sig)) attempted = len(work) @@ -795,7 +798,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]): title, action, detail, - original_file, + _original_file, renamed_file, sig, ) in pool.map(_process_or_refuse, work): @@ -817,7 +820,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]): failed += 1 if gate_enabled and result is not None: state = self._gate_state( - original_file, renamed_file, sig, now_iso + source_stats[renamed_file], renamed_file, sig, now_iso ) if state: new_states.append(state) diff --git a/tests/test_border_replacerr.py b/tests/test_border_replacerr.py index 19980b123..9325752eb 100644 --- a/tests/test_border_replacerr.py +++ b/tests/test_border_replacerr.py @@ -1239,3 +1239,24 @@ def test_an_output_changed_since_is_bordered_again(tmp_path, monkeypatch, same_s os.utime(dest, ns=(old.st_atime_ns, mtime)) br.run(process_all=True) assert (len(encodes), dest.read_bytes() == bordered) == (2, True) + + +def test_a_source_replaced_during_the_pass_is_bordered_again(tmp_path, monkeypatch): + """A source synced in while its poster was being bordered is bordered next run.""" + dest = tmp_path / "kometa" / "poster.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [dest]) + source = tmp_path / "src" / "0.jpg" + Image.new("RGB", (1000, 1500), (200, 0, 0)).save(source, "BMP") + encode = br.replace_borders + + def _encode_then_sync(*a, **k): + result = encode(*a, **k) + Image.new("RGB", (900, 1350), (0, 0, 200)).save(source, "BMP") + return result + + br.replace_borders = _encode_then_sync + br.run(process_all=True) + br.replace_borders = encode + br.run(process_all=True) + r, _g, b = _pixel(dest, (500, 750)) + assert (len(encodes), b > 150 and r < 60) == (2, True) From 210b3b0211a06f1be53deef23b472886389487dd Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:02:37 +0800 Subject: [PATCH 3/4] docs(borders): one-line border_state docstrings and gate comment --- backend/modules/border_replacerr.py | 8 ++------ backend/util/database/border_state.py | 19 ++----------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/backend/modules/border_replacerr.py b/backend/modules/border_replacerr.py index aca956e28..56f7ee4d4 100755 --- a/backend/modules/border_replacerr.py +++ b/backend/modules/border_replacerr.py @@ -613,12 +613,8 @@ def run( # and the list would just be every input. changes: List[Dict[str, str]] = [] - # Skip-gate state: the source and output (mtime, size) and settings - # signature of each poster we last bordered. A poster whose files are - # unchanged and whose signature still matches would re-encode to - # byte-identical bytes, so it's skipped entirely (no decode/resize/encode). - # Disabled on dry-run (we want to report what WOULD happen). Empty - # on first run, so the first pass processes everything as before. + # Skip-gate (see _unchanged_since); off on dry run, which reports what + # WOULD happen. gate_enabled = not dry_run state_map = db.border.get_all_map() if gate_enabled else {} diff --git a/backend/util/database/border_state.py b/backend/util/database/border_state.py index 6e1511416..e816a9978 100644 --- a/backend/util/database/border_state.py +++ b/backend/util/database/border_state.py @@ -2,18 +2,7 @@ class BorderState(DatabaseBase): - """Interface for the border_state table. - - Tracks, per output poster, the (mtime, size) of its source and of the file - border_replacerr last wrote, and a signature of the border settings used. - On the next run a poster is skipped entirely (no decode/resize/encode) when - both files are unchanged AND the signature it would be processed with - matches — i.e. the result would be byte-identical to what's already on disk. - - Keying on the OUTPUT file (not the source) makes this robust whether the - border is applied in place or to a separate path: any upstream re-upload - that replaces the poster bumps its mtime/size and invalidates the gate. - """ + """Per output poster: its source's and its own (mtime, size) and the border settings at its last pass.""" def get_all_map(self) -> dict: """Return {renamed_file: {source/output mtime and size, variant_sig}}.""" @@ -37,11 +26,7 @@ def get_all_map(self) -> dict: } def bulk_record(self, records: list, chunk_size: int = 500) -> int: - """Upsert many border-state rows, one transaction per chunk. - - records: dicts with renamed_file, source_mtime, source_size, - output_mtime, output_size, variant_sig, updated_at. - """ + """Upsert border_state rows (dicts keyed by its columns), one transaction per chunk.""" sql = """ INSERT INTO border_state (renamed_file, source_mtime, source_size, output_mtime, From 46bb15990201592e6ff1b65f3ea93f5ab4f2812d Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:38:25 +0800 Subject: [PATCH 4/4] fix(borders): take the output's stat in the worker right after its write pool.map hands results back in order, so a result could wait behind slower posters while another writer replaced the output; the stat taken then was recorded as the bordered file. --- backend/modules/border_replacerr.py | 33 +++++++++++++++-------- tests/test_border_replacerr.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/backend/modules/border_replacerr.py b/backend/modules/border_replacerr.py index 56f7ee4d4..3787a19cd 100755 --- a/backend/modules/border_replacerr.py +++ b/backend/modules/border_replacerr.py @@ -216,10 +216,14 @@ def _unchanged_since( return all(prev[k] == v for k, v in self._file_stats(src, out).items()) def _gate_state( - self, src: os.stat_result, renamed_file: str, sig: str, now_iso: str + self, + src: os.stat_result, + out: Optional[os.stat_result], + renamed_file: str, + sig: str, + now_iso: str, ) -> Optional[dict]: - """The row that lets the next pass skip this poster; None if the output is gone.""" - out = self._safe_stat(renamed_file, follow_symlinks=False) + """The row that lets the next pass skip this poster; None without an output stat.""" if out is None: return None return { @@ -689,7 +693,7 @@ def _variant_sig(variant) -> str: def _process(item: Tuple[dict, object, str]): """Apply the active border op to one poster. Returns - (result, title, action, detail, original_file, renamed_file, sig); + (result, title, action, detail, output_stat, renamed_file, sig); result is True (written), False (unchanged) or None (failed).""" asset, variant, sig = item original_file = asset["original_file"] @@ -708,7 +712,7 @@ def _process(item: Tuple[dict, object, str]): self.logger.debug( f"[DRY RUN] Would composite {detail} onto: {renamed_file}" ) - return True, title, action, detail, original_file, renamed_file, sig + return True, title, action, detail, None, renamed_file, sig result = self.replace_borders_with_image( original_file, renamed_file, variant, target_size ) @@ -718,7 +722,7 @@ def _process(item: Tuple[dict, object, str]): self.logger.debug( f"[DRY RUN] Would replace border for: {renamed_file}" ) - return True, title, action, detail, original_file, renamed_file, sig + return True, title, action, detail, None, renamed_file, sig result = self.replace_borders( original_file, renamed_file, @@ -732,14 +736,17 @@ def _process(item: Tuple[dict, object, str]): self.logger.debug( f"[DRY RUN] Would remove border for: {renamed_file}" ) - return True, title, action, detail, original_file, renamed_file, sig + return True, title, action, detail, None, renamed_file, sig result = self.remove_borders( original_file, renamed_file, self.config.border_width, target_size, ) - return result, title, action, detail, original_file, renamed_file, sig + # lstat now, not when the result is read: pool.map can hold a result + # behind slower posters while another writer replaces the output. + out = self._safe_stat(renamed_file, follow_symlinks=False) + return result, title, action, detail, out, renamed_file, sig write_refusals: List[PosterWriteRefused] = [] @@ -751,7 +758,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]): self.logger.debug(str(e)) write_refusals.append(e) asset, _variant, sig = item - return None, asset["title"], "", "", asset["original_file"], asset["renamed_file"], sig + return None, asset["title"], "", "", None, asset["renamed_file"], sig # PIL releases the GIL during decode/resize/composite/encode, so a # thread pool gives real parallelism on the CPU-bound encode without @@ -794,7 +801,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]): title, action, detail, - _original_file, + output_stat, renamed_file, sig, ) in pool.map(_process_or_refuse, work): @@ -816,7 +823,11 @@ def _process_or_refuse(item: Tuple[dict, object, str]): failed += 1 if gate_enabled and result is not None: state = self._gate_state( - source_stats[renamed_file], renamed_file, sig, now_iso + source_stats[renamed_file], + output_stat, + renamed_file, + sig, + now_iso, ) if state: new_states.append(state) diff --git a/tests/test_border_replacerr.py b/tests/test_border_replacerr.py index 9325752eb..213afd9fe 100644 --- a/tests/test_border_replacerr.py +++ b/tests/test_border_replacerr.py @@ -1260,3 +1260,45 @@ def _encode_then_sync(*a, **k): br.run(process_all=True) r, _g, b = _pixel(dest, (500, 750)) assert (len(encodes), b > 150 and r < 60) == (2, True) + + +class _RanAheadPool: + """A pool whose workers all finish before the loop reads the first result.""" + + def __init__(self, max_workers): + self.max_workers = max_workers + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def map(self, fn, items): + return [fn(item) for item in items] + + +def test_an_output_replaced_before_its_result_is_read_is_bordered_again( + tmp_path, monkeypatch +): + """A restage between a border write and the read of its result is caught next run.""" + from backend.modules import border_replacerr as border_mod + from backend.util.file_ops import copy_into_place + + first, second = tmp_path / "kometa" / "a.jpg", tmp_path / "kometa" / "b.jpg" + br, encodes, _ = _colour_run(tmp_path, monkeypatch, [first, second]) + monkeypatch.setattr(border_mod, "ThreadPoolExecutor", _RanAheadPool) + source = tmp_path / "src" / "0.jpg" + encode = br.replace_borders + + def _encode_then_restage_first(*a, **k): + result = encode(*a, **k) + if a[1] == str(second): + copy_into_place(str(source), str(first)) + return result + + br.replace_borders = _encode_then_restage_first + br.run(process_all=True) + br.replace_borders = encode + br.run(process_all=True) + assert (len(encodes), first.read_bytes() == source.read_bytes()) == (3, False)