Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 76 additions & 47 deletions backend/modules/border_replacerr.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,10 @@ def convert_to_rgb(self, hex_color: str) -> Tuple[int, int, int]:
return color_code

@staticmethod
def _safe_stat(path: str):
def _safe_stat(path: str, follow_symlinks: bool = True):
"""os.stat(path) or None if it can't be stat'd."""
try:
return os.stat(path)
return os.stat(path, follow_symlinks=follow_symlinks)
except OSError:
return None

Expand All @@ -193,6 +193,46 @@ def _file_digest(path: str) -> str:
return ""
return h.hexdigest()

@staticmethod
def _file_stats(src: os.stat_result, out: os.stat_result) -> dict:
"""The source and output (mtime, size) a border_state row records."""
return {
"source_mtime": src.st_mtime,
"source_size": src.st_size,
"output_mtime": out.st_mtime,
"output_size": out.st_size,
}

def _unchanged_since(
self, prev: Optional[dict], sig: str, src: os.stat_result, renamed_file: str
) -> bool:
"""True when neither file changed since the last pass bordered them with ``sig``."""
if not prev or prev["variant_sig"] != sig:
return False
# lstat: a link (a symlink run's) is compared as itself, not as its target.
out = self._safe_stat(renamed_file, follow_symlinks=False)
if out is None:
return False
return all(prev[k] == v for k, v in self._file_stats(src, out).items())

def _gate_state(
self,
src: os.stat_result,
out: Optional[os.stat_result],
renamed_file: str,
sig: str,
now_iso: str,
) -> Optional[dict]:
"""The row that lets the next pass skip this poster; None without an output stat."""
if out is None:
return None
return {
"renamed_file": renamed_file,
**self._file_stats(src, out),
"variant_sig": sig,
"updated_at": now_iso,
}

def _save_if_changed(self, out_img: "Image.Image", renamed_file: str) -> bool:
"""Write ``out_img`` to ``renamed_file`` if the bytes changed or a link is there; True if written."""
try:
Expand Down Expand Up @@ -577,12 +617,8 @@ def run(
# and the list would just be every input.
changes: List[Dict[str, str]] = []

# Skip-gate state: the (mtime, size, settings-signature) of each
# output we last wrote. A poster whose output file is unchanged and
# whose signature still matches would re-encode to byte-identical
# bytes, so it's skipped entirely (no decode/resize/encode).
# Disabled on dry-run (we want to report what WOULD happen). Empty
# on first run, so the first pass processes everything as before.
# Skip-gate (see _unchanged_since); off on dry run, which reports what
# WOULD happen.
gate_enabled = not dry_run
state_map = db.border.get_all_map() if gate_enabled else {}

Expand All @@ -606,6 +642,9 @@ def _variant_sig(variant) -> str:
# means the worker holds no shared mutable index — the prerequisite
# for concurrency.
work: List[Tuple[dict, object, str]] = []
# Stat'd before the encode reads it: a source replaced mid-pass is
# bordered again next run.
source_stats: Dict[str, os.stat_result] = {}
variant_index = 0
for asset in assets:
if not asset.get("original_file") or not asset.get("renamed_file"):
Expand All @@ -614,7 +653,8 @@ def _variant_sig(variant) -> str:
)
skipped += 1
continue
if not os.path.exists(asset["original_file"]):
src = self._safe_stat(asset["original_file"])
if src is None:
# Full-library sweeps re-encounter posters whose source was
# since removed (gdrive cleanup, one-time drops). Skip them
# quietly instead of letting the encoder fail and tallying a
Expand All @@ -633,26 +673,14 @@ def _variant_sig(variant) -> str:
variant = None
variant_index += 1
sig = _variant_sig(variant)
original_file = asset["original_file"]
renamed_file = asset["renamed_file"]
# Skip only a regular output whose source and settings match the last pass;
# a missing output or a link (a symlink run's) is bordered again.
if (
gate_enabled
and os.path.isfile(renamed_file)
and not os.path.islink(renamed_file)
if gate_enabled and self._unchanged_since(
state_map.get(renamed_file), sig, src, renamed_file
):
prev = state_map.get(renamed_file)
if prev and prev["variant_sig"] == sig:
st = self._safe_stat(original_file)
if (
st is not None
and prev["source_mtime"] == st.st_mtime
and prev["source_size"] == st.st_size
):
gate_skipped += 1
processed += 1
continue
gate_skipped += 1
processed += 1
continue
source_stats[renamed_file] = src
work.append((asset, variant, sig))

attempted = len(work)
Expand All @@ -665,7 +693,7 @@ def _variant_sig(variant) -> str:

def _process(item: Tuple[dict, object, str]):
"""Apply the active border op to one poster. Returns
(result, title, action, detail, original_file, renamed_file, sig);
(result, title, action, detail, output_stat, renamed_file, sig);
result is True (written), False (unchanged) or None (failed)."""
asset, variant, sig = item
original_file = asset["original_file"]
Expand All @@ -684,7 +712,7 @@ def _process(item: Tuple[dict, object, str]):
self.logger.debug(
f"[DRY RUN] Would composite {detail} onto: {renamed_file}"
)
return True, title, action, detail, original_file, renamed_file, sig
return True, title, action, detail, None, renamed_file, sig
result = self.replace_borders_with_image(
original_file, renamed_file, variant, target_size
)
Expand All @@ -694,7 +722,7 @@ def _process(item: Tuple[dict, object, str]):
self.logger.debug(
f"[DRY RUN] Would replace border for: {renamed_file}"
)
return True, title, action, detail, original_file, renamed_file, sig
return True, title, action, detail, None, renamed_file, sig
result = self.replace_borders(
original_file,
renamed_file,
Expand All @@ -708,14 +736,17 @@ def _process(item: Tuple[dict, object, str]):
self.logger.debug(
f"[DRY RUN] Would remove border for: {renamed_file}"
)
return True, title, action, detail, original_file, renamed_file, sig
return True, title, action, detail, None, renamed_file, sig
result = self.remove_borders(
original_file,
renamed_file,
self.config.border_width,
target_size,
)
return result, title, action, detail, original_file, renamed_file, sig
# lstat now, not when the result is read: pool.map can hold a result
# behind slower posters while another writer replaces the output.
out = self._safe_stat(renamed_file, follow_symlinks=False)
return result, title, action, detail, out, renamed_file, sig

write_refusals: List[PosterWriteRefused] = []

Expand All @@ -727,7 +758,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]):
self.logger.debug(str(e))
write_refusals.append(e)
asset, _variant, sig = item
return None, asset["title"], "", "", asset["original_file"], asset["renamed_file"], sig
return None, asset["title"], "", "", None, asset["renamed_file"], sig

# PIL releases the GIL during decode/resize/composite/encode, so a
# thread pool gives real parallelism on the CPU-bound encode without
Expand All @@ -744,8 +775,8 @@ def _process_or_refuse(item: Tuple[dict, object, str]):
)
workers = max(1, min(workers, len(work) or 1))

# Refresh each processed asset's gate state (mtime/size of the SOURCE
# file + the signature it was made with) so the next run can skip it.
# Refresh each processed asset's gate state (mtime/size of the source and
# output files + the signature it was made with) so the next run can skip it.
# Keyed by the output path (stable per-asset identity). Recorded for
# written (True) and unchanged (False) results, not failures (None) —
# failures should retry next run. Timestamp computed once.
Expand All @@ -770,7 +801,7 @@ def _process_or_refuse(item: Tuple[dict, object, str]):
title,
action,
detail,
original_file,
output_stat,
renamed_file,
sig,
) in pool.map(_process_or_refuse, work):
Expand All @@ -791,17 +822,15 @@ def _process_or_refuse(item: Tuple[dict, object, str]):
elif result is None:
failed += 1
if gate_enabled and result is not None:
st = self._safe_stat(original_file)
if st is not None:
new_states.append(
{
"renamed_file": renamed_file,
"source_mtime": st.st_mtime,
"source_size": st.st_size,
"variant_sig": sig,
"updated_at": now_iso,
}
)
state = self._gate_state(
source_stats[renamed_file],
output_stat,
renamed_file,
sig,
now_iso,
)
if state:
new_states.append(state)
processed += 1
done += 1
# Jobs-page bar; no-op without job context, and maps into
Expand Down
36 changes: 14 additions & 22 deletions backend/util/database/border_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,14 @@


class BorderState(DatabaseBase):
"""Interface for the border_state table.

Tracks, per output poster, the (mtime, size) of the file border_replacerr
last wrote and a signature of the border settings used. On the next run
a poster is skipped entirely (no decode/resize/encode) when its output
file is unchanged AND the signature it would be processed with matches —
i.e. the result would be byte-identical to what's already on disk.

Keying on the OUTPUT file (not the source) makes this robust whether the
border is applied in place or to a separate path: any upstream re-upload
that replaces the poster bumps its mtime/size and invalidates the gate.
"""
"""Per output poster: its source's and its own (mtime, size) and the border settings at its last pass."""

def get_all_map(self) -> dict:
"""Return {renamed_file: {source_mtime, source_size, variant_sig}}."""
"""Return {renamed_file: {source/output mtime and size, variant_sig}}."""
rows = (
self.execute_query(
"SELECT renamed_file, source_mtime, source_size, variant_sig "
"FROM border_state",
"SELECT renamed_file, source_mtime, source_size, output_mtime, "
"output_size, variant_sig FROM border_state",
fetch_all=True,
)
or []
Expand All @@ -29,24 +18,25 @@ def get_all_map(self) -> dict:
r["renamed_file"]: {
"source_mtime": r["source_mtime"],
"source_size": r["source_size"],
"output_mtime": r["output_mtime"],
"output_size": r["output_size"],
"variant_sig": r["variant_sig"],
}
for r in rows
}

def bulk_record(self, records: list, chunk_size: int = 500) -> int:
"""Upsert many border-state rows, one transaction per chunk.

records: dicts with renamed_file, source_mtime, source_size,
variant_sig, updated_at.
"""
"""Upsert border_state rows (dicts keyed by its columns), one transaction per chunk."""
sql = """
INSERT INTO border_state
(renamed_file, source_mtime, source_size, variant_sig, updated_at)
VALUES (?, ?, ?, ?, ?)
(renamed_file, source_mtime, source_size, output_mtime,
output_size, variant_sig, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(renamed_file) DO UPDATE SET
source_mtime=excluded.source_mtime,
source_size=excluded.source_size,
output_mtime=excluded.output_mtime,
output_size=excluded.output_size,
variant_sig=excluded.variant_sig,
updated_at=excluded.updated_at
"""
Expand All @@ -60,6 +50,8 @@ def bulk_record(self, records: list, chunk_size: int = 500) -> int:
rec["renamed_file"],
rec["source_mtime"],
rec["source_size"],
rec["output_mtime"],
rec["output_size"],
rec["variant_sig"],
rec["updated_at"],
),
Expand Down
2 changes: 2 additions & 0 deletions backend/util/database/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,8 @@ def _define_schema(self) -> None:
),
ColumnDefinition("source_mtime", "REAL"),
ColumnDefinition("source_size", "INTEGER"),
ColumnDefinition("output_mtime", "REAL"),
ColumnDefinition("output_size", "INTEGER"),
ColumnDefinition("variant_sig", "TEXT"),
ColumnDefinition("updated_at", "TEXT"),
],
Expand Down
Loading
Loading