Refresh selected same-size files and configure upload exclusions

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [0.5.0] - 2026-10-06
+
+### Added
+- Optional `overwrite` file patterns on `upload`, `sync` and `plan_upload` copy selected directory files even when sizes match.
+  Exclusions take precedence. Downloads and deletion settings retain their existing behavior.
+- Optional `upload_exclude` on `sync` overrides upload exclusions without changing the initial download.
+  `None` inherits `exclude`; an empty list clears upload exclusions.
+  With `delete_remote=True`, remote objects matched by `exclude` but not by `upload_exclude` are deleted unless a local file exists at that path.
+  Use `delete_remote=False` to preserve these objects. For a later upload of recording markers, close the first mirror and use a new mirror with `upload(..., interval=None, delete=False)`.
+
 ## [0.4.0] - 2026-09-24
 
 ### Added

README.md

@@ -87,7 +87,7 @@ Registers a path for download. Ensures local copy matches S3 immediately.
 
 **Returns**: `pathlib.Path` to the local directory/file.
 
-### `pos3.upload(remote, local=None, interval=300, delete=True, sync_on_error=False, exclude=None)`
+### `pos3.upload(remote, local=None, interval=300, delete=True, sync_on_error=False, exclude=None, profile=None, overwrite=None)`
 
 Registers a local path for upload. Uploads on exit and optionally in background.
 - `remote`: Destination S3 URL.
@@ -98,11 +98,26 @@ Registers a local path for upload. Uploads on exit and optionally in background.
 
 **Returns**: `pathlib.Path` to the local directory/file.
 
-### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None)`
+Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. This selection applies to file paths relative to a directory source, or the local basename of a single-file source. Patterns use `PurePosixPath.match`, which matches from the right. A directory name does not select its contents. For example, a local `journal` file matches `"*"` or `"journal"`, even when uploaded to a different remote key. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers.
+
+### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None, upload_exclude=None)`
 
 Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint.
 - `delete_local`: Cleanup local files during download.
 - `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history.
+- `overwrite`: Select local files for forced upload using directory-relative paths or a single-file basename. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative.
+- `upload_exclude`: Override exclusions for uploads only. `None` uses `exclude` for both directions. An empty list clears upload exclusions. The initial download always uses `exclude`. With `delete_remote=True`, remote objects matched by `exclude` but not by `upload_exclude` are deleted unless a local file exists at that path. Use `delete_remote=False` to preserve remote objects when narrowing or clearing upload exclusions.
+
+Use `upload_exclude=[".unfinished"]` with `delete_remote=False` to download existing recording markers while withholding new open markers from periodic uploads. After recording and the first mirror context have stopped, a one-shot `upload` in a new `mirror()` context can include markers that remain. Set `delete=False` on this upload to preserve remote objects absent locally.
+
+```python
+with pos3.mirror():
+    local = pos3.sync(remote, local, delete_remote=False, upload_exclude=[".unfinished"])
+    # Run and stop recording before leaving this context.
+
+with pos3.mirror():
+    pos3.upload(remote, local, interval=None, delete=False)
+```
 
 **Returns**: `pathlib.Path` to the local directory/file.
 

pos3/__init__.py

@@ -193,7 +193,13 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non
             yield info
 
 
-def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]:
+def _compute_sync_diff(
+    source: Iterator[FileInfo],
+    target: Iterator[FileInfo],
+    overwrite: list[str] | None = None,
+    *,
+    source_name: str = ".",
+) -> tuple[list[FileInfo], list[FileInfo]]:
     source_map: dict[str, FileInfo] = {info.relative_path: info for info in source}
     target_map: dict[str, FileInfo] = {info.relative_path: info for info in target}
 
@@ -207,7 +213,10 @@ def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -
         elif source_info.is_dir != target_info.is_dir:
             to_delete.append(target_info)
             to_copy.append(source_info)
-        elif not source_info.is_dir and source_info.size != target_info.size:
+        elif not source_info.is_dir and (
+            source_info.size != target_info.size
+            or any(PurePosixPath(relative_path or source_name).match(pattern) for pattern in overwrite or [])
+        ):
             to_copy.append(source_info)
 
     for relative_path, target_info in target_map.items():
@@ -263,6 +272,7 @@ class _UploadRegistration:
     sync_on_error: bool
     exclude: list[str] | None
     profile: Profile | None = None
+    overwrite: list[str] | None = None
     last_sync: float = 0.0
     # The user's original URL, preserving trailing-slash intent. _sync_uploads
     # parses it raw so `s3://bucket/data/` skips head_object('data') in
@@ -281,6 +291,7 @@ def __eq__(self, other):
             and self.sync_on_error == other.sync_on_error
             and self.exclude == other.exclude
             and self.profile == other.profile
+            and self.overwrite == other.overwrite
         )
 
 
@@ -440,6 +451,7 @@ def upload(
         sync_on_error,
         exclude: list[str] | None = None,
         profile: str | Profile | None = None,
+        overwrite: list[str] | None = None,
     ) -> Path:
         """
         Register (and perform if needed) an upload from a local directory or file to a remote S3 bucket path.
@@ -485,6 +497,7 @@ def upload(
             profile=effective_profile,
             last_sync=0,
             raw_remote=remote,
+            overwrite=None if overwrite is None else list(overwrite),
         )
 
         with self._lock:
@@ -557,6 +570,7 @@ def plan_upload(
         local: str | Path | None = None,
         exclude: list[str] | None = None,
         profile: str | Profile | None = None,
+        overwrite: list[str] | None = None,
     ) -> TransferPlan:
         """Compute the :class:`TransferPlan` ``upload()`` would execute.
 
@@ -589,6 +603,8 @@ def plan_upload(
         to_copy, to_delete = _compute_sync_diff(
             _filter_fileinfo(_scan_local(source), exclude),
             _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, effective_profile), exclude),
+            overwrite=overwrite,
+            source_name=source.name,
         )
         copies: list[tuple[str, str]] = []
         for info in to_copy:
@@ -615,6 +631,8 @@ def sync(
         sync_on_error: bool,
         exclude: list[str] | None = None,
         profile: str | Profile | None = None,
+        overwrite: list[str] | None = None,
+        upload_exclude: list[str] | None = None,
     ) -> Path:
         # Let download() and upload() handle profile resolution and normalization
         local_path = self.download(remote, local, delete_local, exclude, profile)
@@ -625,7 +643,16 @@ def sync(
         effective_profile = self._effective_profile(profile, remote)
         # Unregister the download to allow upload registration for the same remote
         self._downloads.pop((normalized, effective_profile), None)
-        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile)
+        return self.upload(
+            remote,
+            local_path,
+            interval,
+            delete_remote,
+            sync_on_error,
+            exclude if upload_exclude is None else upload_exclude,
+            profile,
+            overwrite,
+        )
 
     def ls(self, prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
         """Lists objects under the given prefix, working for both local directories and S3 prefixes."""
@@ -757,7 +784,7 @@ def _final_sync(self, had_error: bool = False) -> None:
                 self._sync_uploads(uploads)
 
     def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
-        tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None]] = []
+        tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None, list[str] | None]] = []
         for registration in registrations:
             if registration.local_path.exists():
                 tasks.append(
@@ -771,6 +798,7 @@ def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
                         registration.delete,
                         registration.exclude,
                         registration.profile,
+                        registration.overwrite,
                     )
                 )
 
@@ -781,13 +809,15 @@ def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
         to_remove: list[tuple[str, str, Profile | None]] = []
         total_bytes = 0
 
-        for remote, local_path, delete, exclude, profile in tasks:
+        for remote, local_path, delete, exclude, profile, overwrite in tasks:
             logger.debug("Syncing upload: %s from %s (delete=%s)", remote, local_path, delete)
             scan_bucket, scan_prefix = _parse_s3_url(remote)
             bucket, out_prefix = _parse_s3_url(_normalize_s3_url(remote))
             to_copy, to_delete = _compute_sync_diff(
                 _filter_fileinfo(_scan_local(local_path), exclude),
                 _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, profile), exclude),
+                overwrite=overwrite,
+                source_name=local_path.name,
             )
 
             for info in to_copy:
@@ -1151,6 +1181,7 @@ def upload(
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
     profile: str | Profile | None = None,
+    overwrite: list[str] | None = None,
 ) -> Path:
     """
     Register a local path for upload. Uploads on exit and optionally in background.
@@ -1162,12 +1193,13 @@ def upload(
         delete: If True (default), deletes S3 files NOT present locally.
         sync_on_error: If True, syncs even if the context exits with an exception.
         profile: S3 profile name or Profile config for custom endpoints.
+        overwrite: Relative-path or single-file basename patterns copied even when sizes match.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile)
+    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile, overwrite)
 
 
 def sync(
@@ -1179,20 +1211,26 @@ def sync(
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
     profile: str | Profile | None = None,
+    overwrite: list[str] | None = None,
+    upload_exclude: list[str] | None = None,
 ) -> Path:
     """
     Bi-directional helper. Performs download() then registers upload().
 
     Args:
         delete_local: Cleanup local files during download.
         delete_remote: Cleanup remote files during upload.
+        overwrite: Relative-path or single-file basename patterns copied even when sizes match.
+        upload_exclude: Upload-only exclusions. None inherits exclude; an empty list clears upload exclusions.
         profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile)
+    return mirror_obj.sync(
+        remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile, overwrite, upload_exclude
+    )
 
 
 def ls(prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
@@ -1232,6 +1270,7 @@ def plan_upload(
     local: str | Path | None = None,
     exclude: list[str] | None = None,
     profile: str | Profile | None = None,
+    overwrite: list[str] | None = None,
 ) -> TransferPlan:
     """Return the :class:`TransferPlan` ``upload()`` would execute.
 
@@ -1242,7 +1281,7 @@ def plan_upload(
     skips registrations with a missing local_path).
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.plan_upload(remote, local, exclude, profile)
+    return mirror_obj.plan_upload(remote, local, exclude, profile, overwrite)
 
 
 __all__ = [

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pos3"
-version = "0.4.0"
+version = "0.5.0"
 description = "S3 Simple Sync - Make using S3 as simple as using local files"
 readme = "README.md"
 requires-python = ">=3.11"