From 4b807ae048321ffd2afe35c1ffbafbae40907740 Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 21:06:30 -0700 Subject: [PATCH 1/9] Test selected same-size metadata overwrite and unchanged defaults --- tests/test_overwrite.py | 89 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/test_overwrite.py diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py new file mode 100644 index 0000000..e4a220d --- /dev/null +++ b/tests/test_overwrite.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import pytest +from botocore.exceptions import ClientError + +import pos3 + + +@pytest.fixture +def storage(monkeypatch): + class Storage: + def __init__(self): + self.objects = {} + self.uploads = [] + self.deletes = [] + + def head_object(self, *, Bucket, Key): + if (Bucket, Key) not in self.objects: + raise ClientError({"Error": {"Code": "404"}}, "HeadObject") + return {"ContentLength": len(self.objects[Bucket, Key])} + + def get_paginator(self, operation): + assert operation == "list_objects_v2" + return self + + def paginate(self, *, Bucket, Prefix): + yield {"Contents": [{"Key": key, "Size": len(body)} for (bucket, key), body in self.objects.items() + if bucket == Bucket and key.startswith(Prefix)]} + + def upload_file(self, filename, bucket, key, *, Callback): + body = Path(filename).read_bytes() + self.objects[bucket, key] = body + self.uploads.append(key) + Callback(len(body)) + + def put_object(self, *, Bucket, Key, Body): + self.objects[Bucket, Key] = Body + + def download_file(self, bucket, key, filename, *, Callback): + body = self.objects[bucket, key] + Path(filename).write_bytes(body) + Callback(len(body)) + + def delete_object(self, *, Bucket, Key): + self.deletes.append(Key) + del self.objects[Bucket, Key] + + result = Storage() + monkeypatch.setattr(pos3._Mirror, "_get_client", lambda self, profile=None: result) + return result + + +@pytest.mark.parametrize("method", ["upload", "sync"]) +@pytest.mark.parametrize("patterns", [None, [], [".recording-attempts.sqlite", "run_metadata_*.yaml"]]) +def test_selected_same_size_files_are_overwritten_without_deleting_remote_objects(tmp_path, storage, method, patterns): + names = [".recording-attempts.sqlite", "run_metadata_1.yaml", "episode.parquet"] + for name in names: + (tmp_path / name).write_bytes(b"new") + storage.objects["bucket", "run/" + name] = b"old" + storage.objects["bucket", "run/remote-only"] = b"keep" + with pos3.mirror(show_progress=False): + plan = pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=patterns) + kwargs = {"delete": False} if method == "upload" else {"delete_remote": False} + getattr(pos3, method)("s3://bucket/run/", tmp_path, interval=None, overwrite=patterns, **kwargs) + expected = names[:2] if patterns else [] + assert {remote for _, remote in plan.to_copy} == {"s3://bucket/run/" + name for name in expected} + assert storage.uploads == [] + assert set(storage.uploads) == {"run/" + name for name in expected} + assert storage.deletes == [] + assert storage.objects["bucket", "run/remote-only"] == b"keep" + for name in names: + assert storage.objects["bucket", "run/" + name] == (b"new" if name in expected else b"old") + with pos3.mirror(show_progress=False): + readback = pos3.download("s3://bucket/run/", local=tmp_path / "readback") + for name in names: + assert (readback / name).read_bytes() == (b"new" if name in expected else b"old") + + +def test_overwrite_selection_survives_registration_replay_and_respects_exclusion(tmp_path, storage): + (tmp_path / "journal").write_bytes(b"new") + storage.objects["bucket", "run/journal"] = b"old" + with pos3.mirror(show_progress=False): + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"]) + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"]) + with pytest.raises(ValueError, match="different parameters"): + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=[]) + assert pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=["journal"], exclude=["journal"]).to_copy == [] + assert storage.uploads == [] + assert storage.objects["bucket", "run/journal"] == b"old" From 21caf0d22a31a7434ba68e7ae3b7071ab3cd5a53 Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 21:07:15 -0700 Subject: [PATCH 2/9] Allow selected metadata paths to overwrite same-size remote files --- README.md | 6 ++++-- pos3/__init__.py | 37 +++++++++++++++++++++++++++++-------- pyproject.toml | 2 +- tests/test_overwrite.py | 17 +++++++++++++---- uv.lock | 2 +- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8e74782..a0fcb30 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Registers a path for download. Ensures local copy matches S3 immediately. **Returns**: `pathlib.Path` to the local directory/file. -### `pos3.upload(remote, local=None, interval=300, delete=True, sync_on_error=False, exclude=None)` +### `pos3.upload(remote, local=None, interval=300, delete=True, sync_on_error=False, exclude=None, profile=None, overwrite=None)` Registers a local path for upload. Uploads on exit and optionally in background. - `remote`: Destination S3 URL. @@ -98,7 +98,9 @@ Registers a local path for upload. Uploads on exit and optionally in background. **Returns**: `pathlib.Path` to the local directory/file. -### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None)` +Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. Patterns use the same relative-path matching as `exclude`. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. + +### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None)` Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint. - `delete_local`: Cleanup local files during download. diff --git a/pos3/__init__.py b/pos3/__init__.py index 842e6d6..b45b1b8 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -193,7 +193,9 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non yield info -def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]: +def _compute_sync_diff( + source: Iterator[FileInfo], target: Iterator[FileInfo], overwrite: list[str] | None = None +) -> tuple[list[FileInfo], list[FileInfo]]: source_map: dict[str, FileInfo] = {info.relative_path: info for info in source} target_map: dict[str, FileInfo] = {info.relative_path: info for info in target} @@ -207,7 +209,10 @@ def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) - elif source_info.is_dir != target_info.is_dir: to_delete.append(target_info) to_copy.append(source_info) - elif not source_info.is_dir and source_info.size != target_info.size: + elif not source_info.is_dir and ( + source_info.size != target_info.size + or any(PurePosixPath(relative_path or ".").match(pattern) for pattern in overwrite or []) + ): to_copy.append(source_info) for relative_path, target_info in target_map.items(): @@ -263,6 +268,7 @@ class _UploadRegistration: sync_on_error: bool exclude: list[str] | None profile: Profile | None = None + overwrite: list[str] | None = None last_sync: float = 0.0 # The user's original URL, preserving trailing-slash intent. _sync_uploads # parses it raw so `s3://bucket/data/` skips head_object('data') in @@ -281,6 +287,7 @@ def __eq__(self, other): and self.sync_on_error == other.sync_on_error and self.exclude == other.exclude and self.profile == other.profile + and self.overwrite == other.overwrite ) @@ -440,6 +447,7 @@ def upload( sync_on_error, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> Path: """ Register (and perform if needed) an upload from a local directory or file to a remote S3 bucket path. @@ -485,6 +493,7 @@ def upload( profile=effective_profile, last_sync=0, raw_remote=remote, + overwrite=overwrite, ) with self._lock: @@ -557,6 +566,7 @@ def plan_upload( local: str | Path | None = None, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> TransferPlan: """Compute the :class:`TransferPlan` ``upload()`` would execute. @@ -589,6 +599,7 @@ def plan_upload( to_copy, to_delete = _compute_sync_diff( _filter_fileinfo(_scan_local(source), exclude), _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, effective_profile), exclude), + overwrite=overwrite, ) copies: list[tuple[str, str]] = [] for info in to_copy: @@ -615,6 +626,7 @@ def sync( sync_on_error: bool, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> Path: # Let download() and upload() handle profile resolution and normalization local_path = self.download(remote, local, delete_local, exclude, profile) @@ -625,7 +637,7 @@ def sync( effective_profile = self._effective_profile(profile, remote) # Unregister the download to allow upload registration for the same remote self._downloads.pop((normalized, effective_profile), None) - return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile) + return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile, overwrite) def ls(self, prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]: """Lists objects under the given prefix, working for both local directories and S3 prefixes.""" @@ -757,7 +769,7 @@ def _final_sync(self, had_error: bool = False) -> None: self._sync_uploads(uploads) def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None: - tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None]] = [] + tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None, list[str] | None]] = [] for registration in registrations: if registration.local_path.exists(): tasks.append( @@ -771,6 +783,7 @@ def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None: registration.delete, registration.exclude, registration.profile, + registration.overwrite, ) ) @@ -781,13 +794,14 @@ def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None: to_remove: list[tuple[str, str, Profile | None]] = [] total_bytes = 0 - for remote, local_path, delete, exclude, profile in tasks: + for remote, local_path, delete, exclude, profile, overwrite in tasks: logger.debug("Syncing upload: %s from %s (delete=%s)", remote, local_path, delete) scan_bucket, scan_prefix = _parse_s3_url(remote) bucket, out_prefix = _parse_s3_url(_normalize_s3_url(remote)) to_copy, to_delete = _compute_sync_diff( _filter_fileinfo(_scan_local(local_path), exclude), _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, profile), exclude), + overwrite=overwrite, ) for info in to_copy: @@ -1151,6 +1165,7 @@ def upload( sync_on_error: bool = False, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> Path: """ Register a local path for upload. Uploads on exit and optionally in background. @@ -1162,12 +1177,13 @@ def upload( delete: If True (default), deletes S3 files NOT present locally. sync_on_error: If True, syncs even if the context exits with an exception. profile: S3 profile name or Profile config for custom endpoints. + overwrite: Relative-path patterns copied on every upload, even when sizes match. Returns: Path to the local directory/file. """ mirror_obj = _require_active_mirror() - return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile) + return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile, overwrite) def sync( @@ -1179,6 +1195,7 @@ def sync( sync_on_error: bool = False, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> Path: """ Bi-directional helper. Performs download() then registers upload(). @@ -1186,13 +1203,16 @@ def sync( Args: delete_local: Cleanup local files during download. delete_remote: Cleanup remote files during upload. + overwrite: Relative-path patterns copied on every upload, even when sizes match. profile: S3 profile name or Profile config for custom endpoints. Returns: Path to the local directory/file. """ mirror_obj = _require_active_mirror() - return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile) + return mirror_obj.sync( + remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile, overwrite + ) def ls(prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]: @@ -1232,6 +1252,7 @@ def plan_upload( local: str | Path | None = None, exclude: list[str] | None = None, profile: str | Profile | None = None, + overwrite: list[str] | None = None, ) -> TransferPlan: """Return the :class:`TransferPlan` ``upload()`` would execute. @@ -1242,7 +1263,7 @@ def plan_upload( skips registrations with a missing local_path). """ mirror_obj = _require_active_mirror() - return mirror_obj.plan_upload(remote, local, exclude, profile) + return mirror_obj.plan_upload(remote, local, exclude, profile, overwrite) __all__ = [ diff --git a/pyproject.toml b/pyproject.toml index abfd7fe..87da497 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pos3" -version = "0.4.0" +version = "0.5.0" description = "S3 Simple Sync - Make using S3 as simple as using local files" readme = "README.md" requires-python = ">=3.11" diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index e4a220d..00a28b4 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -24,8 +24,13 @@ def get_paginator(self, operation): return self def paginate(self, *, Bucket, Prefix): - yield {"Contents": [{"Key": key, "Size": len(body)} for (bucket, key), body in self.objects.items() - if bucket == Bucket and key.startswith(Prefix)]} + yield { + "Contents": [ + {"Key": key, "Size": len(body)} + for (bucket, key), body in self.objects.items() + if bucket == Bucket and key.startswith(Prefix) + ] + } def upload_file(self, filename, bucket, key, *, Callback): body = Path(filename).read_bytes() @@ -80,8 +85,12 @@ def test_overwrite_selection_survives_registration_replay_and_respects_exclusion (tmp_path / "journal").write_bytes(b"new") storage.objects["bucket", "run/journal"] = b"old" with pos3.mirror(show_progress=False): - pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"]) - pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"]) + pos3.upload( + "s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"] + ) + pos3.upload( + "s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"] + ) with pytest.raises(ValueError, match="different parameters"): pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=[]) assert pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=["journal"], exclude=["journal"]).to_copy == [] diff --git a/uv.lock b/uv.lock index 2b40744..6f4692f 100644 --- a/uv.lock +++ b/uv.lock @@ -223,7 +223,7 @@ wheels = [ [[package]] name = "pos3" -version = "0.4.0" +version = "0.5.0" source = { editable = "." } dependencies = [ { name = "boto3" }, From dd4654abc2464be378e1ec5a8007ebfca1c621b1 Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 21:45:43 -0700 Subject: [PATCH 3/9] Document directory overwrite semantics and isolate replay controls. --- CHANGELOG.md | 6 ++++++ README.md | 3 ++- tests/test_overwrite.py | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e07e7f3..e4d9c2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.5.0] - 2026-10-06 + +### Added +- Optional `overwrite` file patterns on `upload`, `sync` and `plan_upload` copy selected directory files even when sizes match. + Exclusions take precedence. Downloads and deletion settings retain their existing behavior. + ## [0.4.0] - 2026-09-24 ### Added diff --git a/README.md b/README.md index a0fcb30..1f6b311 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,14 @@ Registers a local path for upload. Uploads on exit and optionally in background. **Returns**: `pathlib.Path` to the local directory/file. -Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. Patterns use the same relative-path matching as `exclude`. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. +Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. This selection applies to file paths relative to a directory source. Patterns use `PurePosixPath.match`, which matches from the right. A directory name does not select its contents. A single-file source has no relative file path, so overwrite patterns cannot select it. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. ### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None)` Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint. - `delete_local`: Cleanup local files during download. - `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history. +- `overwrite`: Select local directory files for forced upload. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative. **Returns**: `pathlib.Path` to the local directory/file. diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index 00a28b4..34e3059 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -92,7 +92,41 @@ def test_overwrite_selection_survives_registration_replay_and_respects_exclusion "s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["journal"], exclude=["journal"] ) with pytest.raises(ValueError, match="different parameters"): - pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=[]) + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=[], exclude=["journal"]) assert pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=["journal"], exclude=["journal"]).to_copy == [] assert storage.uploads == [] assert storage.objects["bucket", "run/journal"] == b"old" + + +def test_sync_overwrite_keeps_size_only_download_and_uploads_selected_local_bytes(tmp_path, storage): + local = tmp_path / "journal" + local.write_bytes(b"local") + storage.objects["bucket", "run/journal"] = b"other" + with pos3.mirror(show_progress=False): + pos3.sync("s3://bucket/run/", tmp_path, interval=None, delete_remote=False, overwrite=["journal"]) + assert local.read_bytes() == b"local" + assert storage.objects["bucket", "run/journal"] == b"other" + assert storage.objects["bucket", "run/journal"] == b"local" + + +def test_directory_name_does_not_select_its_contents(tmp_path, storage): + directory = tmp_path / "state" + directory.mkdir() + (directory / "journal").write_bytes(b"new") + storage.objects["bucket", "run/state/journal"] = b"old" + storage.objects["bucket", "run/state/"] = b"" + with pos3.mirror(show_progress=False): + assert pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=["state"]).to_copy == [] + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["state"]) + assert storage.objects["bucket", "run/state/journal"] == b"old" + + +def test_single_file_source_keeps_size_only_copying_with_overwrite_patterns(tmp_path, storage): + local = tmp_path / "journal" + local.write_bytes(b"new") + storage.objects["bucket", "journal"] = b"old" + with pos3.mirror(show_progress=False): + assert pos3.plan_upload("s3://bucket/journal", local, overwrite=["*", "journal"]).to_copy == [] + pos3.upload("s3://bucket/journal", local, interval=None, delete=False, overwrite=["*", "journal"]) + assert storage.uploads == [] + assert storage.objects["bucket", "journal"] == b"old" From d955336eabb31e7f409a6eea67de9a6807a20d2d Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 22:34:36 -0700 Subject: [PATCH 4/9] Support upload-only sync exclusions and verify nested overwrite patterns --- CHANGELOG.md | 2 ++ README.md | 5 +++- pos3/__init__.py | 16 +++++++++-- tests/test_overwrite.py | 62 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4d9c2a..da8a1b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Added - Optional `overwrite` file patterns on `upload`, `sync` and `plan_upload` copy selected directory files even when sizes match. Exclusions take precedence. Downloads and deletion settings retain their existing behavior. +- Optional `upload_exclude` on `sync` overrides upload exclusions without changing the initial download. + `None` inherits `exclude`; an empty list clears upload exclusions. ## [0.4.0] - 2026-09-24 diff --git a/README.md b/README.md index 1f6b311..e715b07 100644 --- a/README.md +++ b/README.md @@ -100,12 +100,15 @@ Registers a local path for upload. Uploads on exit and optionally in background. Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. This selection applies to file paths relative to a directory source. Patterns use `PurePosixPath.match`, which matches from the right. A directory name does not select its contents. A single-file source has no relative file path, so overwrite patterns cannot select it. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. -### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None)` +### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None, upload_exclude=None)` Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint. - `delete_local`: Cleanup local files during download. - `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history. - `overwrite`: Select local directory files for forced upload. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative. +- `upload_exclude`: Override exclusions for uploads only. `None` uses `exclude` for both directions. An empty list clears upload exclusions. The initial download always uses `exclude`. + +Use `upload_exclude=[".unfinished"]` to download existing recording markers while withholding new open markers from periodic uploads. A later one-shot `upload` can include markers that remain after recording stops. This option changes no deletion setting. **Returns**: `pathlib.Path` to the local directory/file. diff --git a/pos3/__init__.py b/pos3/__init__.py index b45b1b8..66b4715 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -627,6 +627,7 @@ def sync( exclude: list[str] | None = None, profile: str | Profile | None = None, overwrite: list[str] | None = None, + upload_exclude: list[str] | None = None, ) -> Path: # Let download() and upload() handle profile resolution and normalization local_path = self.download(remote, local, delete_local, exclude, profile) @@ -637,7 +638,16 @@ def sync( effective_profile = self._effective_profile(profile, remote) # Unregister the download to allow upload registration for the same remote self._downloads.pop((normalized, effective_profile), None) - return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile, overwrite) + return self.upload( + remote, + local_path, + interval, + delete_remote, + sync_on_error, + exclude if upload_exclude is None else upload_exclude, + profile, + overwrite, + ) def ls(self, prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]: """Lists objects under the given prefix, working for both local directories and S3 prefixes.""" @@ -1196,6 +1206,7 @@ def sync( exclude: list[str] | None = None, profile: str | Profile | None = None, overwrite: list[str] | None = None, + upload_exclude: list[str] | None = None, ) -> Path: """ Bi-directional helper. Performs download() then registers upload(). @@ -1204,6 +1215,7 @@ def sync( delete_local: Cleanup local files during download. delete_remote: Cleanup remote files during upload. overwrite: Relative-path patterns copied on every upload, even when sizes match. + upload_exclude: Upload-only exclusions. None inherits exclude; an empty list clears upload exclusions. profile: S3 profile name or Profile config for custom endpoints. Returns: @@ -1211,7 +1223,7 @@ def sync( """ mirror_obj = _require_active_mirror() return mirror_obj.sync( - remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile, overwrite + remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile, overwrite, upload_exclude ) diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index 34e3059..4c0ced8 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -121,6 +121,29 @@ def test_directory_name_does_not_select_its_contents(tmp_path, storage): assert storage.objects["bucket", "run/state/journal"] == b"old" +@pytest.mark.parametrize("method", ["upload", "sync"]) +@pytest.mark.parametrize("patterns", [["journal"], ["state/journal"]]) +def test_right_relative_patterns_select_and_copy_nested_same_size_files(tmp_path, storage, method, patterns): + directory = tmp_path / "state" + directory.mkdir() + for name in ["journal", "other"]: + (directory / name).write_bytes(b"new") + storage.objects["bucket", "run/state/" + name] = b"old" + storage.objects["bucket", "run/state/remote-only"] = b"keep" + with pos3.mirror(show_progress=False): + plan = pos3.plan_upload("s3://bucket/run/", tmp_path, overwrite=patterns) + assert plan.to_copy == [(str(directory / "journal"), "s3://bucket/run/state/journal")] + kwargs = {"delete": False} if method == "upload" else {"delete_remote": False} + getattr(pos3, method)("s3://bucket/run/", tmp_path, interval=None, overwrite=patterns, **kwargs) + assert (directory / "journal").read_bytes() == b"new" + assert storage.objects["bucket", "run/state/journal"] == b"old" + assert storage.uploads == ["run/state/journal"] + assert storage.objects["bucket", "run/state/journal"] == b"new" + assert storage.objects["bucket", "run/state/other"] == b"old" + assert storage.objects["bucket", "run/state/remote-only"] == b"keep" + assert storage.deletes == [] + + def test_single_file_source_keeps_size_only_copying_with_overwrite_patterns(tmp_path, storage): local = tmp_path / "journal" local.write_bytes(b"new") @@ -130,3 +153,42 @@ def test_single_file_source_keeps_size_only_copying_with_overwrite_patterns(tmp_ pos3.upload("s3://bucket/journal", local, interval=None, delete=False, overwrite=["*", "journal"]) assert storage.uploads == [] assert storage.objects["bucket", "journal"] == b"old" + + +@pytest.mark.parametrize( + "upload_exclude,expected", [(None, {"upload"}), ([], {"general", "upload"}), (["upload"], {"general"})] +) +def test_sync_upload_exclusions_keep_the_download_exclusions(tmp_path, storage, upload_exclude, expected): + storage.objects["bucket", "run/general"] = b"excluded download" + storage.objects["bucket", "run/baseline"] = b"retained" + with pos3.mirror(show_progress=False): + pos3.sync( + "s3://bucket/run/", + tmp_path, + interval=None, + delete_remote=False, + exclude=["general"], + upload_exclude=upload_exclude, + ) + assert not (tmp_path / "general").exists() + assert (tmp_path / "baseline").read_bytes() == b"retained" + (tmp_path / "general").write_bytes(b"new") + (tmp_path / "upload").write_bytes(b"new") + assert set(storage.uploads) == {"run/" + name for name in expected} + assert storage.deletes == [] + + +def test_sync_can_download_baseline_markers_without_uploading_new_open_markers(tmp_path, storage): + storage.objects["bucket", "run/baseline/.unfinished"] = b"unfinished" + storage.objects["bucket", "run/foreign"] = b"retained" + with pos3.mirror(show_progress=False): + pos3.sync("s3://bucket/run/", tmp_path, interval=None, delete_remote=False, upload_exclude=[".unfinished"]) + assert (tmp_path / "baseline/.unfinished").read_bytes() == b"unfinished" + (tmp_path / "new").mkdir() + (tmp_path / "new/.unfinished").write_bytes(b"unfinished") + (tmp_path / "new/data").write_bytes(b"recording") + assert ("bucket", "run/new/.unfinished") not in storage.objects + assert storage.objects["bucket", "run/new/data"] == b"recording" + assert storage.objects["bucket", "run/baseline/.unfinished"] == b"unfinished" + assert storage.objects["bucket", "run/foreign"] == b"retained" + assert storage.deletes == [] From 391fadde04124746d50fd03a4612cd1cc525f396 Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 22:50:03 -0700 Subject: [PATCH 5/9] test: reproduce overwrite selection mutation and pin sync deletion behavior --- tests/test_overwrite.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index 4c0ced8..8d64463 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -192,3 +192,55 @@ def test_sync_can_download_baseline_markers_without_uploading_new_open_markers(t assert storage.objects["bucket", "run/baseline/.unfinished"] == b"unfinished" assert storage.objects["bucket", "run/foreign"] == b"retained" assert storage.deletes == [] + + storage.objects["bucket", "run/late-foreign"] = b"arrived after phase one" + assert not (tmp_path / "late-foreign").exists() + with pos3.mirror(show_progress=False): + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False) + assert storage.objects["bucket", "run/new/.unfinished"] == b"unfinished" + assert storage.objects["bucket", "run/baseline/.unfinished"] == b"unfinished" + assert storage.objects["bucket", "run/foreign"] == b"retained" + assert storage.objects["bucket", "run/late-foreign"] == b"arrived after phase one" + assert storage.deletes == [] + + +@pytest.mark.parametrize("delete_remote", [False, True]) +@pytest.mark.parametrize("upload_exclude", [None, []]) +def test_sync_narrowed_upload_exclusions_obey_remote_deletion(tmp_path, storage, delete_remote, upload_exclude): + storage.objects["bucket", "run/general"] = b"excluded download" + storage.objects["bucket", "run/baseline"] = b"retained" + with pos3.mirror(show_progress=False): + pos3.sync( + "s3://bucket/run/", + tmp_path, + interval=None, + delete_remote=delete_remote, + exclude=["general"], + upload_exclude=upload_exclude, + ) + assert not (tmp_path / "general").exists() + assert (tmp_path / "baseline").read_bytes() == b"retained" + if delete_remote and upload_exclude == []: + assert ("bucket", "run/general") not in storage.objects + assert storage.deletes == ["run/general"] + else: + assert storage.objects["bucket", "run/general"] == b"excluded download" + assert storage.deletes == [] + assert storage.objects["bucket", "run/baseline"] == b"retained" + assert storage.uploads == [] + + +def test_registered_overwrite_selection_survives_caller_list_mutation(tmp_path, storage): + for name in ["journal", "other"]: + (tmp_path / name).write_bytes(b"new") + storage.objects["bucket", "run/" + name] = b"old" + patterns = ["journal"] + with pos3.mirror(show_progress=False): + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=patterns) + patterns[:] = ["other"] + with pytest.raises(ValueError, match="different parameters"): + pos3.upload("s3://bucket/run/", tmp_path, interval=None, delete=False, overwrite=["other"]) + assert storage.uploads == ["run/journal"] + assert storage.objects["bucket", "run/journal"] == b"new" + assert storage.objects["bucket", "run/other"] == b"old" + assert storage.deletes == [] From aeec681d69f0221f4957954346d57eb49363eb1c Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Tue, 6 Oct 2026 22:50:44 -0700 Subject: [PATCH 6/9] fix: retain registered overwrite selection and document safe marker uploads --- CHANGELOG.md | 2 ++ README.md | 13 +++++++++++-- pos3/__init__.py | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da8a1b2..836c7c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Exclusions take precedence. Downloads and deletion settings retain their existing behavior. - Optional `upload_exclude` on `sync` overrides upload exclusions without changing the initial download. `None` inherits `exclude`; an empty list clears upload exclusions. + With `delete_remote=True`, remote objects matched by `exclude` but not by `upload_exclude` are deleted unless a local file exists at that path. + Use `delete_remote=False` to preserve these objects. For a later upload of recording markers, close the first mirror and use a new mirror with `upload(..., interval=None, delete=False)`. ## [0.4.0] - 2026-09-24 diff --git a/README.md b/README.md index e715b07..4dc692a 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,18 @@ Bi-directional helper. Performs `download()` then registers `upload()`. Useful f - `delete_local`: Cleanup local files during download. - `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history. - `overwrite`: Select local directory files for forced upload. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative. -- `upload_exclude`: Override exclusions for uploads only. `None` uses `exclude` for both directions. An empty list clears upload exclusions. The initial download always uses `exclude`. +- `upload_exclude`: Override exclusions for uploads only. `None` uses `exclude` for both directions. An empty list clears upload exclusions. The initial download always uses `exclude`. With `delete_remote=True`, remote objects matched by `exclude` but not by `upload_exclude` are deleted unless a local file exists at that path. Use `delete_remote=False` to preserve remote objects when narrowing or clearing upload exclusions. -Use `upload_exclude=[".unfinished"]` to download existing recording markers while withholding new open markers from periodic uploads. A later one-shot `upload` can include markers that remain after recording stops. This option changes no deletion setting. +Use `upload_exclude=[".unfinished"]` with `delete_remote=False` to download existing recording markers while withholding new open markers from periodic uploads. After recording and the first mirror context have stopped, a one-shot `upload` in a new `mirror()` context can include markers that remain. Set `delete=False` on this upload to preserve remote objects absent locally. + +```python +with pos3.mirror(): + local = pos3.sync(remote, local, delete_remote=False, upload_exclude=[".unfinished"]) + # Run and stop recording before leaving this context. + +with pos3.mirror(): + pos3.upload(remote, local, interval=None, delete=False) +``` **Returns**: `pathlib.Path` to the local directory/file. diff --git a/pos3/__init__.py b/pos3/__init__.py index 66b4715..b838c6f 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -493,7 +493,7 @@ def upload( profile=effective_profile, last_sync=0, raw_remote=remote, - overwrite=overwrite, + overwrite=None if overwrite is None else list(overwrite), ) with self._lock: From 31150d025a328ae2b6b306032e8a9410a366dd1c Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Wed, 7 Oct 2026 00:17:35 -0700 Subject: [PATCH 7/9] test: reproduce selected single-file same-size overwrite --- tests/test_overwrite.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index 8d64463..f010546 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -144,16 +144,36 @@ def test_right_relative_patterns_select_and_copy_nested_same_size_files(tmp_path assert storage.deletes == [] -def test_single_file_source_keeps_size_only_copying_with_overwrite_patterns(tmp_path, storage): +@pytest.mark.parametrize("method", ["upload", "sync"]) +@pytest.mark.parametrize( + "patterns,selected", + [(None, False), ([], False), (["*"], True), (["journal"], True), (["remote-journal"], False), (["state/journal"], False)], +) +def test_single_file_overwrite_matches_source_basename_and_retains_remote_objects( + tmp_path, storage, method, patterns, selected +): local = tmp_path / "journal" local.write_bytes(b"new") - storage.objects["bucket", "journal"] = b"old" + remote = "s3://bucket/run/remote-journal" + storage.objects["bucket", "run/remote-journal"] = b"old" + storage.objects["bucket", "run/remote-only"] = b"keep" with pos3.mirror(show_progress=False): - assert pos3.plan_upload("s3://bucket/journal", local, overwrite=["*", "journal"]).to_copy == [] - pos3.upload("s3://bucket/journal", local, interval=None, delete=False, overwrite=["*", "journal"]) - assert storage.uploads == [] - assert storage.objects["bucket", "journal"] == b"old" - + plan = pos3.plan_upload(remote, local, overwrite=patterns) + assert plan.to_copy == ([(str(local), remote)] if selected else []) + assert plan.to_delete == [] + assert storage.uploads == [] + assert storage.deletes == [] + kwargs = {"delete": False} if method == "upload" else {"delete_remote": False} + getattr(pos3, method)(remote, local, interval=None, overwrite=patterns, **kwargs) + assert local.read_bytes() == b"new" + assert storage.objects["bucket", "run/remote-journal"] == b"old" + assert storage.uploads == (["run/remote-journal"] if selected else []) + assert storage.deletes == [] + assert storage.objects["bucket", "run/remote-journal"] == (b"new" if selected else b"old") + assert storage.objects["bucket", "run/remote-only"] == b"keep" + with pos3.mirror(show_progress=False): + readback = pos3.download(remote, local=tmp_path / "readback") + assert readback.read_bytes() == (b"new" if selected else b"old") @pytest.mark.parametrize( "upload_exclude,expected", [(None, {"upload"}), ([], {"general", "upload"}), (["upload"], {"general"})] From ef616565b5b425221c4eb2b86eea573200088d7f Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Wed, 7 Oct 2026 00:18:01 -0700 Subject: [PATCH 8/9] test: observe stale transport bytes before planner selection --- tests/test_overwrite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index f010546..2ecfefc 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -159,7 +159,6 @@ def test_single_file_overwrite_matches_source_basename_and_retains_remote_object storage.objects["bucket", "run/remote-only"] = b"keep" with pos3.mirror(show_progress=False): plan = pos3.plan_upload(remote, local, overwrite=patterns) - assert plan.to_copy == ([(str(local), remote)] if selected else []) assert plan.to_delete == [] assert storage.uploads == [] assert storage.deletes == [] @@ -167,9 +166,10 @@ def test_single_file_overwrite_matches_source_basename_and_retains_remote_object getattr(pos3, method)(remote, local, interval=None, overwrite=patterns, **kwargs) assert local.read_bytes() == b"new" assert storage.objects["bucket", "run/remote-journal"] == b"old" + assert storage.objects["bucket", "run/remote-journal"] == (b"new" if selected else b"old") + assert plan.to_copy == ([(str(local), remote)] if selected else []) assert storage.uploads == (["run/remote-journal"] if selected else []) assert storage.deletes == [] - assert storage.objects["bucket", "run/remote-journal"] == (b"new" if selected else b"old") assert storage.objects["bucket", "run/remote-only"] == b"keep" with pos3.mirror(show_progress=False): readback = pos3.download(remote, local=tmp_path / "readback") From d5fae5039d8e4cfd661065706a2d2a1168717ade Mon Sep 17 00:00:00 2001 From: Sasha Sobol Date: Wed, 7 Oct 2026 00:19:03 -0700 Subject: [PATCH 9/9] fix: match selected single-file uploads by source basename --- README.md | 4 ++-- pos3/__init__.py | 14 ++++++++++---- tests/test_overwrite.py | 10 +++++++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4dc692a..7dad71c 100644 --- a/README.md +++ b/README.md @@ -98,14 +98,14 @@ Registers a local path for upload. Uploads on exit and optionally in background. **Returns**: `pathlib.Path` to the local directory/file. -Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. This selection applies to file paths relative to a directory source. Patterns use `PurePosixPath.match`, which matches from the right. A directory name does not select its contents. A single-file source has no relative file path, so overwrite patterns cannot select it. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. +Pass `overwrite=[".recording-attempts.sqlite", "run_metadata_*.yaml"]` to copy matching files even when the remote size matches. This selection applies to file paths relative to a directory source, or the local basename of a single-file source. Patterns use `PurePosixPath.match`, which matches from the right. A directory name does not select its contents. For example, a local `journal` file matches `"*"` or `"journal"`, even when uploaded to a different remote key. Excluded files remain excluded. The default compares file sizes. This option does not change deletion settings. `plan_upload(..., overwrite=...)` reports the same selection without transfers. ### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None, profile=None, overwrite=None, upload_exclude=None)` Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint. - `delete_local`: Cleanup local files during download. - `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history. -- `overwrite`: Select local directory files for forced upload. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative. +- `overwrite`: Select local files for forced upload using directory-relative paths or a single-file basename. The initial download still compares sizes. Existing same-size local files stay local and can replace remote bytes on upload. Use a fresh local root when the remote contents must be authoritative. - `upload_exclude`: Override exclusions for uploads only. `None` uses `exclude` for both directions. An empty list clears upload exclusions. The initial download always uses `exclude`. With `delete_remote=True`, remote objects matched by `exclude` but not by `upload_exclude` are deleted unless a local file exists at that path. Use `delete_remote=False` to preserve remote objects when narrowing or clearing upload exclusions. Use `upload_exclude=[".unfinished"]` with `delete_remote=False` to download existing recording markers while withholding new open markers from periodic uploads. After recording and the first mirror context have stopped, a one-shot `upload` in a new `mirror()` context can include markers that remain. Set `delete=False` on this upload to preserve remote objects absent locally. diff --git a/pos3/__init__.py b/pos3/__init__.py index b838c6f..00f2d41 100644 --- a/pos3/__init__.py +++ b/pos3/__init__.py @@ -194,7 +194,11 @@ def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | Non def _compute_sync_diff( - source: Iterator[FileInfo], target: Iterator[FileInfo], overwrite: list[str] | None = None + source: Iterator[FileInfo], + target: Iterator[FileInfo], + overwrite: list[str] | None = None, + *, + source_name: str = ".", ) -> tuple[list[FileInfo], list[FileInfo]]: source_map: dict[str, FileInfo] = {info.relative_path: info for info in source} target_map: dict[str, FileInfo] = {info.relative_path: info for info in target} @@ -211,7 +215,7 @@ def _compute_sync_diff( to_copy.append(source_info) elif not source_info.is_dir and ( source_info.size != target_info.size - or any(PurePosixPath(relative_path or ".").match(pattern) for pattern in overwrite or []) + or any(PurePosixPath(relative_path or source_name).match(pattern) for pattern in overwrite or []) ): to_copy.append(source_info) @@ -600,6 +604,7 @@ def plan_upload( _filter_fileinfo(_scan_local(source), exclude), _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, effective_profile), exclude), overwrite=overwrite, + source_name=source.name, ) copies: list[tuple[str, str]] = [] for info in to_copy: @@ -812,6 +817,7 @@ def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None: _filter_fileinfo(_scan_local(local_path), exclude), _filter_fileinfo(self._scan_s3(scan_bucket, scan_prefix, profile), exclude), overwrite=overwrite, + source_name=local_path.name, ) for info in to_copy: @@ -1187,7 +1193,7 @@ def upload( delete: If True (default), deletes S3 files NOT present locally. sync_on_error: If True, syncs even if the context exits with an exception. profile: S3 profile name or Profile config for custom endpoints. - overwrite: Relative-path patterns copied on every upload, even when sizes match. + overwrite: Relative-path or single-file basename patterns copied even when sizes match. Returns: Path to the local directory/file. @@ -1214,7 +1220,7 @@ def sync( Args: delete_local: Cleanup local files during download. delete_remote: Cleanup remote files during upload. - overwrite: Relative-path patterns copied on every upload, even when sizes match. + overwrite: Relative-path or single-file basename patterns copied even when sizes match. upload_exclude: Upload-only exclusions. None inherits exclude; an empty list clears upload exclusions. profile: S3 profile name or Profile config for custom endpoints. diff --git a/tests/test_overwrite.py b/tests/test_overwrite.py index 2ecfefc..1aaef3d 100644 --- a/tests/test_overwrite.py +++ b/tests/test_overwrite.py @@ -147,7 +147,14 @@ def test_right_relative_patterns_select_and_copy_nested_same_size_files(tmp_path @pytest.mark.parametrize("method", ["upload", "sync"]) @pytest.mark.parametrize( "patterns,selected", - [(None, False), ([], False), (["*"], True), (["journal"], True), (["remote-journal"], False), (["state/journal"], False)], + [ + (None, False), + ([], False), + (["*"], True), + (["journal"], True), + (["remote-journal"], False), + (["state/journal"], False), + ], ) def test_single_file_overwrite_matches_source_basename_and_retains_remote_objects( tmp_path, storage, method, patterns, selected @@ -175,6 +182,7 @@ def test_single_file_overwrite_matches_source_basename_and_retains_remote_object readback = pos3.download(remote, local=tmp_path / "readback") assert readback.read_bytes() == (b"new" if selected else b"old") + @pytest.mark.parametrize( "upload_exclude,expected", [(None, {"upload"}), ([], {"general", "upload"}), (["upload"], {"general"})] )