-
Notifications
You must be signed in to change notification settings - Fork 268
fix(core): tolerate disappearing guarded files #1383
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
fae73a4
fix(core): tolerate disappearing guarded files
phernandez 3a9ecaf
fix(core): preserve missing checksum outcome
phernandez 5dbdf95
fix(core): preserve checksum service contract
phernandez 88e1381
fix(core): cover directory cleanup checksum race
phernandez 14dc261
Handle delete checksum disappearance race
phernandez aeb0997
Handle note delete checksum disappearance race
phernandez File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| """Race regressions for portable note-file concurrency guards.""" | ||
|
|
||
| from pathlib import Path | ||
| from unittest.mock import AsyncMock, patch | ||
|
|
||
| import pytest | ||
|
|
||
| from basic_memory.file_utils import FileError | ||
| from basic_memory.index.local_notes import LocalNoteFileDeleteStorage | ||
| from basic_memory.index.note_content_materialization import LocalNoteContentStorage | ||
| from basic_memory.indexing.note_file_delete_runner import run_note_file_delete | ||
| from basic_memory.runtime.cleanup import RuntimeDeleteStatus, RuntimeNoteFileDeleteJobRequest | ||
| from basic_memory.runtime.note_file_guards import read_runtime_file_checksum | ||
| from basic_memory.services.file_service import FileService | ||
|
|
||
|
|
||
| async def test_checksum_read_treats_post_probe_deletion_as_absent(tmp_path: Path) -> None: | ||
| """A disappearing object should not poison a durable materialization retry.""" | ||
| file_service = FileService(tmp_path) | ||
| storage = LocalNoteContentStorage(file_service) | ||
|
|
||
| # The file disappears after storage reports it present but before checksum I/O. | ||
| with patch.object(file_service, "exists", AsyncMock(return_value=True)): | ||
| checksum = await read_runtime_file_checksum(storage, "notes/disappeared.md") | ||
|
|
||
| assert checksum is None | ||
|
|
||
|
|
||
| async def test_directory_delete_converges_when_file_disappears_before_delete( | ||
| tmp_path: Path, | ||
| ) -> None: | ||
| """The final guarded checksum should treat a vanished target as a safe no-delete.""" | ||
| file_service = FileService(tmp_path) | ||
| file_path = "notes/disappeared.md" | ||
| await file_service.write_file(file_path, "# Disappearing note\n") | ||
| accepted_checksum = await file_service.compute_checksum(file_path) | ||
| original_compute_checksum = file_service.compute_checksum | ||
| checksum_calls = 0 | ||
|
|
||
| async def delete_before_final_checksum(path: str) -> str: | ||
| nonlocal checksum_calls | ||
| checksum_calls += 1 | ||
| if checksum_calls == 2: | ||
| (tmp_path / path).unlink() | ||
| return await original_compute_checksum(path) | ||
|
|
||
| with patch.object(file_service, "compute_checksum", side_effect=delete_before_final_checksum): | ||
| result = await run_note_file_delete( | ||
| RuntimeNoteFileDeleteJobRequest( | ||
| project_id=101, | ||
| entity_id=42, | ||
| file_path=file_path, | ||
| file_checksum=accepted_checksum, | ||
| ), | ||
| storage=LocalNoteFileDeleteStorage(file_service), | ||
| ) | ||
|
|
||
| assert result.status == RuntimeDeleteStatus.skipped | ||
| assert result.reason == f"file changed before delete: {file_path}" | ||
| assert not (tmp_path / file_path).exists() | ||
|
|
||
|
|
||
| async def test_note_delete_converges_when_file_disappears_before_delete( | ||
| tmp_path: Path, | ||
| ) -> None: | ||
| """Ordinary note cleanup should share the safe final-checksum outcome.""" | ||
| file_service = FileService(tmp_path) | ||
| file_path = "notes/disappeared.md" | ||
| await file_service.write_file(file_path, "# Disappearing note\n") | ||
| accepted_checksum = await file_service.compute_checksum(file_path) | ||
| original_compute_checksum = file_service.compute_checksum | ||
| checksum_calls = 0 | ||
|
|
||
| async def delete_before_final_checksum(path: str) -> str: | ||
| nonlocal checksum_calls | ||
| checksum_calls += 1 | ||
| if checksum_calls == 2: | ||
| (tmp_path / path).unlink() | ||
| return await original_compute_checksum(path) | ||
|
|
||
| with patch.object(file_service, "compute_checksum", side_effect=delete_before_final_checksum): | ||
| result = await run_note_file_delete( | ||
| RuntimeNoteFileDeleteJobRequest( | ||
| project_id=101, | ||
| entity_id=42, | ||
| file_path=file_path, | ||
| file_checksum=accepted_checksum, | ||
| ), | ||
| storage=LocalNoteContentStorage(file_service), | ||
| ) | ||
|
|
||
| assert result.status == RuntimeDeleteStatus.skipped | ||
| assert result.reason == f"file changed before delete: {file_path}" | ||
| assert not (tmp_path / file_path).exists() | ||
|
|
||
|
|
||
| async def test_direct_checksum_preserves_file_service_error_contract(tmp_path: Path) -> None: | ||
| """Direct callers still receive FileError when the checksum source is absent.""" | ||
| file_service = FileService(tmp_path) | ||
|
|
||
| with pytest.raises(FileError): | ||
| await file_service.compute_checksum("notes/disappeared.md") | ||
|
|
||
|
|
||
| async def test_directory_delete_checksum_treats_post_probe_deletion_as_absent( | ||
| tmp_path: Path, | ||
| ) -> None: | ||
| """Directory cleanup should converge when its target disappears before checksum I/O.""" | ||
| file_service = FileService(tmp_path) | ||
| storage = LocalNoteFileDeleteStorage(file_service) | ||
|
|
||
| with patch.object(file_service, "exists", AsyncMock(return_value=True)): | ||
| checksum = await read_runtime_file_checksum(storage, "notes/disappeared.md") | ||
|
|
||
| assert checksum is None |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.