diff --git a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md index 19ea094be..cf025ab1a 100644 --- a/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md +++ b/docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md @@ -154,6 +154,12 @@ dump spanning an in-progress file-copy creation can contain its concept before its locator relations; a successful document restore alone does not establish a usable content restore. +Database backup encryption and restore decryption stream the existing Fernet +file format through private temporary files. Restore authenticates the complete +ciphertext before decrypting or publishing plaintext, so large archives do not +require a whole-file memory allocation. Run the paired restore after bulk +imports have stopped, under the same measured resource budget. + `set_task_project_writer` records a reversible, per-project writer decision and its evidence. Moving to `von` requires successful project reconciliation plus site-retention, normal-use, restore and final-delta receipts. The smallest diff --git a/scripts/backup_von_db.py b/scripts/backup_von_db.py index d26389210..b6b5d01e6 100644 --- a/scripts/backup_von_db.py +++ b/scripts/backup_von_db.py @@ -234,13 +234,12 @@ def _compress_backup_dir_to_zip(backup_root: Path) -> Path: def _encrypt_file_fernet(input_path: Path, *, key: str) -> Path: - from cryptography.fernet import Fernet + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from src.backend.utils.fernet_file import encrypt_file - f = Fernet(key.encode("utf-8")) - data = input_path.read_bytes() - encrypted = f.encrypt(data) out_path = input_path.with_suffix(input_path.suffix + ".enc") - out_path.write_bytes(encrypted) + encrypt_file(input_path, out_path, key=key) # Carry timestamp forward ts = input_path.stat().st_mtime os.utime(out_path, (ts, ts)) diff --git a/scripts/restore_von_db.py b/scripts/restore_von_db.py index 16b86f389..aa0b12521 100644 --- a/scripts/restore_von_db.py +++ b/scripts/restore_von_db.py @@ -87,11 +87,11 @@ def _load_receipt_if_present(path: Path) -> dict[str, Any] | None: def _decrypt_encrypted_zip( encrypted_path: Path, *, fernet_key: str, output_zip_path: Path ) -> Path: - from cryptography.fernet import Fernet + if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + from src.backend.utils.fernet_file import decrypt_file - f = Fernet(fernet_key.encode("utf-8")) - encrypted_bytes = encrypted_path.read_bytes() - output_zip_path.write_bytes(f.decrypt(encrypted_bytes)) + decrypt_file(encrypted_path, output_zip_path, key=fernet_key) return output_zip_path diff --git a/src/backend/utils/fernet_file.py b/src/backend/utils/fernet_file.py new file mode 100644 index 000000000..a6653ae81 --- /dev/null +++ b/src/backend/utils/fernet_file.py @@ -0,0 +1,144 @@ +"""Bounded-memory files in the existing Fernet token format. + +Ciphertext is spooled privately so its entire HMAC can be verified before any +plaintext is decrypted. Completed output replaces its destination atomically. +The algorithm and format are specified at https://github.com/fernet/spec. +""" + +from __future__ import annotations + +import base64 +import binascii +import os +import struct +import tempfile +import time +from contextlib import contextmanager +from pathlib import Path + +from cryptography.exceptions import InvalidSignature +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes, hmac, padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +# Divisible by the AES block size and both base64 input/output group sizes. +_CHUNK_BYTES = 3 * 256 * 1024 +_HEADER_BYTES = 25 +_SIGNATURE_BYTES = 32 + + +def _keys(key: str | bytes) -> tuple[bytes, bytes]: + Fernet(key) # Apply the library's public key validation contract. + decoded = base64.urlsafe_b64decode(key) + return decoded[:16], decoded[16:] + + +@contextmanager +def _atomic_output(destination: Path): + descriptor, name = tempfile.mkstemp( + dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp" + ) + temporary = Path(name) + try: + with os.fdopen(descriptor, "wb") as output: + yield output + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, destination) + finally: + temporary.unlink(missing_ok=True) + + +def encrypt_file(source: Path, destination: Path, *, key: str | bytes) -> None: + """Write a standard Fernet token without loading the file into memory.""" + if source.resolve() == destination.resolve(): + raise ValueError("Fernet input and output must be different files") + signing_key, encryption_key = _keys(key) + iv = os.urandom(16) + header = b"\x80" + struct.pack(">Q", int(time.time())) + iv + signer = hmac.HMAC(signing_key, hashes.SHA256()) + padder = padding.PKCS7(128).padder() + encryptor = Cipher(algorithms.AES(encryption_key), modes.CBC(iv)).encryptor() + with tempfile.TemporaryFile(dir=destination.parent) as ciphertext: + ciphertext.write(header) + signer.update(header) + with source.open("rb") as input_file: + while chunk := input_file.read(_CHUNK_BYTES): + encrypted = encryptor.update(padder.update(chunk)) + ciphertext.write(encrypted) + signer.update(encrypted) + encrypted = encryptor.update(padder.finalize()) + encryptor.finalize() + ciphertext.write(encrypted) + signer.update(encrypted) + ciphertext.write(signer.finalize()) + ciphertext.seek(0) + with _atomic_output(destination) as output: + while chunk := ciphertext.read(_CHUNK_BYTES): + output.write(base64.urlsafe_b64encode(chunk)) + + +def decrypt_file(source: Path, destination: Path, *, key: str | bytes) -> None: + """Authenticate a complete Fernet file, then decrypt with bounded memory. + + Backup tokens have no expiry. Invalid or truncated input never publishes + plaintext and leaves any existing destination unchanged. + """ + if source.resolve() == destination.resolve(): + raise ValueError("Fernet input and output must be different files") + signing_key, encryption_key = _keys(key) + with tempfile.TemporaryFile(dir=destination.parent) as ciphertext: + padded = False + with source.open("rb") as input_file: + while chunk := input_file.read(_CHUNK_BYTES): + if padded: + raise InvalidToken + try: + decoded = base64.b64decode(chunk, altchars=b"-_", validate=True) + except (binascii.Error, ValueError) as exc: + raise InvalidToken from exc + ciphertext.write(decoded) + padded = b"=" in chunk + size = ciphertext.tell() + payload_size = size - _HEADER_BYTES - _SIGNATURE_BYTES + ciphertext.seek(0) + header = ciphertext.read(_HEADER_BYTES) + if ( + len(header) != _HEADER_BYTES + or header[0] != 0x80 + or payload_size < 16 + or payload_size % 16 + ): + raise InvalidToken + signer = hmac.HMAC(signing_key, hashes.SHA256()) + signer.update(header) + remaining = payload_size + while remaining: + chunk = ciphertext.read(min(_CHUNK_BYTES, remaining)) + if not chunk: + raise InvalidToken + signer.update(chunk) + remaining -= len(chunk) + try: + signer.verify(ciphertext.read(_SIGNATURE_BYTES)) + except InvalidSignature as exc: + raise InvalidToken from exc + + # Only authenticated ciphertext reaches the decryptor or output file. + decryptor = Cipher( + algorithms.AES(encryption_key), modes.CBC(header[9:25]) + ).decryptor() + unpadder = padding.PKCS7(128).unpadder() + ciphertext.seek(_HEADER_BYTES) + remaining = payload_size + with _atomic_output(destination) as output: + while remaining: + chunk = ciphertext.read(min(_CHUNK_BYTES, remaining)) + if not chunk: + raise InvalidToken + output.write(unpadder.update(decryptor.update(chunk))) + remaining -= len(chunk) + try: + output.write(unpadder.update(decryptor.finalize())) + output.write(unpadder.finalize()) + except ValueError as exc: + raise InvalidToken from exc diff --git a/tests/test_fernet_file.py b/tests/test_fernet_file.py new file mode 100644 index 000000000..a478c5d25 --- /dev/null +++ b/tests/test_fernet_file.py @@ -0,0 +1,136 @@ +import base64 +import hashlib +import hmac +import os +from pathlib import Path + +import pytest +from cryptography.fernet import Fernet, InvalidToken + +from scripts.backup_von_db import _encrypt_file_fernet +from scripts.restore_von_db import _decrypt_encrypted_zip +from src.backend.utils import fernet_file + + +@pytest.mark.parametrize("size", [0, 15, 16, 17, 47, 48, 49, 96, 257]) +def test_file_tokens_interoperate_with_fernet(tmp_path, monkeypatch, size): + monkeypatch.setattr(fernet_file, "_CHUNK_BYTES", 48) + key = Fernet.generate_key() + data = os.urandom(size) + source = tmp_path / "source" + encrypted = tmp_path / "encrypted" + restored = tmp_path / "restored" + source.write_bytes(data) + + fernet_file.encrypt_file(source, encrypted, key=key) + assert Fernet(key).decrypt(encrypted.read_bytes()) == data + encrypted.write_bytes(Fernet(key).encrypt(data)) + fernet_file.decrypt_file(encrypted, restored, key=key) + assert restored.read_bytes() == data + if os.name != "nt": + assert restored.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize("offset", [0, 1, 9, 25, -1]) +def test_tampered_tokens_are_rejected_before_decryption( + tmp_path, monkeypatch, offset +): + key = Fernet.generate_key() + token = bytearray(base64.urlsafe_b64decode(Fernet(key).encrypt(b"private data"))) + token[offset] ^= 1 + source = tmp_path / "source" + destination = tmp_path / "destination" + source.write_bytes(base64.urlsafe_b64encode(token)) + destination.write_bytes(b"preserve existing output") + + def reject_decryptor(*args, **kwargs): + pytest.fail("Unauthenticated ciphertext reached the decryptor") + + monkeypatch.setattr(fernet_file, "Cipher", reject_decryptor) + with pytest.raises(InvalidToken): + fernet_file.decrypt_file(source, destination, key=key) + assert destination.read_bytes() == b"preserve existing output" + assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"] + + +@pytest.mark.parametrize("damage", ["wrong_key", "truncated", "invalid_base64", "appended"]) +def test_invalid_files_never_publish_plaintext(tmp_path, damage): + key = Fernet.generate_key() + token = Fernet(key).encrypt(b"private data") + if damage == "wrong_key": + key = Fernet.generate_key() + elif damage == "truncated": + token = token[:-8] + elif damage == "invalid_base64": + token = b"!" + token[1:] + else: + token += b"AAAA" + source = tmp_path / "source" + destination = tmp_path / "destination" + source.write_bytes(token) + with pytest.raises(InvalidToken): + fernet_file.decrypt_file(source, destination, key=key) + assert not destination.exists() + assert list(tmp_path.iterdir()) == [source] + + +def test_authenticated_invalid_padding_preserves_destination(tmp_path): + key = Fernet.generate_key() + token = bytearray(base64.urlsafe_b64decode(Fernet(key).encrypt(b"A" * 16))) + # The second block is all padding. This changes its final byte to zero. + token[25 + 15] ^= 16 + token[-32:] = hmac.digest(base64.urlsafe_b64decode(key)[:16], token[:-32], hashlib.sha256) + source = tmp_path / "source" + destination = tmp_path / "destination" + source.write_bytes(base64.urlsafe_b64encode(token)) + destination.write_bytes(b"previous") + with pytest.raises(InvalidToken): + fernet_file.decrypt_file(source, destination, key=key) + assert destination.read_bytes() == b"previous" + assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"] + + +def test_backup_restore_entrypoints_do_not_read_whole_files(tmp_path, monkeypatch): + key = Fernet.generate_key().decode() + data = os.urandom(fernet_file._CHUNK_BYTES * 3 + 17) + source = tmp_path / "backup.zip" + destination = tmp_path / "restored.zip" + source.write_bytes(data) + timestamp = 1_700_000_000 + os.utime(source, (timestamp, timestamp)) + + with monkeypatch.context() as patch: + patch.setattr(Path, "read_bytes", lambda *args: pytest.fail("Whole-file read")) + encrypted = _encrypt_file_fernet(source, key=key) + _decrypt_encrypted_zip(encrypted, fernet_key=key, output_zip_path=destination) + assert encrypted.stat().st_mtime == timestamp + assert destination.read_bytes() == data + assert Fernet(key).decrypt(encrypted.read_bytes()) == data + + +@pytest.mark.parametrize("operation", [fernet_file.encrypt_file, fernet_file.decrypt_file]) +def test_failed_publication_preserves_existing_output(tmp_path, monkeypatch, operation): + key = Fernet.generate_key() + source = tmp_path / "source" + destination = tmp_path / "destination" + data = b"private data" + source.write_bytes(Fernet(key).encrypt(data) if operation is fernet_file.decrypt_file else data) + destination.write_bytes(b"previous") + + def fail_replace(*args): + raise OSError("simulated publication failure") + + monkeypatch.setattr(fernet_file.os, "replace", fail_replace) + with pytest.raises(OSError, match="publication failure"): + operation(source, destination, key=key) + assert destination.read_bytes() == b"previous" + assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"] + + +@pytest.mark.parametrize("operation", [fernet_file.encrypt_file, fernet_file.decrypt_file]) +def test_same_path_is_rejected(tmp_path, operation): + source = tmp_path / "source" + source.write_bytes(b"keep") + with pytest.raises(ValueError, match="different files"): + operation(source, source, key=Fernet.generate_key()) + assert source.read_bytes() == b"keep"