Stream backup encryption and authenticated restore (JVNAUTOSCI-2737)

docs/engineering/workflow_first_jira_to_von_migration_jvnautosci_1223.md

@@ -154,6 +154,12 @@ dump spanning an in-progress file-copy creation can contain its concept before
 its locator relations; a successful document restore alone does not establish
 a usable content restore.
 
+Database backup encryption and restore decryption stream the existing Fernet
+file format through private temporary files. Restore authenticates the complete
+ciphertext before decrypting or publishing plaintext, so large archives do not
+require a whole-file memory allocation. Run the paired restore after bulk
+imports have stopped, under the same measured resource budget.
+
 `set_task_project_writer` records a reversible, per-project writer decision and
 its evidence. Moving to `von` requires successful project reconciliation plus
 site-retention, normal-use, restore and final-delta receipts. The smallest

scripts/backup_von_db.py

@@ -234,13 +234,12 @@ def _compress_backup_dir_to_zip(backup_root: Path) -> Path:
 
 
 def _encrypt_file_fernet(input_path: Path, *, key: str) -> Path:
-    from cryptography.fernet import Fernet
+    if str(_REPO_ROOT) not in sys.path:
+        sys.path.insert(0, str(_REPO_ROOT))
+    from src.backend.utils.fernet_file import encrypt_file
 
-    f = Fernet(key.encode("utf-8"))
-    data = input_path.read_bytes()
-    encrypted = f.encrypt(data)
     out_path = input_path.with_suffix(input_path.suffix + ".enc")
-    out_path.write_bytes(encrypted)
+    encrypt_file(input_path, out_path, key=key)
     # Carry timestamp forward
     ts = input_path.stat().st_mtime
     os.utime(out_path, (ts, ts))

scripts/restore_von_db.py

@@ -87,11 +87,11 @@ def _load_receipt_if_present(path: Path) -> dict[str, Any] | None:
 def _decrypt_encrypted_zip(
     encrypted_path: Path, *, fernet_key: str, output_zip_path: Path
 ) -> Path:
-    from cryptography.fernet import Fernet
+    if str(_REPO_ROOT) not in sys.path:
+        sys.path.insert(0, str(_REPO_ROOT))
+    from src.backend.utils.fernet_file import decrypt_file
 
-    f = Fernet(fernet_key.encode("utf-8"))
-    encrypted_bytes = encrypted_path.read_bytes()
-    output_zip_path.write_bytes(f.decrypt(encrypted_bytes))
+    decrypt_file(encrypted_path, output_zip_path, key=fernet_key)
     return output_zip_path
 
 

src/backend/utils/fernet_file.py

@@ -0,0 +1,144 @@
+"""Bounded-memory files in the existing Fernet token format.
+
+Ciphertext is spooled privately so its entire HMAC can be verified before any
+plaintext is decrypted. Completed output replaces its destination atomically.
+The algorithm and format are specified at https://github.com/fernet/spec.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import os
+import struct
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.fernet import Fernet, InvalidToken
+from cryptography.hazmat.primitives import hashes, hmac, padding
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+# Divisible by the AES block size and both base64 input/output group sizes.
+_CHUNK_BYTES = 3 * 256 * 1024
+_HEADER_BYTES = 25
+_SIGNATURE_BYTES = 32
+
+
+def _keys(key: str | bytes) -> tuple[bytes, bytes]:
+    Fernet(key)  # Apply the library's public key validation contract.
+    decoded = base64.urlsafe_b64decode(key)
+    return decoded[:16], decoded[16:]
+
+
+@contextmanager
+def _atomic_output(destination: Path):
+    descriptor, name = tempfile.mkstemp(
+        dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp"
+    )
+    temporary = Path(name)
+    try:
+        with os.fdopen(descriptor, "wb") as output:
+            yield output
+            output.flush()
+            os.fsync(output.fileno())
+        os.replace(temporary, destination)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def encrypt_file(source: Path, destination: Path, *, key: str | bytes) -> None:
+    """Write a standard Fernet token without loading the file into memory."""
+    if source.resolve() == destination.resolve():
+        raise ValueError("Fernet input and output must be different files")
+    signing_key, encryption_key = _keys(key)
+    iv = os.urandom(16)
+    header = b"\x80" + struct.pack(">Q", int(time.time())) + iv
+    signer = hmac.HMAC(signing_key, hashes.SHA256())
+    padder = padding.PKCS7(128).padder()
+    encryptor = Cipher(algorithms.AES(encryption_key), modes.CBC(iv)).encryptor()
+    with tempfile.TemporaryFile(dir=destination.parent) as ciphertext:
+        ciphertext.write(header)
+        signer.update(header)
+        with source.open("rb") as input_file:
+            while chunk := input_file.read(_CHUNK_BYTES):
+                encrypted = encryptor.update(padder.update(chunk))
+                ciphertext.write(encrypted)
+                signer.update(encrypted)
+        encrypted = encryptor.update(padder.finalize()) + encryptor.finalize()
+        ciphertext.write(encrypted)
+        signer.update(encrypted)
+        ciphertext.write(signer.finalize())
+        ciphertext.seek(0)
+        with _atomic_output(destination) as output:
+            while chunk := ciphertext.read(_CHUNK_BYTES):
+                output.write(base64.urlsafe_b64encode(chunk))
+
+
+def decrypt_file(source: Path, destination: Path, *, key: str | bytes) -> None:
+    """Authenticate a complete Fernet file, then decrypt with bounded memory.
+
+    Backup tokens have no expiry. Invalid or truncated input never publishes
+    plaintext and leaves any existing destination unchanged.
+    """
+    if source.resolve() == destination.resolve():
+        raise ValueError("Fernet input and output must be different files")
+    signing_key, encryption_key = _keys(key)
+    with tempfile.TemporaryFile(dir=destination.parent) as ciphertext:
+        padded = False
+        with source.open("rb") as input_file:
+            while chunk := input_file.read(_CHUNK_BYTES):
+                if padded:
+                    raise InvalidToken
+                try:
+                    decoded = base64.b64decode(chunk, altchars=b"-_", validate=True)
+                except (binascii.Error, ValueError) as exc:
+                    raise InvalidToken from exc
+                ciphertext.write(decoded)
+                padded = b"=" in chunk
+        size = ciphertext.tell()
+        payload_size = size - _HEADER_BYTES - _SIGNATURE_BYTES
+        ciphertext.seek(0)
+        header = ciphertext.read(_HEADER_BYTES)
+        if (
+            len(header) != _HEADER_BYTES
+            or header[0] != 0x80
+            or payload_size < 16
+            or payload_size % 16
+        ):
+            raise InvalidToken
+        signer = hmac.HMAC(signing_key, hashes.SHA256())
+        signer.update(header)
+        remaining = payload_size
+        while remaining:
+            chunk = ciphertext.read(min(_CHUNK_BYTES, remaining))
+            if not chunk:
+                raise InvalidToken
+            signer.update(chunk)
+            remaining -= len(chunk)
+        try:
+            signer.verify(ciphertext.read(_SIGNATURE_BYTES))
+        except InvalidSignature as exc:
+            raise InvalidToken from exc
+
+        # Only authenticated ciphertext reaches the decryptor or output file.
+        decryptor = Cipher(
+            algorithms.AES(encryption_key), modes.CBC(header[9:25])
+        ).decryptor()
+        unpadder = padding.PKCS7(128).unpadder()
+        ciphertext.seek(_HEADER_BYTES)
+        remaining = payload_size
+        with _atomic_output(destination) as output:
+            while remaining:
+                chunk = ciphertext.read(min(_CHUNK_BYTES, remaining))
+                if not chunk:
+                    raise InvalidToken
+                output.write(unpadder.update(decryptor.update(chunk)))
+                remaining -= len(chunk)
+            try:
+                output.write(unpadder.update(decryptor.finalize()))
+                output.write(unpadder.finalize())
+            except ValueError as exc:
+                raise InvalidToken from exc

tests/test_fernet_file.py

@@ -0,0 +1,136 @@
+import base64
+import hashlib
+import hmac
+import os
+from pathlib import Path
+
+import pytest
+from cryptography.fernet import Fernet, InvalidToken
+
+from scripts.backup_von_db import _encrypt_file_fernet
+from scripts.restore_von_db import _decrypt_encrypted_zip
+from src.backend.utils import fernet_file
+
+
+@pytest.mark.parametrize("size", [0, 15, 16, 17, 47, 48, 49, 96, 257])
+def test_file_tokens_interoperate_with_fernet(tmp_path, monkeypatch, size):
+    monkeypatch.setattr(fernet_file, "_CHUNK_BYTES", 48)
+    key = Fernet.generate_key()
+    data = os.urandom(size)
+    source = tmp_path / "source"
+    encrypted = tmp_path / "encrypted"
+    restored = tmp_path / "restored"
+    source.write_bytes(data)
+
+    fernet_file.encrypt_file(source, encrypted, key=key)
+    assert Fernet(key).decrypt(encrypted.read_bytes()) == data
+    encrypted.write_bytes(Fernet(key).encrypt(data))
+    fernet_file.decrypt_file(encrypted, restored, key=key)
+    assert restored.read_bytes() == data
+    if os.name != "nt":
+        assert restored.stat().st_mode & 0o777 == 0o600
+
+
+@pytest.mark.parametrize("offset", [0, 1, 9, 25, -1])
+def test_tampered_tokens_are_rejected_before_decryption(
+    tmp_path, monkeypatch, offset
+):
+    key = Fernet.generate_key()
+    token = bytearray(base64.urlsafe_b64decode(Fernet(key).encrypt(b"private data")))
+    token[offset] ^= 1
+    source = tmp_path / "source"
+    destination = tmp_path / "destination"
+    source.write_bytes(base64.urlsafe_b64encode(token))
+    destination.write_bytes(b"preserve existing output")
+
+    def reject_decryptor(*args, **kwargs):
+        pytest.fail("Unauthenticated ciphertext reached the decryptor")
+
+    monkeypatch.setattr(fernet_file, "Cipher", reject_decryptor)
+    with pytest.raises(InvalidToken):
+        fernet_file.decrypt_file(source, destination, key=key)
+    assert destination.read_bytes() == b"preserve existing output"
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"]
+
+
+@pytest.mark.parametrize("damage", ["wrong_key", "truncated", "invalid_base64", "appended"])
+def test_invalid_files_never_publish_plaintext(tmp_path, damage):
+    key = Fernet.generate_key()
+    token = Fernet(key).encrypt(b"private data")
+    if damage == "wrong_key":
+        key = Fernet.generate_key()
+    elif damage == "truncated":
+        token = token[:-8]
+    elif damage == "invalid_base64":
+        token = b"!" + token[1:]
+    else:
+        token += b"AAAA"
+    source = tmp_path / "source"
+    destination = tmp_path / "destination"
+    source.write_bytes(token)
+    with pytest.raises(InvalidToken):
+        fernet_file.decrypt_file(source, destination, key=key)
+    assert not destination.exists()
+    assert list(tmp_path.iterdir()) == [source]
+
+
+def test_authenticated_invalid_padding_preserves_destination(tmp_path):
+    key = Fernet.generate_key()
+    token = bytearray(base64.urlsafe_b64decode(Fernet(key).encrypt(b"A" * 16)))
+    # The second block is all padding. This changes its final byte to zero.
+    token[25 + 15] ^= 16
+    token[-32:] = hmac.digest(base64.urlsafe_b64decode(key)[:16], token[:-32], hashlib.sha256)
+    source = tmp_path / "source"
+    destination = tmp_path / "destination"
+    source.write_bytes(base64.urlsafe_b64encode(token))
+    destination.write_bytes(b"previous")
+    with pytest.raises(InvalidToken):
+        fernet_file.decrypt_file(source, destination, key=key)
+    assert destination.read_bytes() == b"previous"
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"]
+
+
+def test_backup_restore_entrypoints_do_not_read_whole_files(tmp_path, monkeypatch):
+    key = Fernet.generate_key().decode()
+    data = os.urandom(fernet_file._CHUNK_BYTES * 3 + 17)
+    source = tmp_path / "backup.zip"
+    destination = tmp_path / "restored.zip"
+    source.write_bytes(data)
+    timestamp = 1_700_000_000
+    os.utime(source, (timestamp, timestamp))
+
+    with monkeypatch.context() as patch:
+        patch.setattr(Path, "read_bytes", lambda *args: pytest.fail("Whole-file read"))
+        encrypted = _encrypt_file_fernet(source, key=key)
+        _decrypt_encrypted_zip(encrypted, fernet_key=key, output_zip_path=destination)
+    assert encrypted.stat().st_mtime == timestamp
+    assert destination.read_bytes() == data
+    assert Fernet(key).decrypt(encrypted.read_bytes()) == data
+
+
+@pytest.mark.parametrize("operation", [fernet_file.encrypt_file, fernet_file.decrypt_file])
+def test_failed_publication_preserves_existing_output(tmp_path, monkeypatch, operation):
+    key = Fernet.generate_key()
+    source = tmp_path / "source"
+    destination = tmp_path / "destination"
+    data = b"private data"
+    source.write_bytes(Fernet(key).encrypt(data) if operation is fernet_file.decrypt_file else data)
+    destination.write_bytes(b"previous")
+
+    def fail_replace(*args):
+        raise OSError("simulated publication failure")
+
+    monkeypatch.setattr(fernet_file.os, "replace", fail_replace)
+    with pytest.raises(OSError, match="publication failure"):
+        operation(source, destination, key=key)
+    assert destination.read_bytes() == b"previous"
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["destination", "source"]
+
+
+@pytest.mark.parametrize("operation", [fernet_file.encrypt_file, fernet_file.decrypt_file])
+def test_same_path_is_rejected(tmp_path, operation):
+    source = tmp_path / "source"
+    source.write_bytes(b"keep")
+    with pytest.raises(ValueError, match="different files"):
+        operation(source, source, key=Fernet.generate_key())
+    assert source.read_bytes() == b"keep"