From 8cd8cc4b567ba8450c555e0a93064218016c5f70 Mon Sep 17 00:00:00 2001 From: PureTensor Date: Sat, 17 Oct 2026 19:32:52 +0100 Subject: [PATCH] v0.1.4: a new embedding model needs --rebuild instead of wiping the index `puremind index` with a changed embedder name or embedder_revision emptied the index and re-embedded every note. A `watch` left on the old config and a fresh `index` on the new one wiped each other in turn, with a full re-embed each time. - index_vault raises ModelMismatch (nothing changed, no probe embedded) when the model or revision differs from the one the index was built with and rebuild is not set. --rebuild keeps the probe-then-clear path. An index that holds no notes still takes a new model at once. - `puremind index` reports it with exit 2 and a --rebuild hint. - `puremind watch` stops on it, like a vault mismatch, with a hint to restart the watch, so a stale watch leaves the rebuilt index alone. - README, --rebuild help, CHANGELOG (0.1.4, unreleased) and a note at PINNED_REVISIONS that changing the pinned embedder revision now makes every existing index need --rebuild. Tests: the four tests that relied on the wipe now use rebuild=True and check the refusal; new tests cover the empty-index case, the CLI index refusal and the stale-watch scenario. The behavioural ones fail against the previous source. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011zd8G1cibJoQPVbZEpVLYs --- CHANGELOG.md | 4 ++++ README.md | 4 ++-- src/puremind_mac/__init__.py | 2 +- src/puremind_mac/cli.py | 23 +++++++++++++++------ src/puremind_mac/indexer.py | 24 +++++++++++++++------- src/puremind_mac/models.py | 4 +++- tests/test_config_cli.py | 12 +++++++++++ tests/test_indexer.py | 33 +++++++++++++++++++++++------ tests/test_watcher.py | 40 ++++++++++++++++++++++++++++++++++++ 9 files changed, 123 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b04621b..aa5687b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.1.4 (unreleased) + +- A new embedding model, or a new `embedder_revision`, now needs `puremind index --rebuild`. `puremind index` without it stops with an error and changes nothing; 0.1.3 and earlier emptied the index and embedded every note again. A `puremind watch` still running on the old configuration, and a fresh `puremind index` on the new one, used to wipe each other's index in turn, with a full re-embed each time. Now the stale watch stops with an error and a hint to restart it, and the index stays as it is. An index that holds no notes takes a new model without `--rebuild`. + ## 0.1.3 (2026-10-17) Fixes from a second review of 0.1.2. Upgrading from 0.1.2 embeds nothing again. The `v0.1.2` tag was made before these fixes, so an install from `main` between the two releases reports 0.1.2 but already has them. diff --git a/README.md b/README.md index 5d90ee6..e5b53a4 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The reranker costs about 0.2 seconds per candidate on a CPU, so with the default - **Model revisions.** The default models are pinned to `Qwen/Qwen3-Embedding-0.6B` at `97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3` and `BAAI/bge-reranker-v2-m3` at `953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e`. Another model uses its repository's default branch as it was at the first download, and is not refreshed afterwards unless you set a revision or clear the cache. - **Model names.** A name is a Hugging Face model id unless it starts with `/`, `./`, `../` or `~`, which mark a local directory. A directory in your working directory is never picked up by accident. Models load with `trust_remote_code` off. -- **Changing the embedding model** makes the next `puremind index` rebuild the index, since vectors from different models are not comparable. The new model must first load and embed a test passage, so a mistyped name leaves the existing index untouched. The index is then emptied, and searches find nothing until the run finishes. `puremind index --rebuild` with the same model, by contrast, replaces one note at a time, so searches keep working. +- **Changing the embedding model**, or `embedder_revision`, takes `puremind index --rebuild`, since vectors from different models are not comparable. Until then `puremind index` and `puremind watch` stop with an error and change nothing, and searches are refused. A `puremind watch` keeps the model it started with, so restart it after the change; one left running stops instead of wiping the rebuilt index. With `--rebuild`, the new model must first load and embed a test passage, so a mistyped name leaves the existing index untouched. The index is then emptied, and searches find nothing until the run finishes. `puremind index --rebuild` with the same model, by contrast, replaces one note at a time, so searches keep working. - **The reranker window.** 800 characters is the setting the benchmark below was measured with. Raising `rerank_chars` lets the reranker see more of each passage, at the cost of slower searches. - **Indexes from 0.1.1 or earlier** are brought up to date by the next `puremind index` or `puremind watch` run, which cuts and embeds every note again. It takes about as long as the first index did, and searches keep working meanwhile. @@ -113,7 +113,7 @@ The index is a second copy of your notes' text. It is created readable only by y | Command | What it does | |---|---| | `puremind init [--force]` | Writes `config.toml`. | -| `puremind index [vault_dir] [--rebuild]` | Indexes the vault incrementally; `--rebuild` embeds every note again. | +| `puremind index [vault_dir] [--rebuild]` | Indexes the vault incrementally; `--rebuild` embeds every note again, and is needed after changing the embedding model. | | `puremind watch [vault_dir] [--debounce SECONDS]` | Indexes, then re-indexes when notes change. | | `puremind search [--limit N] [--no-rerank] [--lexical-only]` | Searches from the terminal; `--limit` is 1 to 50. | | `puremind serve [--no-rerank]` | Runs the MCP server on stdio. Your agent starts this for you. | diff --git a/src/puremind_mac/__init__.py b/src/puremind_mac/__init__.py index 5b3150f..8a08aad 100644 --- a/src/puremind_mac/__init__.py +++ b/src/puremind_mac/__init__.py @@ -1,3 +1,3 @@ """pureMind for Mac: local memory for AI coding agents, served over MCP.""" -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/src/puremind_mac/cli.py b/src/puremind_mac/cli.py index 903ba97..62c238e 100644 --- a/src/puremind_mac/cli.py +++ b/src/puremind_mac/cli.py @@ -99,12 +99,15 @@ def cmd_init(args, cfg: Config) -> int: def cmd_index(args, cfg: Config) -> int: - from .indexer import index_vault + from .indexer import ModelMismatch, index_vault vault = _vault(cfg, args.vault_dir) cfg = dataclasses.replace(cfg, vault=vault) svc = _service(cfg) - stats = index_vault(vault, svc.store, svc.embedder, rebuild=args.rebuild) + try: + stats = index_vault(vault, svc.store, svc.embedder, rebuild=args.rebuild) + except ModelMismatch as exc: + return _fail(str(exc), "run `puremind index --rebuild` to embed every note again with the configured model") print(stats.summary()) for item in stats.skipped: print(f" skipped: {_one_line(item)}", file=sys.stderr) @@ -112,18 +115,20 @@ def cmd_index(args, cfg: Config) -> int: def cmd_watch(args, cfg: Config) -> int: - from .indexer import VaultMismatch, index_vault + from .indexer import ModelMismatch, VaultMismatch, index_vault from .watcher import watch vault = _vault(cfg, args.vault_dir) svc = _service(dataclasses.replace(cfg, vault=vault)) stop = threading.Event() - refused: list[VaultMismatch] = [] + refused: list[ValueError] = [] def run() -> None: # always called on this thread, one run at a time try: stats = index_vault(vault, svc.store, svc.embedder) - except VaultMismatch as exc: # a symlinked vault now names another directory: every run would fail + except (VaultMismatch, ModelMismatch) as exc: + # A symlinked vault that now names another directory, or an index rebuilt for another + # model since this watch read its config: every later run would fail the same way. refused.append(exc) stop.set() return @@ -137,6 +142,12 @@ def run() -> None: # always called on this thread, one run at a time finally: svc.store.close() if refused: + if isinstance(refused[0], ModelMismatch): + return _fail( + str(refused[0]), + "`puremind watch` keeps the model it started with: restart it if the model has changed " + "since, or else run `puremind index --rebuild`", + ) raise refused[0] return 0 @@ -381,7 +392,7 @@ def build_parser() -> argparse.ArgumentParser: s = sub.add_parser("index", help="index the vault (incremental)") s.add_argument("vault_dir", nargs="?") s.add_argument( - "--rebuild", action="store_true", help="embed every note again (searches keep working meanwhile)" + "--rebuild", action="store_true", help="embed every note again; needed after changing the embedding model" ) s.set_defaults(fn=cmd_index) diff --git a/src/puremind_mac/indexer.py b/src/puremind_mac/indexer.py index 69fed38..e2f93e3 100644 --- a/src/puremind_mac/indexer.py +++ b/src/puremind_mac/indexer.py @@ -22,7 +22,7 @@ frontmatter_end, ) from .chunker import VERSION as CHUNKER_VERSION -from .models import Embedder, ModelLoadError, embedder_revision, model_key +from .models import Embedder, ModelLoadError, embedder_revision, model_key, model_label from .store import Store, StoreError log = logging.getLogger("puremind.index") @@ -45,6 +45,11 @@ class VaultMismatch(ValueError): """The index belongs to another vault directory. Nothing was changed.""" +class ModelMismatch(ValueError): + """The index was built with another embedding model or revision, and no rebuild was asked for. + Nothing was changed.""" + + @dataclass class IndexStats: scanned: int = 0 @@ -419,10 +424,11 @@ def index_vault( """Bring the index in line with the vault. ``rebuild`` embeds every note again, replacing each one's rows in turn, so searches keep - finding the old version meanwhile. A new embedding model, or a new revision of it, discards - the index first, since vectors from two models cannot be searched together. Notes cut by an - earlier chunker are cut and embedded again. ``should_stop`` is polled between notes; once it - returns True the run ends early without removing anything. + finding the old version meanwhile. A new embedding model, or a new revision of it, is refused + with ``ModelMismatch`` unless ``rebuild`` is set, and then discards the index first, since + vectors from two models cannot be searched together. Notes cut by an earlier chunker are cut + and embedded again. ``should_stop`` is polled between notes; once it returns True the run + ends early without removing anything. """ vault = vault.expanduser().resolve() if not vault.is_dir(): @@ -441,9 +447,13 @@ def index_vault( changed = known_model is not None and ( model_key(known_model) != model_key(embedder.name) or (known_rev is not None and known_rev != revision) ) + if changed and store.stats()["files"]: + if not rebuild: # else a `watch` left on the old config and a fresh `index` wipe each other in turn + built = model_label(known_model) + (f" at revision {known_rev}" if known_rev else "") + wanted = model_label(embedder.name) + (f" at revision {revision}" if revision else "") + raise ModelMismatch(f"the index was built with {built} but {wanted} is configured; nothing was changed") + _embed(embedder, [PROBE_TEXT]) # a model that cannot run must not cost the user an index if changed: - if store.stats()["files"]: - _embed(embedder, [PROBE_TEXT]) # a model that cannot run must not cost the user an index log.warning( "embedder changed (%s%s -> %s%s); rebuilding the index", known_model, f" at {known_rev}" if known_rev else "", embedder.name, f" at {revision}" if revision else "", diff --git a/src/puremind_mac/models.py b/src/puremind_mac/models.py index f65ccda..f8a76d2 100644 --- a/src/puremind_mac/models.py +++ b/src/puremind_mac/models.py @@ -39,7 +39,9 @@ log = logging.getLogger("puremind.models") -# Revision used when none is configured for a default model. +# Revision used when none is configured for a default model. A new embedder revision here makes +# `index` and `watch` refuse every existing index until `puremind index --rebuild`, so a release +# that changes it must say so. PINNED_REVISIONS = {DEFAULT_EMBEDDER: DEFAULT_EMBEDDER_REVISION, DEFAULT_RERANKER: DEFAULT_RERANKER_REVISION} # What the loaders read: configuration, tokenizer files and safetensors weights (never pickle files). # A download stays at the weights the models need even when a repository also ships other formats. diff --git a/tests/test_config_cli.py b/tests/test_config_cli.py index 2388b93..9c23c1b 100644 --- a/tests/test_config_cli.py +++ b/tests/test_config_cli.py @@ -612,6 +612,18 @@ def test_cli_exit_codes(home, vault, tmp_path, capsys, fake_models): assert excinfo.value.code == 2 +def test_cli_index_refuses_a_new_model_until_rebuild_is_given(home, vault, capsys, fake_models, embedder): + assert run("index", str(vault)) == 0 + embedder.name = "another-model" + capsys.readouterr() + assert run("index", str(vault)) == 2 + err = capsys.readouterr().err + assert "error: the index was built with fake-embedder but another-model is configured; nothing was changed" in err + assert "hint: run `puremind index --rebuild`" in err + assert run("index", "--rebuild", str(vault)) == 0 + assert "3 added" in capsys.readouterr().out + + @pytest.mark.parametrize("value", ["0", "-1", "51", "1000", "x", "1.5", ""]) def test_search_limit_must_be_a_whole_number_from_1_to_50(home, capsys, value): with pytest.raises(SystemExit) as excinfo: diff --git a/tests/test_indexer.py b/tests/test_indexer.py index 8ef69b3..d86ed87 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -16,6 +16,7 @@ EMBED_BATCH, MAX_PATH_CHARS, EmbedderError, + ModelMismatch, discover, index_vault, title_of, @@ -123,14 +124,29 @@ def test_notes_cut_by_an_earlier_chunker_are_cut_again_once(vault, store, embedd assert index_vault(vault, store, embedder).unchanged == 3 -def test_changing_the_embedder_triggers_a_rebuild(vault, store, embedder): +def test_a_new_embedder_is_refused_without_rebuild(vault, store, embedder): index_vault(vault, store, embedder) - embedder.name = "another-model" - stats = index_vault(vault, store, embedder) + before, calls = store.stats(), embedder.doc_calls + embedder.name = "another-model" # a `watch` on the old config and a fresh `index` must not wipe each other + with pytest.raises(ModelMismatch, match="built with fake-embedder but another-model is configured; nothing was"): + index_vault(vault, store, embedder) + assert store.stats() == before and store.get_meta("embedder") == "fake-embedder" + assert embedder.doc_calls == calls # refused before anything is embedded, the probe included + stats = index_vault(vault, store, embedder, rebuild=True) assert stats.added == 3 assert store.get_meta("embedder") == "another-model" +def test_an_index_with_no_notes_takes_a_new_embedder_without_rebuild(tmp_path, store, embedder): + empty = tmp_path / "empty" + empty.mkdir() + index_vault(empty, store, embedder) # records the model, holds nothing that could be lost + (empty / "a.md").write_text("# A\n\nhello\n") + embedder.name = "another-model" + assert index_vault(empty, store, embedder).added == 1 + assert store.get_meta("embedder") == "another-model" + + def test_a_different_vault_cannot_share_an_index(vault, store, embedder, tmp_path): index_vault(vault, store, embedder) other = tmp_path / "other" @@ -515,7 +531,10 @@ def test_a_new_embedder_revision_embeds_every_note_again(vault, store, embedder) index_vault(vault, store, embedder) assert index_vault(vault, store, embedder).unchanged == 3 and store.get_meta("embedder_revision") == "rev-a" embedder.revision = "rev-b" # vectors of two revisions cannot be searched together - stats = index_vault(vault, store, embedder) + with pytest.raises(ModelMismatch, match="at revision rev-a but fake-embedder at revision rev-b"): + index_vault(vault, store, embedder) + assert store.get_meta("embedder_revision") == "rev-a" and store.stats()["files"] == 3 + stats = index_vault(vault, store, embedder, rebuild=True) assert stats.added == 3 and stats.unchanged == 0 and store.get_meta("embedder_revision") == "rev-b" @@ -571,8 +590,10 @@ def test_a_model_that_cannot_run_does_not_destroy_the_index(vault, store, embedd index_vault(vault, store, embedder) before = store.stats() typo = _Wrapped(embedder, _boom, name="Qwen/Qwen3-Embeding-0.6B") # a mistyped model name - with pytest.raises(EmbedderError, match="could not load the model"): + with pytest.raises(ModelMismatch): index_vault(vault, store, typo) + with pytest.raises(EmbedderError, match="could not load the model"): + index_vault(vault, store, typo, rebuild=True) assert store.stats() == before and store.get_meta("embedder") == "fake-embedder" with pytest.raises(EmbedderError): index_vault(vault, store, _Wrapped(embedder, _boom), rebuild=True) @@ -751,7 +772,7 @@ def test_a_rebuild_leaves_nothing_of_the_old_index_in_the_files(vault, private_i (vault / "secret.md").unlink() if how == "new model": embedder.name = "another-model" - stats = index_vault(vault, private_index, embedder, rebuild=how == "rebuild") + stats = index_vault(vault, private_index, embedder, rebuild=True) assert stats.added + stats.updated == 3 assert _copies(home, b"zqxcanary7f3a9") == 0 diff --git a/tests/test_watcher.py b/tests/test_watcher.py index d09f536..2368e43 100644 --- a/tests/test_watcher.py +++ b/tests/test_watcher.py @@ -473,6 +473,46 @@ def two_runs(vault, run_index, delay, stop): assert "error: this index belongs to" in capsys.readouterr().err +def test_a_stale_puremind_watch_stops_instead_of_wiping_an_index_rebuilt_for_another_model( + vault, tmp_path, monkeypatch, capsys +): + import numpy as np + + from puremind_mac import cli + from puremind_mac.indexer import index_vault + from puremind_mac.store import Store + + home = tmp_path / "home" + monkeypatch.setenv("PUREMIND_HOME", str(home)) + monkeypatch.delenv("PUREMIND_DB", raising=False) + + def stub(name): + return SimpleNamespace(name=name, embed_documents=lambda texts: np.ones((len(texts), 4), dtype="float32")) + + monkeypatch.setattr("puremind_mac.service.SentenceTransformerEmbedder", lambda *a, **k: stub("old-model")) + + def two_runs(vault, run_index, delay, stop): + run_index() + store = Store(home / "index.sqlite") + try: # `puremind index --rebuild` with the new config, while this watch still has the old one + index_vault(vault, store, stub("new-model"), rebuild=True) + finally: + store.close() + run_index() + assert stop.is_set() + + monkeypatch.setattr("puremind_mac.watcher.watch", two_runs) + assert cli.main(["watch", str(vault)]) == 2 + err = capsys.readouterr().err + assert "error: the index was built with new-model but old-model is configured; nothing was changed" in err + assert "hint: `puremind watch` keeps the model it started with: restart it if the model has changed" in err + store = Store(home / "index.sqlite") + try: + assert store.get_meta("embedder") == "new-model" and store.stats()["files"] == 3 + finally: + store.close() + + def test_puremind_watch_names_a_missing_vault(tmp_path, monkeypatch, capsys): from puremind_mac import cli