release: prepare v0.22.0-rc.1 storage and MCP profiles

.github/workflows/ci.yml

@@ -113,3 +113,15 @@ jobs:
           npm run build
           npx tsc --noEmit
           git diff --exit-code -- main.js
+
+  windows-storage:
+    name: Windows storage + recovery
+    runs-on: windows-2025
+    steps:
+      - uses: actions/checkout@v7
+      - name: Select Rust toolchain
+        run: |
+          rustup toolchain install 1.88.0 --profile minimal
+          rustup default 1.88.0
+      - name: Test storage and maintenance
+        run: cargo test --locked --test nosync_layout --test database_maintenance

.github/workflows/release.yml

@@ -51,6 +51,9 @@ jobs:
           package_version="$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -n 1)"
           test "$tag_version" = "$package_version"
 
+      - name: Test storage and maintenance
+        run: cargo test --locked --test nosync_layout --test database_maintenance
+
       - name: Build release binary
         env:
           RUSTFLAGS: -D warnings
@@ -141,15 +144,19 @@ jobs:
           case "$GITHUB_REF_NAME" in
             *-*) release_flags+=(--prerelease) ;;
           esac
+          notes_flags=(--generate-notes)
+          if [ -f "docs/releases/$GITHUB_REF_NAME.md" ]; then
+            notes_flags=(--notes-file "docs/releases/$GITHUB_REF_NAME.md")
+          fi
           if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then
             gh release upload "$GITHUB_REF_NAME" dist/* --clobber
           else
             gh release create "$GITHUB_REF_NAME" dist/* \
-              --title "Noema $GITHUB_REF_NAME" --generate-notes "${release_flags[@]}"
+              --title "Noema $GITHUB_REF_NAME" "${notes_flags[@]}" "${release_flags[@]}"
           fi
 
       - name: Update Homebrew tap
-        if: env.HOMEBREW_TAP_TOKEN != ''
+        if: env.HOMEBREW_TAP_TOKEN != '' && !contains(github.ref_name, '-')
         env:
           GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
           HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "noema"
-version = "0.21.9"
+version = "0.22.0-rc.1"
 edition = "2024"
 rust-version = "1.88"
 description = "The intentional memory layer for your AI agents"

README.md

@@ -200,6 +200,12 @@ noema get 20260329-we-chose-local-sqlite
 noema init --name <name> [--path <dir>]   Create a new Cortex
 noema use <name>                          Set the default Cortex
 noema cortex list                         List all known Cortexes
+noema cortex storage <name> [--json]      Inspect database, WAL, and reusable-space statistics
+noema cortex storage <name> --database <default|nosync> --backup <path>
+                                          Safely change database storage with a backup
+noema cortex storage <name> --resume       Finish an interrupted storage migration
+noema cortex compact <name> --backup <path> [--json]
+                                          Compact a stopped database with backup and integrity checks
 noema cortex remove <name> [--purge] [--force]
                                           Unregister a Cortex (--purge also deletes its directory)
 noema cortex backup <name> [-o <path>] [--force]
@@ -346,6 +352,22 @@ noema version                             Print version, commit, and build date
 2. `NOEMA_CORTEX` environment variable
 3. Default set via `noema use <name>`
 
+### Database storage in iCloud Drive
+
+Use opt-in `nosync` storage to keep the database and its SQLite sidecar files in
+`db.nosync/` while Markdown traces continue syncing. The storage command creates
+a backup and safely moves an existing database. See
+[database storage](docs/database-storage.md) for enabling, reversing, recovering,
+and backing up this layout, including iCloud's parent-folder eviction limits.
+
+### Database maintenance
+
+`noema cortex storage <name>` reports database size, reusable free space, WAL size,
+and compaction headroom. Add `--json` for structured output. For deliberate
+compaction, stop clients and use `noema cortex compact <name> --backup <archive>`.
+Noema requires a complete external backup and verifies database integrity before
+and after compaction. See [database maintenance](docs/database-maintenance.md).
+
 ### Durability profiles
 
 Noema defaults to the `standard` durability profile. It matches the mutation
@@ -373,6 +395,16 @@ for measurements and the full trade-off.
 
 ## Agent Integrations
 
+MCP read-only hints describe the purpose of a tool. Retrieval tools such as
+`get_trace`, `search_traces`, `recall_context`, and `find_similar_traces` advertise
+`readOnlyHint: true` while retaining Noema's built-in access and search-hit
+tracking. These signals help Noema assess memory importance and are part of
+retrieval, not a separate operation agents must remember to perform. The existing
+`record_usage` argument controls explicit read counts; it does not change the
+read hint or disable automatic search-hit tracking. Read hints do not promise
+zero database writes: internal metrics and metrics retention also continue.
+Tools that can explicitly change trace content, tags, or lifecycle remain writes.
+
 `noema integrate` connects a Cortex to supported coding agents and installs the
 small startup bootstrap that tells each agent to call `get_instructions`. An MCP
 entry without that bootstrap is callable but not reliably memory-aware after a
@@ -555,10 +587,29 @@ Noema can run as an [MCP](https://modelcontextprotocol.io) server, giving any MC
 
 Call `get_instructions` first in any new agent session for concise Markdown guidance. Use `cortex_usage` when a client needs structured JSON context. MCP tool discovery and each tool's schema remain the authoritative callable tool reference.
 
-Set `NOEMA_MCP_TOOL_PROFILE=continuity-read` on an MCP server process to expose only
-`recall_context`, or `NOEMA_MCP_TOOL_PROFILE=continuity-capture` to expose only
-`get_instructions` and `create_traces`. These opt-in profiles reduce model-facing tool schemas for
-bounded continuity tasks; the default remains the full tool set.
+HTTP servers expose the full tool set at `/mcp` and optional role endpoints:
+
+| Endpoint | Tool set |
+| --- | --- |
+| `/mcp` | All tools, including future additions; existing clients remain compatible |
+| `/mcp/agent` | Everyday memory retrieval, capture, editing, tagging, voting, and lifecycle tools |
+| `/mcp/maintainer` | Agent tools plus tag cleanup and operational diagnostics |
+| `/mcp/curator` | Agent tools plus consolidation candidates and distilled-memory creation |
+| `/mcp/federation` | Cortex identity, event exchange, and usage-signal exchange |
+
+All endpoints share the same cortex and usage tracking. Role endpoints enforce
+explicit tool allowlists for both discovery and execution; they share the existing
+authentication and are not separate authorization roles. Existing clients and
+federation peers can keep using `/mcp`. See [MCP tool endpoints](docs/mcp-endpoints.md)
+for exact tool sets and configuration.
+
+For **stdio** servers, `NOEMA_MCP_TOOL_PROFILE` accepts `full` (the default),
+`agent`, `maintainer`, `curator`, or `federation`. The existing bounded profiles
+remain available: `continuity-read` exposes only `recall_context`, and
+`continuity-capture` exposes only `get_instructions` and `create_traces`.
+HTTP servers reject a restricted `NOEMA_MCP_TOOL_PROFILE` at startup; unset it
+and choose a role URL instead. This prevents an existing restricted configuration
+from silently widening access when `/mcp` becomes the full interface.
 
 ### stdio (Claude Desktop, Claude Code, any MCP client)
 

docs/database-maintenance.md

@@ -0,0 +1,86 @@
+# Database maintenance
+
+Noema exposes storage diagnostics and explicit compaction for both `db/` and
+`db.nosync/`. It does not schedule full vacuuming or change SQLite's auto-vacuum
+setting automatically.
+
+## Inspect storage
+
+```sh
+noema cortex storage mycortex
+noema cortex storage mycortex --json
+```
+
+Diagnostics can run while Noema clients are active. They read database statistics
+without running cortex startup, rebuilding search, pruning records, or requesting
+a WAL checkpoint. A missing database is reported rather than created.
+
+The report includes:
+
+- The selected layout and database path.
+- Physical database file size and logical size, including pages represented in WAL.
+- Completely free pages, their byte size, and their percentage of the database.
+- WAL file size and the current auto-vacuum mode.
+- Available space on the database filesystem and conservative compaction headroom.
+
+Free pages remain available for future database writes. Their total is not an exact
+prediction of compaction savings: a full vacuum can also repack partially filled
+pages. A large WAL is a separate issue from free space in the main database.
+Statistics are a snapshot; file sizes can change while other clients write.
+JSON size fields use bytes and `reusable_percent` uses the range 0–100.
+
+## Compact explicitly
+
+Stop all servers, watchers, and agent connections using the cortex, then choose
+a new backup archive outside its directory:
+
+```sh
+noema cortex compact mycortex --backup /path/to/backups/before-compact.tar.gz
+```
+
+Add `--json` for a structured result containing `before`, `after`, `backup_path`,
+and `reclaimed_bytes`. The ordinary output reports before/after file sizes, free
+space, the final WAL size, and the backup location. `reclaimed_bytes` measures
+reduction of the main database file, not changes in the WAL.
+
+The command:
+
+1. Acquires the exclusive Noema storage lock and opens the existing database.
+2. Checks database integrity and available space.
+3. Checkpoints and closes SQLite while retaining the Noema maintenance lock, then
+   creates a complete cortex backup. Existing backup files
+   are never overwritten, and backup paths inside the cortex are refused.
+4. Reopens SQLite, rechecks space after creating the backup, then runs `VACUUM`.
+5. Checks integrity again, checkpoints/truncates the WAL, and reports the result.
+
+Supported Noema clients prevent compaction while their database connection is
+open. Older clients and direct SQLite tools must also be stopped; they do not
+participate in Noema's process lock. Compaction does not stop or restart services
+for you. Choose a maintenance window and restart clients after success.
+
+The space check conservatively requires twice the larger of the logical database
+size and the main file size to be available on the database filesystem, in addition
+to the space used by the backup. SQLite may also use temporary storage; ensure
+that filesystem has room if configured separately. Space checks cannot reserve
+capacity against other applications writing concurrently.
+
+Compaction preserves retained traces, event history, embeddings, federation state,
+and the selected storage layout. It does not implement retention or remove old
+events, and it does not rewrite Markdown files or configuration.
+
+## Interrupted or unsuccessful compaction
+
+Noema uses SQLite's transactional `VACUUM`; it does not replace the database file
+with a separately generated copy. SQLite's normal journal/WAL recovery applies
+after a process interruption. Leave the database and its sidecars together.
+
+The complete backup is written before vacuuming begins. If a later step fails,
+the error identifies the retained backup path. After stopping competing clients
+or resolving disk-space errors, verify the cortex and retry using a new backup
+filename. There is no compaction-specific `--resume` journal. The existing
+`noema cortex restore` workflow can restore the backup if recovery is needed.
+
+A full vacuum rewrites the database and can generate substantial I/O. Consider it
+after large deletions or when diagnostics show enough unused space to justify a
+maintenance window. A fixed calendar interval is not required for normal use.
+See [SQLite's VACUUM documentation](https://www.sqlite.org/lang_vacuum.html).

docs/database-storage.md

@@ -0,0 +1,113 @@
+# Keeping the database local in iCloud Drive
+
+For size diagnostics and explicit compaction, see [database maintenance](database-maintenance.md).
+
+Noema normally stores its SQLite database in `db/` inside each cortex. Database
+writes, including background federation bookkeeping, can cause frequent iCloud
+uploads even when no Markdown traces change. Opt-in `nosync` storage moves the
+whole database directory to `db.nosync/`. Traces remain in their usual folders.
+
+## Enable or disable
+
+Stop every Noema server, watcher, and agent connection using this cortex before
+changing storage. Upgrade all clients that will open it. New clients enforce a
+local process lock outside the cortex, under `noema/storage-locks/` in
+`XDG_RUNTIME_DIR` (or the operating system's temporary directory). Keep that
+runtime directory local and consistent across clients. The lock uses the
+canonical cortex path, so path aliases share the same lock. Replacing the
+compatibility lock inside a synced cortex cannot bypass this runtime lock.
+
+Clients also retain the previous cortex-local lock for upgrade compatibility.
+Restart existing clients to gain the runtime-lock protection; versions predating
+storage locking must be stopped manually. Locks do not coordinate different
+computers over iCloud.
+
+This is an offline maintenance operation. Stop direct SQLite tools and older
+clients too, and prevent automatic restarts until migration finishes. A successful
+initial checkpoint does not reserve the database against a new, nonparticipating
+SQLite writer. Noema's process lock coordinates compatible Noema clients; it
+cannot guarantee safe directory migration while another program ignores that
+lock. The required backup captures the state before migration, not subsequent
+writes by an unsupported concurrent client.
+
+Choose a new backup filename outside the cortex, preferably outside iCloud Drive:
+
+```sh
+noema cortex storage mycortex
+noema cortex storage mycortex --database nosync --backup /path/to/backups/before-nosync.tar.gz
+```
+
+The command checks database integrity, checkpoints the WAL, writes a complete
+backup, and moves the database directory. It preserves event history, federation
+state, embeddings, pending recovery records, and files in the database directory.
+It refuses conflicting database directories and never overwrites an existing
+backup. Restart your Noema clients after it succeeds.
+
+To return to the original layout, stop clients and run:
+
+```sh
+noema cortex storage mycortex --database default --backup /path/to/backups/before-default.tar.gz
+```
+
+Reversing the setting makes the database eligible for iCloud syncing again.
+Repeating an already completed setting is a no-op. A previously renamed
+`db.nosync/` directory can be adopted using the same enable command and backup.
+
+## Configuration and compatibility
+
+The command maintains `storage.yaml` in the cortex root:
+
+```yaml
+database: nosync # default or nosync
+```
+
+Use the command to change an existing cortex; editing this field alone does not
+move the database. Normal cortexes without this file continue using `db/`.
+The command preserves other YAML keys, comments, and their order; the `database`
+entry must use a top-level `database:` line for command-based editing.
+On Unix, existing `storage.yaml` permissions are preserved, including when
+resuming an interrupted migration; a new file defaults to `0640`.
+It leaves `cortex.md` and the global registry untouched.
+
+With `nosync` enabled, `db/noema.db` is a small, static compatibility guard, not a
+second database. Older Noema versions fail to open it instead of creating an
+empty database. Do not delete or replace this guard. Use a compatible version
+to reverse the migration before downgrading. Do not rename the WAL or SHM files
+individually, or replace the database directory with a symlink.
+
+## Interrupted migrations and backups
+
+An interrupted migration blocks normal database access. After stopping clients,
+finish it with:
+
+```sh
+noema cortex storage mycortex --resume
+```
+
+Resume completes the original direction. To undo it, finish the interrupted
+migration and then run the command for the opposite storage mode. The original
+backup is also available through Noema's ordinary cortex restore workflow.
+Do not delete the migration journal to bypass recovery.
+
+`noema cortex backup` includes `db.nosync`, its recovery files, the configuration,
+and the compatibility guard. Restore preserves this layout. Keep complete Noema
+backups: Markdown alone does not preserve the database's event and federation
+state. Historical event records may name a recovery artifact under the previous
+directory prefix; the artifact moves with the database directory.
+
+## iCloud scope
+
+Apple documents `.nosync` as an exclusion from iCloud transfer. It also documents
+that deleting or evicting a parent directory removes its `.nosync` children.
+Keep the cortex's parent folder downloaded and maintain backups outside iCloud.
+See [Apple's iCloud storage documentation](https://developer.apple.com/library/archive/documentation/General/Conceptual/iCloudDesignGuide/Chapters/iCloudFundametals.html).
+
+This setting does not disable Markdown syncing, change SQLite's WAL behavior,
+or promise exclusion from Dropbox, OneDrive, or other providers. The database
+can continue changing locally while iCloud ignores its directory.
+
+On another device, iCloud can deliver the setting and guard without the excluded
+database. Noema reports the missing local database rather than silently creating
+one. Restore a complete Noema backup to establish that device's local copy;
+do not treat iCloud as replication for the database or run copied cortex
+identities as independent federation peers.