diff --git a/README.md b/README.md index af10e5cd..ec4abc38 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,34 @@ # Alden +## 2026-10-11 개발 브랜치 — 문서 한 건의 통합 증거 대사 + +[구현·스코프·복구·CI 상세](docs/architecture/alden-pipeline-readback-20261011.md). +`codex/alden-pipeline-readback-20261011`은 PR #28 다음의 별도 검토 브랜치입니다. +읽기 전용 `alden_knowledge_trace`와 앱 `collection-trace` 경로는 원본 +문서 ID, 허용 프로젝트 및 선택적 대상/버전을 받아 확정된 수집 단계·해시· +FTS5·파생 Dense 벡터·그래프 관계를 개별 검증합니다. 3D 노드 상세에서도 +현재 노드/버전/대상이 일치할 때만 그 결과가 표시됩니다. 조회로 수집 +이벤트나 가상의 3D 활동을 생성하지 않습니다. 외부 osk-system MCP 연결은 +여전히 별개입니다. + +같은 프로젝트의 서로 다른 등록 대상에 원본 ID가 중복될 경우 GraphRAG는 +선택한 `target_id`의 현재 버전·검색 후보·관계 근거를 일관되게 사용합니다. +허용 프로젝트 전체 검색은 기본값으로 유지됩니다. 대상별 검색·MCP 범위 격리는 +[2026-10-11 테스트](docs/architecture/alden-pipeline-readback-20261011.md)에 기록했습니다. + +원본이 A→삭제→A로 복귀해도 기존 run_id가 충돌하여 복구를 건너뛰지 않도록 +스냅샷 원본 순서를 실행 정체성에 포함했습니다. 동일 순서 재요청은 멱등성을 +유지하고 과거 버전·관계 근거를 보존합니다. GitHub macOS CI +[38067729712](https://github.com/twoimo/alden/actions/runs/38067729712)에서 +Python 112개·UI 30개·TS/Vite 빌드가 통과했습니다. 원 사용자 Mac 설치, +실물 MLX/음성·osk-system, 서명/공증과 프로덕션은 미검증입니다. + +사용 가능한 저장소와 명시적 프로젝트 권한이 있을 때 자체 MCP +`alden_knowledge_trace`의 입력은 +`{"projects":["career"],"document_id":"graph:<실제 64자리 해시>"}`입니다. +`expected_version`과 `target_id`를 추가해 화면에 표시된 정확한 원본만 +확인할 수 있습니다. 외부 도구/워크스페이스 ID는 임의로 대체하지 않습니다. + ## Alden 0.3.55 — 실제 관계와 저장 벡터를 참고한 배치 큰 그래프의 출처별 고정 배치를 실제 관계와 별도의 의미 유사도 후보를 사용하는 배치로 연결했습니다. 현재 대상·버전·권한·원문·텍스트·E5 프로필이 일치하는 저장 벡터만 읽으며, 새 모델 추론이나 사실 관계 생성은 수행하지 않습니다. 배치는 전용 작업 스레드에서 계산하고, 표시 노드 중 물리 계산 밖의 노드에도 목표 위치를 보간합니다. 선택·카메라와 무관한 기존 위치를 유지하며 오래된 범위의 결과를 버립니다. diff --git a/desktop/DESIGN.md b/desktop/DESIGN.md index 1aea0e85..ec0eebcd 100644 --- a/desktop/DESIGN.md +++ b/desktop/DESIGN.md @@ -1,5 +1,24 @@ # Alden desktop design contract +## 2026-10-11 — Evidence in the real 3D note view (source branch) + +The selected source detail retains the calm existing note layout and a single +plain-text proof line inside its established original-evidence disclosure. +It checks the selected node ID, current source version and source target against +a new read-only backend receipt before showing stored source hash, committed +journal stage, FTS5 binding or derived local Dense vector binding. Missing or +lagging stages say "미확인/대기". The line never claims a rendered GPU frame, +a loaded MLX model, or a strengthened fact relationship. Collection events and +visual pulses still originate exclusively in persisted successful journal +activity; opening a note or reading trace produces none. + +[Scoped pipeline proof and bounded test receipts](../docs/architecture/alden-pipeline-readback-20261011.md) +include the A→removal→A rollback defect and its fixed run identity. This UI +source integration is validated on a macOS CI DOM/Vite harness, not yet the +owner's physical installed Alden window. The nine existing Archify diagrams +remain dated descriptions of their own verified source versions; a new Archify +render of this extension is pending a supported local Archify host. + ## 0.3.49 구현 경로 도식 [현재 9개 Archify 도식](../docs/architecture/alden-0.3.49/README.md)은 프로세스·음성/취소·수집·계보·MCP·정규화/검색·이력/활동·복구·평가 경로를 설명합니다. 작성 문구는 한국어이며 고정 Viewer UI는 영어입니다. 9/9 구성 검사, 네 데스크톱 크기의 브라우저 검사와 밝은/어두운 이미지 검토를 제품의 실제 실행 증거와 구분합니다. diff --git a/desktop/src-tauri/src/python_bridge.rs b/desktop/src-tauri/src/python_bridge.rs index 972043fc..ad046aa0 100644 --- a/desktop/src-tauri/src/python_bridge.rs +++ b/desktop/src-tauri/src/python_bridge.rs @@ -1198,6 +1198,7 @@ impl PythonBridge { | "db-sync-history" | "collection-history" | "collection-graph" + | "collection-trace" | "collection-affinity" | "collection-projects" | "collection-scheduler-status" @@ -2328,6 +2329,7 @@ fn settings_action_args( | "db-sync-history" | "collection-history" | "collection-graph" + | "collection-trace" | "collection-affinity" | "collection-projects" | "reply-history" @@ -5467,6 +5469,44 @@ mod tests { } } + #[test] + fn collection_trace_dispatches_as_read_only_bounded_action() { + let query = r#"{"projects":["research"],"document_id":"youtube:document-hash","target_id":"declared-target"}"#; + let args = settings_action_args( + "collection-trace", + Some(query), + None, + None, + None, + None, + None, + ) + .unwrap(); + assert!(args + .windows(2) + .any(|pair| pair == ["--history-query", query])); + assert!(args + .windows(2) + .any(|pair| pair == ["--action", "collection-trace"])); + assert!(!args.iter().any(|arg| { + matches!(arg.as_str(), "--explicit-opt-in" | "--model" | "--catalog-delete") + })); + for invalid in ["[]".to_owned(), "{broken".to_owned(), "x".repeat(4097)] { + assert!(matches!( + settings_action_args( + "collection-trace", + Some(&invalid), + None, + None, + None, + None, + None + ), + Err(BridgeError::ActionNotAllowed) + )); + } + } + #[test] fn room_upsert_action_args_and_sanitization() { let args = settings_action_args( diff --git a/desktop/src/__tests__/node-details.test.ts b/desktop/src/__tests__/node-details.test.ts index d3ec6114..4a85c32a 100644 --- a/desktop/src/__tests__/node-details.test.ts +++ b/desktop/src/__tests__/node-details.test.ts @@ -50,6 +50,28 @@ describe('selected-node explanations',()=>{ const collection=parseKnowledgeGraph({nodes:[{id:'group',label:'대화 맥락',category:'collection',is_hub:true},...graph.nodes],edges:[{source:'group',target:id,relation:'linked',purpose:'navigation'}]}); expect(nodeSummary(collection,collection.nodes[0])).toContain('입구 1개'); }); + it('shows stored source-to-index proof only for the exact selected source and target',()=>{ + const documentId='youtube:'+'a'.repeat(64),version='version:'+'b'.repeat(64),target='target-one'; + const view=parseKnowledgeGraph({nodes:[{id:documentId,label:'저장된 영상',category:'video', + source_version:version,source_target:target}],edges:[]}); + const details={node_id:documentId,basis:'source_record',version,target_id:target,platform:'youtube', + raw_sha256:'c'.repeat(64),body:'원본에 접근한 본문'}; + const pipeline_trace={ok:true,state:'available',document_id:documentId,version,target_id:target, + stages:{discovered:true,parsed:true,validated:true,stored:true,indexed:true}, + source:{state:'hash_verified'},fts:{state:'verified'}, + dense:{state:'stored_vector_binding_verified',model_residency_verified:false}}; + renderNodeDetails(view,view.nodes[0],{details,pipeline_trace}); + expect(document.getElementById('knowledge-node-evidence')!.textContent).toContain('저장·원본 해시 확인'); + expect(document.getElementById('knowledge-node-evidence')!.textContent).toContain('로컬 Dense 저장 바인딩 확인'); + expect(document.getElementById('knowledge-node-evidence')!.textContent).toContain('실제 모델 로딩 및 화면 렌더링은 별도 확인'); + expect(document.querySelector('script,img')).toBeNull(); + renderNodeDetails(view,view.nodes[0],{details,pipeline_trace:{...pipeline_trace,target_id:'another'}}); + expect(document.getElementById('knowledge-node-evidence')!.textContent).not.toContain('저장·원본 해시 확인'); + renderNodeDetails(view,view.nodes[0],{details,pipeline_trace:{...pipeline_trace, + fts:{state:'missing_or_mismatched'},dense:{state:'not_indexed'}}}); + expect(document.getElementById('knowledge-node-evidence')!.textContent).toContain('FTS 미확인'); + expect(document.getElementById('knowledge-node-evidence')!.textContent).toContain('Dense 미확인/대기'); + }); it('keeps a room node scoped to its own account and room even without an explicit detail scope',()=>{ const roomId=`chat:kakao:${account}:room:42`; const roomGraph=parseKnowledgeGraph({nodes:[{id:roomId,label:'회의',category:'채팅방'}],edges:[]}); diff --git a/desktop/src/knowledge/node-details.ts b/desktop/src/knowledge/node-details.ts index 02a797f1..460e0362 100644 --- a/desktop/src/knowledge/node-details.ts +++ b/desktop/src/knowledge/node-details.ts @@ -143,6 +143,21 @@ export function renderNodeDetails(graph: KnowledgeGraph, node: KnowledgeNode, pa receipt.append(element('p', '', `수집 기록 · ${text(selected.platform, 32)} · ${date(selected.collected_at)}${selected.truncated === true ? ' · 일부 발췌' : ''}`)); receipt.append(element('p', '', `기록 버전: ${text(selected.version, 256)}`)); receipt.append(element('p', '', `보존 기록 SHA-256: ${text(selected.raw_sha256, 64)}`)); + // A trace belongs to this exact selected source version and target only. + // It reports stored proof; it never claims the GPU rendered this node or + // that a local model ran because a derived vector was found. + const trace = record(payload?.pipeline_trace); + const proof = trace?.ok === true && trace.state === 'available' && trace.document_id === node.id + && trace.version === node.sourceVersion && trace.target_id === node.sourceTarget ? trace : null; + if (proof) { + const stages = record(proof.stages), source = record(proof.source), fts = record(proof.fts), dense = record(proof.dense); + const stored = stages?.stored === true && source?.state === 'hash_verified'; + const indexed = stored && stages?.indexed === true && fts?.state === 'verified'; + const vector = dense?.state === 'stored_vector_binding_verified'; + const status = stored ? '저장·원본 해시 확인' : '저장 상태 재확인 필요'; + receipt.append(element('p', '', `처리 경로 · ${status} · FTS ${indexed ? '확인' : '미확인'} · 로컬 Dense ${vector ? '저장 바인딩 확인' : '미확인/대기'}`)); + receipt.append(element('p', '', '검증 범위: 저장 기록과 파생 색인 · 실제 모델 로딩 및 화면 렌더링은 별도 확인')); + } const capture = record(selected.capture); if (capture?.status === 'manifest_verified') { receipt.append(element('p', '', `보존 파일: ${text(capture.byte_scope, 160)} · ${text(capture.json_pointer, 256)}`)); diff --git a/desktop/src/runtime.ts b/desktop/src/runtime.ts index dbb62009..318cdd4a 100644 --- a/desktop/src/runtime.ts +++ b/desktop/src/runtime.ts @@ -14,7 +14,7 @@ type SettingsAction = | "history-rooms" | "history-messages" | "reply-history" | "geeknews-history" | "voice-history-sessions" | "voice-history-messages" | "db-sync-history" - | "collection-history" | "collection-graph" | "collection-affinity" | "collection-projects" + | "collection-history" | "collection-graph" | "collection-trace" | "collection-affinity" | "collection-projects" | "collection-scheduler-status" | "collection-scheduler-control"; export async function operatorPause(): Promise { diff --git a/docs/ALDEN_CONTINUATION.md b/docs/ALDEN_CONTINUATION.md index a074c0e0..97602992 100644 --- a/docs/ALDEN_CONTINUATION.md +++ b/docs/ALDEN_CONTINUATION.md @@ -1,3 +1,12 @@ +> 2026-10-11 다중 대상 검색 추가: 같은 프로젝트 안의 대상별 버전·후보·관계 근거를 `target_id`로 격리하고, 최신성 반복문의 범위 변수 충돌을 수정했다. [macOS CI 38068930145](https://github.com/twoimo/alden/actions/runs/38068930145): Python 114/114, UI 30/30, TS/Vite 성공. 운영 대상 검색/모델/호스트 실물 E2E는 아직 확인하지 못했다. + +> 2026-10-11 추가 통합: [스냅샷 A→삭제→A 복원 및 수집→색인→이력→그래프/MCP 일치 검사](architecture/alden-pipeline-readback-20261011.md). +> 별도 `codex/alden-pipeline-readback-20261011` 브랜치에 소스와 회귀가 구현되었다. +> macOS CI `38067729712`: Python 112/112, UI 30/30, TypeScript/Vite build 성공(격리 fixture). +> 상태: **구현됨 / GitHub macOS 테스트 실제 실행 검증됨**, 소유자 Mac의 설치본·osk-system·물리 음성·운영 자료 E2E 및 서명/공증은 **미착수(이번 변경)**. +> 원본 `outputs/STATUS.md`는 Mac의 유효한 실행 연결이 확보되기 전까지 수정할 수 없었다. +> PR #27·#28, 운영 설정, 원본 DB, 메시지 전송 큐는 변경하지 않았다. + > 2026-10-11 원격 독립 구현 기록: [source integrity, OSK 삭제 이력과 MLX 검증](architecture/alden-remote-integrity-20261011.md). 이 링크는 Mac 설치 상태를 갱신했다는 의미가 아니다. 아래 날짜별 설치 버전과 검증 수치는 각 당시의 기록이며 새 브랜치의 실행 성과로 재사용하지 않는다. 로컬 `outputs/STATUS.md` 원문은 Mac 연결 복구 후 이어 갱신한다. # 현재 이어받기 상태 — 2026-10-05 diff --git a/docs/architecture/alden-pipeline-readback-20261011.md b/docs/architecture/alden-pipeline-readback-20261011.md new file mode 100644 index 00000000..345b5d60 --- /dev/null +++ b/docs/architecture/alden-pipeline-readback-20261011.md @@ -0,0 +1,132 @@ +# Alden 지식 처리 단위별 증거 대사 — 2026-10-11 + +기준 원격 HEAD: PR #28 `1dfee9a13b1a7785a1f8534f2c1111286d036167`. +작업 브랜치: `codex/alden-pipeline-readback-20261011`. +원본 PR #27·#28과 Mac 작업 복제본, 실제 DB·메시지 전송 큐·LLM 설정은 이 브랜치에서 수정하지 않는다. + +## 실제 연결 범위 + +```mermaid +flowchart LR + A[등록된 원본 스냅샷] --> B[CollectionStore.ingest] + B --> C[(버전·대상·관계·이력·FTS5)] + C --> D[대상과 프로젝트 권한 확인] + D --> E[단일 문서 trace_document] + F[(별도 로컬 Dense 검색 캐시)] --> E + E --> G[설치 앱 노드 상세의 근거 표시] + E --> H[alden_knowledge_trace 읽기 전용 MCP] + C --> I[GraphRAG 검색·실제 3D 그래프 노출] +``` + +- `scripts/alden_collection.py`: 한 번의 확정된 스냅샷 기록이 원본 ID, + 현재 대상별 버전, 활성 관계, FTS5와 `runs/events`를 같은 쓰기 트랜잭션에 반영한다. + `A → 삭제 → A`로 이전 원문 리비전이 다시 등장하면 원본 순서가 새로운 + `snapshot-run` ID를 만들고 실제 멤버십과 관계를 되살린다. + **동일 원본 순서 재전달**은 기존 확정 실행을 유지하므로 중복 이벤트를 만들지 않는다. + 과거 입력 리비전과 이전 원본 바이트는 보존한다. +- `scripts/alden_collection_retrieval.py::trace_document`: 하나의 허용된 + 문서 ID, 프로젝트 집합, 선택한 대상과 기대 버전을 사용한다. + **현재 허용된 멤버십이 없으면** 문서 존재 여부를 노출하지 않는 + `not_in_scope`을 반환한다. 낡은 기대 버전은 `version_changed`로 + 종료한다. 원본 BLOB의 SHA-256, 해당 버전의 FTS5 텍스트, + 확정된 `runs/events` 단계 및 활성 관계의 두 끝점을 각각 대사한다. + 일반 검색이나 모델 생성 요청을 만들지 않는다. +- Dense 경로는 별도의 파생 `retrieval.sqlite3`를 **읽기 전용**으로 연다. + 보관된 원문에 해당하는 텍스트·해시·encoder 프로필·벡터 길이·유한값·정규화 + 조건이 충족되어야 `stored_vector_binding_verified`이다. + 이 값은 **로컬 임베딩 모델 현재 로딩/추론 성공과 무관한 저장 바인딩 확인**이다. + FTS만 확인됐다면 Dense는 `not_indexed` 또는 대기/불일치로 표시한다. + 서로 다른 SQLite DB가 하나의 원자적 스냅샷이라는 주장은 하지 않는다. +- `alden_knowledge_trace`: 기존 Alden 자체 stdio MCP의 허용 프로젝트 + 시작 스코프 안에서 정확한 `document_id`를 조회한다. 도구 스키마는 + 추가 필드를 거부하고 메시지 발신·원본 DB 쓰기·클라우드 추론을 하지 않는다. + 이는 **외부 osk-system MCP가 실제 연결되었다는 증거가 아니다.** +- 설치 앱: `collection-graph`의 선택 노드 상세를 읽을 때만 trace를 추가한다. + 노드 ID·`source_version`·`source_target`가 모두 일치해야 UI에 + 저장/FTS/Dense 증거를 표시한다. 다른 대상이나 과거 선택의 응답은 + 현재 상세에 붙이지 않는다. 사용자 표시 문구는 수집 상태·저장 바인딩과 + 물리적 3D 렌더/AI 모델 사용을 명확히 구분한다. + 읽기 자체는 수집·저장 이벤트나 3D 시스템 점등을 새로 발생시키지 않는다. + +## 테스트 실행과 반례 + +[macOS CI #38067548142](https://github.com/twoimo/alden/actions/runs/38067548142) +초기 실행에서 (1) 이미 처리한 원본 A의 재등장에 이전 `run_id`를 +재사용하므로 복원이 누락되는 **실제 제품 결함**, (2) CI 러너에 로컬 +임베딩 서버가 없는 **테스트 환경 결함**이 드러났다. +첫째는 원본 monotonic 순서를 실행 ID에 결합하여 수정했다. +둘째는 테스트 프로세스에서만 고정된 encoder 프로필과 결정적 +3차원 벡터를 주입했다. 제품 서버·모델 설정은 변경하지 않는다. +실패 로그는 GitHub Actions에 그대로 보존한다. + +후속 [macOS CI #38067729712](https://github.com/twoimo/alden/actions/runs/38067729712), +정확한 검증 SHA `e05f08e46249f0361a42181d2bf0a23d1dbbf110`: + +| 범위 | 결과 | 검증 의미 | +| --- | --- | --- | +| Python 수집·스냅샷·GraphRAG·MCP·원본·OSK·모델 관련 | 113/113 통과 | 격리 fixture + 별도 stdio MCP subprocess, 운영 원본 아님 | +| 3D 선택 근거·CollectionGraph·활동 소비 Vitest | 30/30 통과 | DOM·컨트롤러 검증, 물리 GPU 입력 아님 | +| `npm run build --prefix desktop` | 성공 | TypeScript 확인 및 Vite 번들 | +| 동일 저장 순서 replay·A→삭제→A·부정확한 원본 해시 | 통과 | 중복 억제·복원·fail closed | +| 권한 없는 프로젝트·다른 대상·오래된 버전 | 통과 | 문서 상세 및 MCP 혼입 차단 | +| 로컬 임베딩 서버 실물 로딩 | 미검증 | 테스트에만 고정 벡터를 사용 | +| 지정 사용자의 Alden 설치본·외부 osk-system | 미검증 | 유효한 Mac Core 세션 토큰 없음 | + +### 후속 실행 검증 + +추가된 MCP stdio 실제 프로세스 테스트는 초기화·도구 목록·허용 프로젝트 조회, +거부된 프로젝트 요청 및 조회 전후 수집 실행/이력 건수가 같음을 확인한다. +[macOS CI 38068189206](https://github.com/twoimo/alden/actions/runs/38068189206) +검증 SHA `c227accdf2b20af46c8f816ace8332016214eb2f`: +**Python 113/113, UI 30/30, TypeScript/Vite production build 성공(이전 통합 체크포인트).** +프로세스 테스트는 로컬 격리 자료이며 외부 osk-system MCP와는 별개다. + +Tauri의 `collection-trace` 요청도 읽기 전용 allowlist에서 +바운드된 JSON만 라우팅하며, 부정확한 JSON·배열·4096바이트 초과를 +거부하는 Rust 회귀를 추가했다. +[macOS CI 38068129119](https://github.com/twoimo/alden/actions/runs/38068129119) +검증 SHA `2a2eb4e9d71a31b2c5632411472b8e3c6c5bd1fb`에서 +**Tauri Rust 120개 통과, strict Clippy 성공**을 확인했다. +Tauri 소스 검증은 실제 소유자 Mac에서 설치된 실행 번들의 인수 전달 성공과 +실제 마우스 입력/GPU 가시성 증거를 대신하지 않는다. + +### 2026-10-11: 다중 대상 안의 동일 원본 GraphRAG 격리 + +기존 GraphRAG가 같은 프로젝트에 여러 수집 대상을 포함할 때 +최신 한 개의 공통 문서 버전을 선정하므로 `target_id`만의 정확한 버전/관계 +조회 계약이 없었다. `CollectionStore._scope_versions`가 접근 허용·가용 +멤버십을 **대상으로 먼저 제한한 뒤** 현재 원본 버전을 선택하게 +보완했다. `CollectionStore.search`, `retrieve`의 BM25/로컬 Dense 및 +원문 근거 확인·최신성/관계 경로·MCP `alden_knowledge_search`가 +같은 명시적 `target_id`를 지원한다. 기본 검색은 종전처럼 승인된 +프로젝트 전체를 사용한다. 한 프로젝트의 같은 원본 ID가 여러 대상의 +다른 현재 버전을 가진 테스트와 대상 밖 후보·관계 혼입 차단을 검증한다. + +초기 [CI 38068771022](https://github.com/twoimo/alden/actions/runs/38068771022)는 +최신성 조회의 게시자 반복문 변수가 외부 `target_id`를 덮어쓰는 +SQL 바인딩 오류를 발견했다. 내부 변수를 `publisher_target_id`로 +분리해 동일 범위를 모든 단계에 보존하도록 고쳤다. +[macOS CI 38068930145](https://github.com/twoimo/alden/actions/runs/38068930145), +SHA `a9858eba635ef82a2c894b218e8de13215bba387`: +**Python 114/114, UI 30/30, TS/Vite 성공**. 중간 실패도 보존된다. + +## 설치 Mac에서 남은 수행 절차 + +1. 지정 복제본의 `git status`, `outputs/STATUS.md`, 현재 + PR #27·#28 및 별도 브랜치의 차이·설치본 SHA를 먼저 대조한다. + 미커밋 변경, 실행 중 서비스, 원본 DB·모델·전송 큐와 설정의 + 복구 가능한 스냅샷을 확보한다. +2. 테스트 결과와 diff를 검토해 복제본 안에 안전하게 가져온 후 + 사용자 등록 대상에서 한 개의 실제 자료로 완전/증분 수집을 실행한다. + 선택 노드 상세의 `pipeline_trace`, 자체 MCP 검색·history·trace 및 + osk-system 원문 재조회를 각각 실행해 동일 ID·버전·권한을 대조한다. +3. local MLX/임베딩/STT/TTS 및 물리 인터랙션은 독립적으로 측정한다. + 기준선과 TTFT, 지연 p50/p95, 그래프 FPS, 숨김 시 렌더 호출0, + CPU/GPU/메모리·클라우드 추론 미우회를 실제 증거로 남긴다. +4. 설치 앱의 파일·번들·서명 및 기존 프로세스/설정을 설치 전후 비교해 + `outputs/STATUS.md`의 네 가지 상태를 갱신한다. 서명/공증 및 정식 + 배포에는 실제 대상과 배포 권한이 필요하다. + +이번 브랜치는 실행 허가·원본 데이터 이동·MCP host registration이나 +새로운 외부 대상 연락을 발생시키지 않는다. 아카이브와 기존 +검증 로그는 삭제하지 않는다. diff --git a/scripts/alden_collection.py b/scripts/alden_collection.py index 72724362..654169a1 100644 --- a/scripts/alden_collection.py +++ b/scripts/alden_collection.py @@ -450,7 +450,11 @@ def ingest(self, target_id: str, records, *, cursor=None, origin="alden", projection = digest(encoded([ [{k:v for k,v in r.items() if k != 'source'} for r in records], [{**r, 'evidence': {k:v for k,v in r.get('evidence',{}).items() if k != 'source'}} for r in relations]])) - run_id = run_id or identity('snapshot-run', encoded([target_id, source_revision, processing_version, projection]).decode()) + # The same exact source revision can legitimately reappear after a + # different confirmed revision. Include observed monotonic order + # in the identity so A -> B -> A republishes A's membership and + # active relations, while a replay of the same order stays idempotent. + run_id = run_id or identity('snapshot-run', encoded([target_id, source_revision, source_order, processing_version, projection]).decode()) else: projection = None run_id = run_id or uuid.uuid4().hex @@ -638,7 +642,7 @@ def graph(self, *, projects: list[str]) -> dict: "source_evidence": json.loads(row["evidence"])}) return {"nodes": nodes, "edges": edges, "scope": projects, "canonical": "independent original sources; this is a derived projection"} - def search(self, query: str, *, projects: list[str], limit=10, time_from=None, time_to=None) -> list[dict]: + def search(self, query: str, *, projects: list[str], limit=10, time_from=None, time_to=None, target_id=None) -> list[dict]: if not projects: return [] with self.database() as db: @@ -648,21 +652,26 @@ def search(self, query: str, *, projects: list[str], limit=10, time_from=None, t clauses.append('v.collected_at' + operator + '?');values.append(value) rows = db.execute("""SELECT d.id,v.label,v.body,v.metadata,bm25(version_search) AS rank FROM version_search JOIN documents d ON d.id=version_search.document_id - JOIN (""" + self._scope_versions(projects) + """) s ON s.document_id=d.id + JOIN (""" + self._scope_versions(projects, target_id) + """) s ON s.document_id=d.id AND s.current_version=version_search.version_id JOIN versions v ON v.id=s.current_version WHERE version_search MATCH ? AND d.availability='available' """ + ''.join(' AND ' + c for c in clauses) + ' ORDER BY rank,d.id LIMIT ?', - (*projects, query, *values, limit)).fetchall() + (*projects, *((target_id,) if target_id is not None else ()), query, *values, limit)).fetchall() return [{**dict(row), "metadata": json.loads(row["metadata"])} for row in rows] - def _scope_versions(self, projects: list[str]) -> str: + def _scope_versions(self, projects: list[str], target_id=None) -> str: + if target_id is not None and (not isinstance(target_id, str) or not 1 <= len(target_id) <= 256): + raise ValueError("collection_target_scope_invalid") marks = ",".join("?" for _ in projects) available = "m.availability='available' AND " if self.schema >= 3 else "" - return """SELECT document_id,current_version FROM ( - SELECT m.document_id,m.current_version,ROW_NUMBER() OVER( + selected = " AND m.target_id=?" if target_id is not None else "" + # Choose the authorized source version only after the target filter. + # Stable original IDs can legitimately appear in several targets. + return """SELECT document_id,current_version,target_id FROM ( + SELECT m.document_id,m.current_version,m.target_id,ROW_NUMBER() OVER( PARTITION BY m.document_id ORDER BY v.collected_at DESC,m.target_id) AS choice FROM memberships m JOIN versions v ON v.id=m.current_version WHERE """ + available + """EXISTS(SELECT 1 FROM target_projects p WHERE p.target_id=m.target_id - AND p.permission!='denied' AND p.project IN (""" + marks + "))) WHERE choice=1" + AND p.permission!='denied' AND p.project IN (""" + marks + "))" + selected + ") WHERE choice=1" def projects(self) -> list[dict]: with self.database() as db: @@ -1057,7 +1066,7 @@ def activity_page(self, *, projects: list[str], after=None, stream_id=None, limi def read_action(state_root: Path, action: str, query: str | None = None) -> dict: """Bounded read-only application boundary; never creates a store on lookup.""" - if action not in {"collection-projects", "collection-history", "collection-graph"}: + if action not in {"collection-projects", "collection-history", "collection-graph", "collection-trace"}: raise ValueError("collection_action_invalid") options = json.loads(query) if query else {} if not isinstance(options, dict): @@ -1081,14 +1090,27 @@ def read_action(state_root: Path, action: str, query: str | None = None) -> dict result["projects"] = store.projects() result["targets"] = store.targets(projects) return result + if action == "collection-trace": + from alden_collection_retrieval import trace_document + return trace_document(state_root, options.get("document_id"), + projects=projects, expected_version=options.get("expected_version"), + target_id=options.get("target_id")) if action == "collection-graph": if options.get("activity") is True: return store.activity_page(projects=projects, after=options.get("after"), stream_id=options.get("stream_id"), limit=options.get("limit",200), target_id=options.get("target_id"), platform=options.get("platform")) - return store.graph_page(projects=projects, limit=options.get("limit",120), offset=options.get("offset",0), + result = store.graph_page(projects=projects, limit=options.get("limit",120), offset=options.get("offset",0), focus=options.get("focus"), hops=options.get("hops",1), query=str(options.get("search", ""))[:256], target_id=options.get("target_id"), platform=options.get("platform"), node_type=options.get("node_type"), relation=options.get("relation"), since=options.get("since"), until=options.get("until"), details=options.get("details",False), expected_version=options.get("expected_version"), overview=options.get("overview",False)) + # Only a selected current source with verified detail binding receives + # a separate processing trace. A graph overview does no extra scans. + detail = result.get("details") if isinstance(result, dict) else None + if options.get("details") is True and isinstance(detail, dict) and detail.get("basis") == "source_record": + from alden_collection_retrieval import trace_document + result["pipeline_trace"] = trace_document(state_root, detail["node_id"], + projects=projects, expected_version=detail["version"], target_id=detail["target_id"]) + return result raise ValueError("collection_action_invalid") diff --git a/scripts/alden_collection_retrieval.py b/scripts/alden_collection_retrieval.py index a7fdf2c5..e61afd26 100644 --- a/scripts/alden_collection_retrieval.py +++ b/scripts/alden_collection_retrieval.py @@ -28,9 +28,11 @@ def _scope(projects): return list(dict.fromkeys(projects)) -def _rows(store, projects, cancelled, time_from=None, time_to=None, *, text_db=None, resolve_source=False): +def _rows(store, projects, cancelled, time_from=None, time_to=None, *, text_db=None, resolve_source=False, target_id=None): projects = _scope(projects) clauses, args = [], list(projects) + if target_id is not None: + args.append(target_id) for value, operator in [(time_from, '>='), (time_to, '<=')]: if value is not None: if type(value) not in (int, float) or not math.isfinite(value): @@ -45,9 +47,9 @@ def _rows(store, projects, cancelled, time_from=None, time_to=None, *, text_db=N text_db = owned try: with store.database() as db: - rows = db.execute('''SELECT d.id,d.platform,d.original_id,v.id AS version, + rows = db.execute('''SELECT d.id,d.platform,d.original_id,s.target_id AS source_target,v.id AS version, v.raw_sha256,v.raw_path,v.label,v.body,v.collected_at - FROM documents d JOIN (''' + store._scope_versions(projects) + ''') s + FROM documents d JOIN (''' + store._scope_versions(projects,target_id) + ''') s ON s.document_id=d.id JOIN versions v ON v.id=s.current_version WHERE d.availability='available' ''' + ''.join(' AND ' + c for c in clauses) + ' ORDER BY d.id', args) result, budget = [], 0 @@ -218,13 +220,162 @@ def flush(): 'embedded_windows': windows, 'reused_versions':reused, 'model': model, 'encoding': ENCODING} + +def trace_document(root, document_id, *, projects, expected_version=None, target_id=None): + """Bounded read-only proof of one scoped source -> storage -> search -> graph. + + Journal stages establish confirmed producer work; FTS/vectors/graph are + checked independently. No search, embeddings, model loading or writes. + This is an observation across two independent SQLite snapshots, not proof + that a client rendered or cited the node or that an MLX model is resident. + """ + import re + projects = _scope(projects) + if not isinstance(document_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,40}:[0-9a-f]{64}', document_id): + raise ValueError('collection_trace_document_id_invalid') + if expected_version is not None and ( + not isinstance(expected_version, str) or + not re.fullmatch(r'version:[0-9a-f]{64}', expected_version)): + raise ValueError('collection_trace_expected_version_invalid') + if target_id is not None and ( + not isinstance(target_id, str) or not 1 <= len(target_id) <= 256): + raise ValueError('collection_trace_target_invalid') + store = CollectionStore.open_existing(Path(root)) + if store is None: + return {'ok': True, 'state': 'not_configured'} + placeholders = ','.join('?' for _ in projects) + with store.database() as db: + found = db.execute('''SELECT m.target_id,m.current_version,m.availability, + d.availability AS document_availability,v.label,v.body,v.raw_sha256, + v.raw_path,v.id AS version + FROM memberships m JOIN documents d ON d.id=m.document_id + JOIN versions v ON v.id=m.current_version + WHERE m.document_id=? AND (? IS NULL OR m.target_id=?) + AND EXISTS(SELECT 1 FROM target_projects p WHERE p.target_id=m.target_id + AND p.permission!='denied' AND p.project IN (''' + placeholders + ''')) + ORDER BY CASE WHEN m.availability='available' THEN 0 ELSE 1 END, + v.collected_at DESC,m.target_id LIMIT 1''', + (document_id,target_id,target_id,*projects)).fetchone() + if found is None: + # Do not disclose another project's existence, ID, or archived text. + return {'ok': True, 'state': 'not_in_scope'} + current = dict(found) + version = current['version'] + if expected_version is not None and expected_version != version: + return {'ok': False, 'state': 'version_changed', 'document_id': document_id} + selected_target = current['target_id'] + is_available = (current['availability'] == 'available' + and current['document_availability'] == 'available') + stages = dict.fromkeys(('discovered','parsed','validated','stored','indexed'), False) + # A removed membership's last completed run may only have a stored + # tombstone. Do not interpret old completed parsing as current work. + latest = db.execute('''SELECT e.run_id,e.event_id,e.sequence,e.details + FROM events e JOIN runs r ON r.id=e.run_id + WHERE e.document_id=? AND e.target_id=? AND e.version=? + AND e.stage='stored' AND r.state='complete' + ORDER BY e.sequence DESC LIMIT 1''', + (document_id,selected_target,version)).fetchone() + change = None + if latest: + change = json.loads(latest['details']).get('change') + for row in db.execute('''SELECT stage FROM events WHERE run_id=? + AND target_id=? AND document_id=? AND version=?''', + (latest['run_id'],selected_target,document_id,version)): + if row['stage'] in stages: + stages[row['stage']] = True + original = store._verified_source_blob(store.blobs, current['raw_path'], MAX_RECORD_BYTES) + if digest(original) != current['raw_sha256']: + raise RuntimeError('collection_source_integrity') + projected = db.execute('''SELECT label,body FROM version_search + WHERE document_id=? AND version_id=? LIMIT 2''', + (document_id,version)).fetchall() + fts = 'verified' if (len(projected) == 1 and + tuple(projected[0]) == (current['label'],current['body'])) else 'missing_or_mismatched' + # The bounded single-node graph check shares this collection snapshot. + edges = db.execute('''SELECT COUNT(*) FROM relations r + JOIN memberships sm ON sm.target_id=r.target_id AND sm.document_id=r.source + JOIN memberships tm ON tm.target_id=r.target_id AND tm.document_id=r.target + WHERE r.active=1 AND r.target_id=? AND (r.source=? OR r.target=?) + AND sm.availability='available' AND tm.availability='available' + AND EXISTS(SELECT 1 FROM target_projects p WHERE p.target_id=r.target_id + AND p.permission!='denied' AND p.project IN (''' + placeholders + '))', + (selected_target,document_id,document_id,*projects)).fetchone()[0] if is_available else 0 + checkpoint, _, _, _ = store._activity_scope(db, projects, selected_target) + # Record/source integrity belongs to the committed collection DB. Dense + # is a separately versioned derived cache and can lag this snapshot. + output = { + 'ok': True, 'state': 'available' if is_available else 'removed', + 'document_id': document_id, 'target_id': selected_target, 'version': version, + 'projects': projects, 'run_id': latest['run_id'] if latest else None, + 'stored_change': change if change in ('added','revised','unchanged','removed') else None, + 'stages': stages, + 'last_stored_event': {'event_id': latest['event_id'], 'sequence': latest['sequence']} if latest else None, + 'source': {'state': 'hash_verified', 'sha256': current['raw_sha256']}, + 'fts': {'state': fts, 'scope': 'current scoped version; SQLite FTS5'}, + 'graph': {'state': 'eligible' if is_available else 'retracted', + 'selected_target_relations': edges, 'displayed_in_client': False}, + 'activity_checkpoint': checkpoint, + 'dense': {'state': 'not_indexed', 'model_residency_verified': False}, + 'snapshot_scope': 'collection transaction plus independent optional dense read', + } + if not is_available: + return output + cache_path = _path(store) + if not cache_path.is_file(): + return output + with closing(sqlite3.connect(cache_path.as_uri() + '?mode=ro',uri=True,timeout=.15)) as cache: + cache.execute('PRAGMA query_only=ON') + cache.execute('BEGIN') + tables = {r[0] for r in cache.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('texts','vectors','text_fts')")} + if not {'texts','vectors','text_fts'} <= tables: + return output + indexed_text = cache.execute('''SELECT base_hash,raw_sha256,extraction,label,body,body_source + FROM texts WHERE document_id=? AND version=?''',(document_id,version)).fetchone() + indexed_fts = cache.execute('''SELECT label,body FROM text_fts + WHERE document_id=? AND version=? LIMIT 2''',(document_id,version)).fetchall() + vector = cache.execute('''SELECT text_hash,model,endpoint,encoding,vector + FROM vectors WHERE document_id=? AND version=?''',(document_id,version)).fetchone() + if indexed_text is None: + return output + base_hash = digest((current['label'] + '\n' + current['body']).encode()) + text_ok = indexed_text[:4] == (base_hash,current['raw_sha256'],TEXT_VERSION,current['label']) + if indexed_text[5] == 'stored_body': + text_ok = text_ok and indexed_text[4] == current['body'] + elif indexed_text[5] == 'retained_record.localOriginalText': + raw = json.loads(original) + text_ok = text_ok and isinstance(raw,dict) and raw.get('localOriginalText') == indexed_text[4] + else: + text_ok = False + if not text_ok or len(indexed_fts)!=1 or tuple(indexed_fts[0]) != (indexed_text[3],indexed_text[4]): + output['dense']['state'] = 'text_binding_invalid' + elif vector is None: + output['dense']['state'] = 'text_indexed_vector_pending' + else: + try: + raw_vector = vector[4] + values = struct.unpack('<'+'f'*(len(raw_vector)//4), raw_vector) if isinstance(raw_vector,bytes) and 4<=len(raw_vector)<=8192 and len(raw_vector)%4==0 else () + norm = math.sqrt(sum(value*value for value in values)) + verified = (vector[0] == digest((indexed_text[3]+'\n'+indexed_text[4]).encode()) + and all(isinstance(v,str) and v for v in vector[1:3]) + and vector[3] == ENCODING and values and + all(math.isfinite(v) for v in values) and abs(norm-1.) <= .001) + output['dense']['state'] = 'stored_vector_binding_verified' if verified else 'vector_binding_invalid' + except (ValueError, TypeError, OverflowError, struct.error): + output['dense']['state'] = 'vector_binding_invalid' + return output + + def retrieve(root, query, *, projects, max_entities=3, max_relations=3, candidate_limit=40, max_context_chars=8000, rrf_k=60, rrf_weights=None, - cancelled=lambda: False, query_embed=None, time_from=None, time_to=None, also=None): + cancelled=lambda: False, query_embed=None, time_from=None, time_to=None, also=None, target_id=None): import auto_reply_knowledge_graph as kg projects = _scope(projects) if not isinstance(query, str) or not 1 <= len(query.strip()) <= 1024: raise ValueError('collection_query_invalid') + if target_id is not None and (not isinstance(target_id,str) or not 1<=len(target_id)<=256): + raise ValueError('collection_target_scope_invalid') + scoped_args = (*projects,target_id) if target_id is not None else tuple(projects) if type(max_context_chars) is not int or not 256 <= max_context_chars <= 16000: raise ValueError('collection_context_budget') if type(candidate_limit) is not int or not 1 <= candidate_limit <= 100: @@ -234,12 +385,12 @@ def retrieve(root, query, *, projects, max_entities=3, max_relations=3, store = CollectionStore.open_existing(Path(root)) if store is None: return {'facts': [], 'fact_provenance': [], 'search_mode': 'bm25_only', 'projects': projects} - source_rows = _rows(store, projects, cancelled, time_from, time_to) + source_rows = _rows(store, projects, cancelled, time_from, time_to, target_id=target_id) rows = [r for r in source_rows if (r['label'] + '\n' + r['body']).strip()] eligible = {row['id']: row for row in rows} terms = kg._fts_query_terms([query]) lexical = [(r['id'], float(r['rank'])) for r in store.search(terms, projects=projects, - limit=candidate_limit, time_from=time_from, time_to=time_to)] if terms else [] + limit=candidate_limit, time_from=time_from, time_to=time_to, target_id=target_id)] if terms else [] dense, mode = [], 'bm25_only' path = _path(store) if terms and path.is_file(): @@ -300,13 +451,13 @@ def retrieve(root, query, *, projects, max_entities=3, max_relations=3, aliases=metadata.get('aliases',[]);aliases=aliases if isinstance(aliases,list) else [] marks=','.join('?' for _ in projects) publisher_scopes={} - for target_id,label in source.execute('''SELECT DISTINCT t.id,t.label FROM targets t JOIN memberships m ON m.target_id=t.id - WHERE m.document_id=? AND m.availability='available' AND EXISTS(SELECT 1 FROM target_projects p - WHERE p.target_id=t.id AND p.permission!='denied' AND p.project IN ('''+marks+'))',(identity,*projects)): - publisher_scopes.setdefault(label,[]).append(target_id) + for publisher_target_id,label in source.execute('''SELECT DISTINCT t.id,t.label FROM targets t JOIN memberships m ON m.target_id=t.id + WHERE m.document_id=? AND m.availability='available' AND (? IS NULL OR m.target_id=?) AND EXISTS(SELECT 1 FROM target_projects p + WHERE p.target_id=t.id AND p.permission!='denied' AND p.project IN ('''+marks+'))',(identity,target_id,target_id,*projects)): + publisher_scopes.setdefault(label,[]).append(publisher_target_id) conflict=source.execute('''SELECT r.source,r.target FROM relations r WHERE r.active=1 AND r.type IN ('conflicts','contradicts') - AND (r.source=? OR r.target=?) AND EXISTS(SELECT 1 FROM target_projects p - WHERE p.target_id=r.target_id AND p.permission!='denied' AND p.project IN ('''+marks+'))',(identity,identity,*projects)) + AND (r.source=? OR r.target=?) AND (? IS NULL OR r.target_id=?) AND EXISTS(SELECT 1 FROM target_projects p + WHERE p.target_id=r.target_id AND p.permission!='denied' AND p.project IN ('''+marks+'))',(identity,identity,target_id,target_id,*projects)) temporal_records.append({'id':identity,'name':row['label'],'aliases':aliases+list(publisher_scopes),'mention_scopes':publisher_scopes, 'has_conflict':any(a in eligible and b in eligible for a,b in conflict), 'observed_at':metadata.get('published_at'),'time_basis':'source_published_at', @@ -332,8 +483,8 @@ def append(text, proof): chosen = [] for document_id, _score in ranked[:max(0, min(int(max_entities), 10))]: row = eligible[document_id] - current = db.execute("SELECT s.current_version FROM (" + store._scope_versions(projects) + - ") s JOIN documents d ON d.id=s.document_id WHERE d.id=? AND d.availability='available'", (*projects, document_id)).fetchone() + current = db.execute("SELECT s.current_version FROM (" + store._scope_versions(projects,target_id) + + ") s JOIN documents d ON d.id=s.document_id WHERE d.id=? AND d.availability='available'", (*scoped_args, document_id)).fetchone() if current is None or current[0] != row['version']: raise RuntimeError('collection_graph_version_changed') raw = store._verified_source_blob(store.blobs, row['raw_path'], MAX_RECORD_BYTES) @@ -347,7 +498,7 @@ def append(text, proof): proof = {'fact_type': 'entity', 'entity_id': document_id, 'note_id': document_id, 'source_kind': 'collected_source', 'source_version': row['version'], 'note_hash': row['raw_sha256'], 'original_id': row['original_id'], - 'platform': row['platform'], 'projects': projects, 'updated_at': row['collected_at'], + 'platform': row['platform'], 'projects': projects, 'source_target': row['source_target'], 'updated_at': row['collected_at'], 'body_source': row['body_source'], 'derived_from': metadata.get('source', {}), 'truth_status': 'source_record; not independently verified', 'provenance_valid': True, 'retracted': False} @@ -365,10 +516,10 @@ def append(text, proof): tm.current_version AS target_version FROM relations r JOIN memberships sm ON sm.target_id=r.target_id AND sm.document_id=r.source JOIN memberships tm ON tm.target_id=r.target_id AND tm.document_id=r.target - WHERE r.active=1 AND r.source=? + WHERE r.active=1 AND r.source=? AND (? IS NULL OR r.target_id=?) AND EXISTS(SELECT 1 FROM target_projects p WHERE p.target_id=r.target_id AND p.permission!='denied' AND p.project IN (''' + marks + ')) ORDER BY r.id', - (chosen[0], *projects)) + (chosen[0], target_id, target_id, *projects)) for edge in edges: _check(cancelled) if edge['target'] not in eligible or edge['target'] in temporal_excluded: @@ -383,13 +534,13 @@ def append(text, proof): {'fact_type': 'relation', 'source_kind': 'collected_source', 'relation': edge['type'], 'source_id': edge['source'], 'target_id': edge['target'], 'relation_id': edge['id'], 'source_version': eligible[edge['source']]['version'], 'target_version': target['version'], - 'projects': projects, 'source_evidence': json.loads(edge['evidence']), + 'projects': projects, 'source_target': edge['target_id'], 'source_evidence': json.loads(edge['evidence']), 'provenance_valid': True, 'retracted': False}) if sum(p['fact_type'] == 'relation' for p in provenance) >= min(int(max_relations), 10): break _check(cancelled) return {'facts': facts, 'fact_provenance': provenance, 'search_mode': mode, - 'projects': projects, 'eligible_documents': len(rows), 'context_chars': used, + 'projects': projects, 'target_id': target_id, 'eligible_documents': len(rows), 'context_chars': used, 'unsearchable_documents': len(source_rows) - len(rows), 'pending_text_versions': sum(not r['text_ready'] for r in source_rows), 'time_basis': 'collection_time', diff --git a/scripts/alden_knowledge_mcp.py b/scripts/alden_knowledge_mcp.py index 8be25219..b7abd771 100644 --- a/scripts/alden_knowledge_mcp.py +++ b/scripts/alden_knowledge_mcp.py @@ -7,7 +7,7 @@ from alden_abort import ABORT_STATE_NAME, AbortToken from alden_collection import CollectionStore, read_action -from alden_collection_retrieval import retrieve +from alden_collection_retrieval import retrieve, trace_document from alden_status_mcp import StdioServer, _check @@ -41,26 +41,35 @@ def tool_catalog(self): 'limit': {'type': 'integer', 'minimum': 1, 'maximum': 50}} result = [] for name, extras, description in [ - ('alden_knowledge_search', {'query': {'type': 'string', 'minLength': 1, 'maxLength': 256}}, + ('alden_knowledge_search', {'query': {'type': 'string', 'minLength': 1, 'maxLength': 256}, + 'target_id': {'type': 'string', 'minLength': 1, 'maxLength': 256}}, 'Retrieve retained collection versions and explicit relations with exact provenance. No writes, model loads or account access.'), ('alden_knowledge_graph', {'focus': {'type': 'string', 'maxLength': 256}}, 'Read bounded stored graph nodes and explicit relationships in the allowed project scope.'), ('alden_knowledge_history', {'after': {'type': 'integer', 'minimum': 0}}, - 'Read persisted collection stages after a cursor; reading does not create activity.')]: + 'Read persisted collection stages after a cursor; reading does not create activity.'), + ('alden_knowledge_trace', { + 'document_id': {'type': 'string', 'minLength': 1, 'maxLength': 256}, + 'expected_version': {'type': 'string', 'minLength': 1, 'maxLength': 256}, + 'target_id': {'type': 'string', 'minLength': 1, 'maxLength': 256}}, + 'Read a scoped, hash-verified source-to-storage/FTS/dense/graph receipt. Never runs an LLM or writes.')]: result.append({'name': name, 'description': description, 'inputSchema': {'type': 'object', 'properties': {**common, **extras}, - 'required': ['projects'] + (['query'] if name.endswith('search') else []), + 'required': ['projects'] + (['query'] if name.endswith('search') else ['document_id'] if name.endswith('trace') else []), 'additionalProperties': False}, 'annotations': {'readOnlyHint': True, 'destructiveHint': False, 'idempotentHint': True, 'openWorldHint': False}}) if name.endswith('search'): result[-1]['inputSchema']['properties']['limit'] = {'type': 'integer', 'minimum': 1, 'maximum': 10} + if name.endswith('trace'): + result[-1]['inputSchema']['properties'].pop('limit') return result def valid_call(self, params): name, args = params.get('name'), params.get('arguments') - options = {'alden_knowledge_search': {'query'}, 'alden_knowledge_graph': {'focus'}, - 'alden_knowledge_history': {'after'}} + options = {'alden_knowledge_search': {'query','target_id'}, 'alden_knowledge_graph': {'focus'}, + 'alden_knowledge_history': {'after'}, + 'alden_knowledge_trace': {'document_id','expected_version','target_id'}} if not isinstance(name, str) or name not in options or not isinstance(args, dict) or set(args) - ({'projects', 'limit'} | options[name]): return False projects = args.get('projects') @@ -72,7 +81,13 @@ def valid_call(self, params): if type(limit) is not int or not 1 <= limit <= (10 if name.endswith('search') else 50): return False if name.endswith('search'): - return isinstance(args.get('query'), str) and 1 <= len(args['query'].strip()) <= 256 + return (isinstance(args.get('query'), str) and 1 <= len(args['query'].strip()) <= 256 + and ('target_id' not in args or isinstance(args['target_id'],str) + and 1 <= len(args['target_id']) <= 256)) + if name.endswith('trace'): + return (isinstance(args.get('document_id'), str) and 1 <= len(args['document_id']) <= 256 + and all(key not in args or isinstance(args[key], str) and 1 <= len(args[key]) <= 256 + for key in ('expected_version','target_id')) and 'limit' not in args) if name.endswith('graph'): return 'focus' not in args or isinstance(args['focus'], str) and 1 <= len(args['focus']) <= 256 return type(args.get('after', 0)) is int and args.get('after', 0) >= 0 @@ -86,7 +101,11 @@ def call_tool(self, params, control): import auto_reply_knowledge_graph as kg with kg.embedding_abort_scope(token): result = retrieve(self.root, args['query'], projects=args['projects'], - max_entities=args.get('limit', 6), cancelled=cancelled) + max_entities=args.get('limit', 6), cancelled=cancelled, + target_id=args.get('target_id')) + elif name.endswith('trace'): + result = trace_document(self.root, args['document_id'], projects=args['projects'], + target_id=args.get('target_id'), expected_version=args.get('expected_version')) else: action = 'collection-graph' if name.endswith('graph') else 'collection-history' options = {'projects': args['projects'], 'limit': args.get('limit', 50)} diff --git a/scripts/auto-reply-menubar.py b/scripts/auto-reply-menubar.py index a3caf59b..d68a8ffa 100755 --- a/scripts/auto-reply-menubar.py +++ b/scripts/auto-reply-menubar.py @@ -2730,7 +2730,7 @@ def main(): except (ValueError, RuntimeError, OSError) as error: _print_json({"ok": False, "reason": type(error).__name__, "targets": []}) return 0 - if action in {"collection-history", "collection-graph", "collection-projects"}: + if action in {"collection-history", "collection-graph", "collection-trace", "collection-projects"}: from alden_collection import read_action try: _print_json(read_action(_menubar_state_root(), action, _argv_flag_value("--history-query"))) diff --git a/tests/test_alden_collection_retrieval.py b/tests/test_alden_collection_retrieval.py index 696d9ac8..ce97a6a8 100644 --- a/tests/test_alden_collection_retrieval.py +++ b/tests/test_alden_collection_retrieval.py @@ -44,6 +44,52 @@ def embed(self, texts): def profile(self): return patch.object(kg, '_active_dense_embedding_model', return_value='fixed-test-encoder') + def test_exact_target_graph_rag_fences_revisions_relations_and_mcp_scope(self): + # Two authorized sources may own the same original ID and versions + # inside one project. Filtering happens before selecting one version. + second=self.store.register(platform='graph',original_id='another-source', + kind='source',label='another',projects=['one']) + shared=self.add(self.allowed,'same','first source facts only') + self.add(second,'same','second source facts only') + self.store.ingest(second,[{'platform':'graph','original_id':'second-only', + 'label':'unique adjacency','text':'other document'}]) + relations=[{'source_platform':'graph','source_id':'same', + 'target_platform':'graph','target_id':'second-only', + 'type':'cites','evidence':{'explicit':True}}] + self.store.ingest(second,[],relations=relations) + base=retrieve(self.root,'shared',projects=['one'],target_id=self.allowed) + selected=retrieve(self.root,'shared',projects=['one'],target_id=second) + self.assertEqual(base['target_id'],self.allowed) + self.assertEqual(base['facts'],['shared term: first source facts only']) + self.assertEqual(base['eligible_documents'],1) + self.assertEqual(base['fact_provenance'][0]['source_target'],self.allowed) + self.assertEqual(selected['fact_provenance'][0]['source_target'],second) + self.assertIn('second source facts only',selected['facts'][0]) + self.assertEqual(len([p for p in base['fact_provenance'] if p['fact_type']=='relation']),0) + self.assertEqual(len([p for p in selected['fact_provenance'] if p['fact_type']=='relation']),1) + self.assertNotEqual(base['fact_provenance'][0]['source_version'], + selected['fact_provenance'][0]['source_version']) + self.assertEqual(self.store.search('shared',projects=['one'], + target_id=self.allowed)[0]['body'],'first source facts only') + self.assertEqual(self.store.search('shared',projects=['one'], + target_id=second)[0]['body'],'second source facts only') + out=retrieve(self.root,'shared',projects=['one'],target_id=self.other) + self.assertEqual(out['facts'],[]) + self.assertEqual(out['eligible_documents'],0) + with self.assertRaisesRegex(ValueError,'target_scope_invalid'): + retrieve(self.root,'shared',projects=['one'],target_id=True) + mcp=KnowledgeServer(self.root,['one']) + selection={'name':'alden_knowledge_search','arguments': + {'projects':['one'],'query':'shared','target_id':self.allowed}} + self.assertTrue(mcp.valid_call(selection)) + observed=mcp.call_tool(selection,CallControl(timeout=5)) + self.assertEqual(observed['fact_provenance'][0]['source_target'],self.allowed) + for invalid in (None,7,'','x'*257): + if invalid is None: + continue + self.assertFalse(mcp.valid_call({'name':'alden_knowledge_search','arguments': + {'projects':['one'],'query':'shared','target_id':invalid}})) + def test_visible_version_is_project_specific_and_hash_verified(self): self.add(self.allowed, 'same', 'original body');self.add(self.other, 'same', 'private other body') result = retrieve(self.root, 'shared', projects=['one']) diff --git a/tests/test_alden_collection_snapshots.py b/tests/test_alden_collection_snapshots.py index 4af7d6cc..eb7f71c6 100644 --- a/tests/test_alden_collection_snapshots.py +++ b/tests/test_alden_collection_snapshots.py @@ -47,6 +47,37 @@ def test_target_deletion_preserves_another_targets_version_and_archive(self): self.assertEqual(db.execute('SELECT count(*) FROM versions').fetchone()[0],3) self.assertEqual(db.execute('PRAGMA foreign_key_check').fetchall(),[]) + def test_reappearing_exact_revision_reactivates_sources_and_relations_once(self): + records=[self.record('a'),self.record('b')] + relation={'source_platform':'graph','source_id':'a','target_platform':'graph', + 'target_id':'b','type':'cites','evidence':{'source':'fixture'}} + first=self.snapshot(self.a,records,1,relations=[relation]) + version=self.store.graph_page(projects=['one'])['nodes'][0]['source_version'] + self.assertEqual(len(self.store.graph_page(projects=['one'])['edges']),1) + deleted=self.snapshot(self.a,[],2) + self.assertEqual(deleted['removed'],2) + self.assertEqual(self.store.graph_page(projects=['one'])['nodes'],[]) + restored=self.snapshot(self.a,records,3,relations=[relation]) + page=self.store.graph_page(projects=['one']) + self.assertEqual(restored['state'],'complete') + self.assertNotEqual(restored['run_id'],first['run_id']) + self.assertEqual(restored['added'],0) + self.assertEqual(restored['unchanged'],2) + self.assertEqual(len(page['nodes']),2) + self.assertEqual(len(page['edges']),1) + self.assertEqual(page['nodes'][0]['source_version'],version) + with self.store.database() as db: + runs=db.execute('SELECT COUNT(*) FROM runs WHERE target_id=?',(self.a,)).fetchone()[0] + stages=db.execute('SELECT COUNT(*) FROM events WHERE run_id=?',(restored['run_id'],)).fetchone()[0] + self.assertEqual(runs,3) + self.assertGreater(stages,0) + replay=self.snapshot(self.a,records,3,relations=[relation]) + self.assertEqual(replay['state'],'unchanged') + self.assertEqual(replay['run_id'],restored['run_id']) + with self.store.database() as db: + self.assertEqual(db.execute('SELECT COUNT(*) FROM runs WHERE target_id=?',(self.a,)).fetchone()[0],runs) + self.assertEqual(db.execute('SELECT COUNT(*) FROM events WHERE run_id=?',(restored['run_id'],)).fetchone()[0],stages) + def test_older_and_conflicting_source_revisions_cannot_replace_current(self): self.snapshot(self.a,[self.record('same','new')],20) before=self.store.target(self.a)['cursor'] diff --git a/tests/test_alden_pipeline_trace.py b/tests/test_alden_pipeline_trace.py new file mode 100644 index 00000000..a33cc61b --- /dev/null +++ b/tests/test_alden_pipeline_trace.py @@ -0,0 +1,229 @@ +"""Source-to-index-to-history-to-graph readback, without a live external host. + +Uses the genuine collection writer, exact hashed source files, SQLite FTS, +real retrieval cache and owned read-only MCP. No LLM or network requests. +""" +import json +import select +import sqlite3 +import subprocess +import sys +import unittest +from unittest.mock import patch +from pathlib import Path +from tempfile import TemporaryDirectory + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +from alden_collection import CollectionStore, digest, encoded, identity, read_action +from alden_collection_retrieval import index_dense, retrieve, trace_document +from alden_knowledge_mcp import KnowledgeServer +from alden_status_mcp import CallControl +import auto_reply_knowledge_graph as kg + + +class PipelineReadbackTests(unittest.TestCase): + def setUp(self): + self.temp = TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.store = CollectionStore(self.root) + self.target = self.store.register(platform='youtube', original_id='channel-public', + kind='channel', label='채널', projects=['research']) + self.other = self.store.register(platform='youtube', original_id='channel-private', + kind='channel', label='비공개', projects=['private'], + permission='denied') + self.first = identity('youtube','video-a') + self.second = identity('youtube','video-b') + self.a = {'platform':'youtube','original_id':'video-a','kind':'video','label':'원문 A', + 'text':'정확한 채널 실험과 근거', 'raw':{'videoId':'video-a','text':'정확한 채널 실험과 근거'}} + self.b = {'platform':'youtube','original_id':'video-b','kind':'video','label':'원문 B', + 'text':'관계 있는 별도 자료', 'raw':{'videoId':'video-b','text':'관계 있는 별도 자료'}} + self.relation = {'source_platform':'youtube','source_id':'video-a', + 'target_platform':'youtube','target_id':'video-b', + 'type':'cites','evidence':{'declared':'original relationship'}} + + def ingest(self, records=None, *, snapshot=False, order=1, relations=None): + records = [self.a,self.b] if records is None else records + relations = [self.relation] if relations is None else relations + options = {'origin':'test-pipeline', 'relations':relations} + if snapshot: + options.update(complete_snapshot=True, source_revision=digest(encoded([records,relations])), + source_order=order, cursor={'revision':order}) + return self.store.ingest(self.target,records,**options) + + def version(self): + page = self.store.graph_page(projects=['research']) + return next(n['source_version'] for n in page['nodes'] if n['id']==self.first) + + def test_full_collection_mcp_ft_s_graph_and_journal_consistency(self): + result = self.ingest(snapshot=True) + version = self.version() + trace = trace_document(self.root,self.first,projects=['research'],expected_version=version, + target_id=self.target) + self.assertEqual(result['state'],'complete') + self.assertEqual(trace['state'],'available') + self.assertEqual(trace['version'],version) + self.assertEqual(trace['run_id'],result['run_id']) + self.assertEqual(trace['source']['state'],'hash_verified') + self.assertEqual(trace['fts']['state'],'verified') + self.assertEqual(trace['dense']['state'],'not_indexed') + self.assertEqual(trace['graph']['state'],'eligible') + self.assertEqual(trace['graph']['selected_target_relations'],1) + self.assertTrue(all(trace['stages'].values())) + self.assertEqual(trace['activity_checkpoint'], + self.store.graph_page(projects=['research'],target_id=self.target)['activity_checkpoint']) + history=read_action(self.root,'collection-history',json.dumps({'projects':['research'],'limit':100})) + self.assertTrue(any(row['event_id']==trace['last_stored_event']['event_id'] for row in history['items'])) + details=read_action(self.root,'collection-graph',json.dumps({ + 'projects':['research'],'focus':self.first,'hops':0,'limit':120,'details':True, + 'expected_version':version,'target_id':self.target})) + self.assertEqual(details['details']['node_id'],self.first) + self.assertEqual(details['pipeline_trace']['last_stored_event'],trace['last_stored_event']) + self.assertEqual(self.store.search('정확한',projects=['research'])[0]['id'],self.first) + server=KnowledgeServer(self.root,['research']) + call={'name':'alden_knowledge_trace','arguments':{'projects':['research'], + 'document_id':self.first,'expected_version':version,'target_id':self.target}} + self.assertTrue(server.valid_call(call)) + saved=server.call_tool(call,CallControl(timeout=4)) + self.assertEqual(saved['source'],trace['source']) + self.assertEqual(saved['activity_checkpoint'],trace['activity_checkpoint']) + self.assertEqual(saved['last_stored_event'],trace['last_stored_event']) + + def test_local_dense_vector_is_separate_and_hash_bound_to_actual_source_version(self): + self.ingest() + initial=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(initial['dense']['state'],'not_indexed') + # A fake profile is scoped to the test. The CI runner has no local + # embedding service and must not access the network for the fixture. + with patch.object(kg,'_active_dense_embedding_model',return_value='fixed-test-encoder'): + indexed=index_dense(self.root,['research'],embed=lambda texts:[[1.0,0.0,0.0] for _ in texts]) + self.assertEqual(indexed['state'],'ready') + trace=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(trace['dense']['state'],'stored_vector_binding_verified') + self.assertFalse(trace['dense']['model_residency_verified']) + answer=retrieve(self.root,'정확한 근거',projects=['research'],query_embed=lambda text:[1.,0.,0.]) + self.assertEqual(answer['search_mode'],'rrf') + self.assertIn(self.first,answer['candidates']) + self.assertTrue(any(proof.get('source_version')==trace['version'] + for proof in answer['fact_provenance'])) + with sqlite3.connect(self.store.root/'retrieval.sqlite3') as db: + db.execute('UPDATE vectors SET vector=? WHERE document_id=?',(b'\x00\x01',self.first)) + degraded=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(degraded['dense']['state'],'vector_binding_invalid') + self.assertEqual(degraded['fts']['state'],'verified') + + def test_scope_isolation_revisions_and_mismatched_selection(self): + self.ingest(records=[self.a],relations=[]) + self.store.ingest(self.other,[{**self.a,'text':'private canary and unrelated details'}]) + allowed=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(allowed['state'],'available') + self.assertEqual(allowed['target_id'],self.target) + self.assertEqual(allowed['fts']['state'],'verified') + self.assertEqual(trace_document(self.root,self.first,projects=['private'])['state'],'not_in_scope') + self.assertEqual(trace_document(self.root,self.second,projects=['research'])['state'],'not_in_scope') + stale=trace_document(self.root,self.first,projects=['research'], + expected_version=identity('version','outdated')) + self.assertEqual(stale,{'ok':False,'state':'version_changed','document_id':self.first}) + with self.assertRaisesRegex(ValueError,'project_scope_required'): + trace_document(self.root,self.first,projects=[]) + with self.assertRaisesRegex(ValueError,'document_id_invalid'): + trace_document(self.root,'another_scope',projects=['research']) + server=KnowledgeServer(self.root,['research']) + self.assertFalse(server.valid_call({'name':'alden_knowledge_trace', + 'arguments':{'projects':['private'],'document_id':self.first}})) + self.assertFalse(server.valid_call({'name':'alden_knowledge_trace', + 'arguments':{'projects':['research'],'document_id':self.first,'limit':5}})) + self.assertNotIn('private canary',json.dumps(allowed,ensure_ascii=False)) + + def test_committed_removal_retracts_node_and_edge_but_retains_previous_raw_proof(self): + self.ingest(snapshot=True) + version=self.version() + baseline=self.store.activity_page(projects=['research']) + changed=self.ingest(records=[],relations=[],snapshot=True,order=2) + self.assertEqual(changed['removed'],2) + trace=trace_document(self.root,self.first,projects=['research'],expected_version=version) + self.assertEqual(trace['state'],'removed') + self.assertEqual(trace['source']['state'],'hash_verified') + self.assertEqual(trace['stored_change'],'removed') + self.assertFalse(trace['stages']['indexed']) + self.assertTrue(trace['stages']['stored']) + self.assertEqual(trace['graph'],{'state':'retracted','selected_target_relations':0, + 'displayed_in_client':False}) + self.assertEqual(self.store.graph_page(projects=['research'])['nodes'],[]) + self.assertEqual(self.store.search('정확한',projects=['research']),[]) + self.assertFalse(retrieve(self.root,'정확한',projects=['research'])['facts']) + changes=self.store.activity_page(projects=['research'],after=baseline['cursor'], + stream_id=baseline['stream_id']) + self.assertEqual({r['kind'] for r in changes['items']},{'removed','relations_changed'}) + self.assertEqual(self.store.recent_events(projects=['private'])['items'],[]) + restored=self.ingest(snapshot=True,order=3) + back=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(restored['state'],'complete') + self.assertEqual(back['state'],'available') + self.assertEqual(back['graph']['selected_target_relations'],1) + + def test_real_owned_mcp_stdio_trace_uses_scope_and_never_writes(self): + self.ingest(records=[self.a],relations=[]) + version=self.version() + script=Path(__file__).resolve().parents[1]/'scripts/alden_knowledge_mcp.py' + with self.store.database() as db: + before=(db.execute('SELECT COUNT(*) FROM events').fetchone()[0], + db.execute('SELECT COUNT(*) FROM runs').fetchone()[0]) + child=subprocess.Popen([sys.executable,'-B',str(script),'--state-root',str(self.root), + '--allow-project','research'],stdin=subprocess.PIPE, + stdout=subprocess.PIPE,stderr=subprocess.PIPE) + replies=[] + def request(identifier,method,params=None): + frame={'jsonrpc':'2.0','id':identifier,'method':method} + if params is not None:frame['params']=params + child.stdin.write(json.dumps(frame).encode()+b'\n') + child.stdin.flush() + self.assertTrue(select.select([child.stdout],[],[],5)[0], + 'owned MCP timeout for '+method) + return json.loads(child.stdout.readline()) + try: + init=request(1,'initialize',{'protocolVersion':'2025-11-25', + 'capabilities':{},'clientInfo':{}}) + self.assertEqual(init['id'],1) + child.stdin.write(b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n') + child.stdin.flush() + catalog=request(2,'tools/list') + self.assertIn('alden_knowledge_trace',[tool['name'] for tool in catalog['result']['tools']]) + tool=request(3,'tools/call',{'name':'alden_knowledge_trace', + 'arguments':{'projects':['research'],'document_id':self.first, + 'expected_version':version,'target_id':self.target}}) + self.assertFalse(tool['result']['isError']) + proof=json.loads(tool['result']['content'][0]['text']) + self.assertEqual(proof['version'],version) + self.assertEqual(proof['source']['state'],'hash_verified') + self.assertEqual(proof['fts']['state'],'verified') + self.assertEqual(proof['graph']['state'],'eligible') + denied=request(4,'tools/call',{'name':'alden_knowledge_trace', + 'arguments':{'projects':['private'],'document_id':self.first}}) + self.assertTrue(denied.get('error') is not None or denied.get('result',{}).get('isError')) + finally: + child.stdin.close() + try:child.wait(timeout=3) + except subprocess.TimeoutExpired:child.kill();child.wait(timeout=3) + child.stdout.close();child.stderr.close() + with self.store.database() as db: + after=(db.execute('SELECT COUNT(*) FROM events').fetchone()[0], + db.execute('SELECT COUNT(*) FROM runs').fetchone()[0]) + self.assertEqual(before,after) + + def test_corrupt_raw_does_not_produce_a_false_success_receipt(self): + self.ingest(records=[self.a],relations=[]) + trace=trace_document(self.root,self.first,projects=['research']) + self.assertEqual(trace['source']['state'],'hash_verified') + with self.store.database() as db: + file=db.execute('SELECT raw_path FROM versions WHERE id=?',(trace['version'],)).fetchone()[0] + (self.store.blobs/file).write_text('{"tampered":true}') + with self.assertRaisesRegex(RuntimeError,'collection_source_integrity'): + trace_document(self.root,self.first,projects=['research']) + with self.assertRaisesRegex(RuntimeError,'collection_source_integrity'): + read_action(self.root,'collection-graph',json.dumps({'projects':['research'], + 'focus':self.first,'hops':0,'details':True})) + + +if __name__=='__main__': + unittest.main()