From b9e8adc82635869175eb5459707718f8640e1b6a Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 18:54:09 -0400 Subject: [PATCH 01/12] feat: ship repository-owned pointer pendant source --- .github/workflows/ci.yml | 8 +- .github/workflows/release.yml | 20 ++ docs/knowledge/README.md | 35 +++ docs/knowledge/alignment-status.json | 8 + docs/knowledge/alignments.ttl | 6 + docs/knowledge/repository-knowledge.json | 138 ++++++++++ .../knowledge/repository-knowledge.shapes.ttl | 52 ++++ docs/knowledge/stored-queries.json | 32 +++ queries/repository-share-bobbin.rq | 75 ++++++ scripts/build-repository-knowledge.py | 92 +++++++ scripts/build-repository-share.py | 235 ++++++++++++++++++ scripts/publish-repository-share.sh | 5 + scripts/publish_repository_share.py | 149 +++++++++++ scripts/test-repository-knowledge.py | 121 +++++++++ scripts/test-repository-share-publication.py | 99 ++++++++ scripts/test-repository-share-query.py | 76 ++++++ scripts/verify-repository-share.py | 74 ++++++ 17 files changed, 1224 insertions(+), 1 deletion(-) create mode 100644 docs/knowledge/README.md create mode 100644 docs/knowledge/alignment-status.json create mode 100644 docs/knowledge/alignments.ttl create mode 100644 docs/knowledge/repository-knowledge.json create mode 100644 docs/knowledge/repository-knowledge.shapes.ttl create mode 100644 docs/knowledge/stored-queries.json create mode 100644 queries/repository-share-bobbin.rq create mode 100644 scripts/build-repository-knowledge.py create mode 100644 scripts/build-repository-share.py create mode 100644 scripts/publish-repository-share.sh create mode 100644 scripts/publish_repository_share.py create mode 100644 scripts/test-repository-knowledge.py create mode 100644 scripts/test-repository-share-publication.py create mode 100644 scripts/test-repository-share-query.py create mode 100644 scripts/verify-repository-share.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c14956a..1a3951d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: with: python-version: "3.12" - name: Install private-key scanner - run: python3 -m pip install pre-commit==4.6.1 pyyaml + run: python3 -m pip install pre-commit==4.6.1 pyyaml rdflib==7.6.0 - name: Private-key refusal controls and tracked-file scan run: python3 scripts/check-private-keys.py @@ -31,6 +31,12 @@ jobs: - name: Crates publication contract run: python3 scripts/test-crates-workflow.py + - name: Repository pendant source and refusal controls + run: | + python3 scripts/test-repository-share-query.py + python3 scripts/test-repository-knowledge.py + python3 scripts/test-repository-share-publication.py + - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y protobuf-compiler cmake g++ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f83f2b01..c0b2b289 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -658,3 +658,23 @@ jobs: with: expected-version: ${{ steps.want.outputs.version }} dry-run: 'false' + + repository-pendant: + name: Verify repository pendant + needs: [release-please, release] + if: always() && needs.release.result == 'success' + runs-on: ubuntu-22.04 + timeout-minutes: 45 + permissions: + contents: read + env: + VERSION: ${{ inputs.version || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || needs.release-please.outputs.tag_name }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout exact published source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: refs/tags/${{ env.VERSION }} + fetch-depth: 0 + - name: Require repository-owned trusted-host pendant + run: python3 scripts/verify-repository-share.py "$VERSION" "$(git rev-parse HEAD)" diff --git a/docs/knowledge/README.md b/docs/knowledge/README.md new file mode 100644 index 00000000..a7ac3df3 --- /dev/null +++ b/docs/knowledge/README.md @@ -0,0 +1,35 @@ +# Repository pendant source + +The repository-owned release job exports code and document **pointers**, plus +reviewed summaries of ideas, interfaces, decisions, freshness and starting points. +It does not export source chunk text or line spans. Source witnesses bind each +summary to its defining tracked file. The native index must resolve every +curated pointer before the build can proceed. Stored queries and validating +shapes travel with the share. + +`alignments.ttl` is currently a valid empty SSSOM/RDF carrier. Its accompanying +status file explicitly records zero approved mappings and seals those bytes. +This is not evidence of global absence. Missing, changed or malformed input +refuses; an explicit empty carrier does not. Dependencies and glossary references +are not identity alignments. Code/commit deduplication and unrelated exactMatch +records are not cross-repository mapping approval. + +Candidate mappings must go through Quipu's existing `align propose` and +`align decide` curation path. Keep their source graphs and source witnesses; +do not fill an approval author from a lexical score. Only the reviewed mapping +record may produce derived identity through the engine's existing apply path. +The current source profile deliberately does not import arbitrary nonempty +mapping data until an actual curated artifact and its transport are reviewed. + +The copied release workflow verifies the three public assets only after the +ordinary release finishes. The managed trusted host runs this repository's own +publisher, keeps its publication policy private, pins both native engine +executables and the indexer, and refuses retained uncertain upload outcomes. +Default configuration still publishes only the existing Quipu corpus. Enabling +another repository requires its actual trusted host configuration and a clean +release containing this job; committing this source does not publish an asset. + +The publisher requires the engine's model/digest/dimension manifest recipe. +Loading and vector rebuilding belong to the native engine. No hosted job gets +private publication policy or a new credential route. These source controls do +not establish public asset availability, live loading or service acceptance. diff --git a/docs/knowledge/alignment-status.json b/docs/knowledge/alignment-status.json new file mode 100644 index 00000000..1e9dd274 --- /dev/null +++ b/docs/knowledge/alignment-status.json @@ -0,0 +1,8 @@ +{ + "status": "no-approved-mappings", + "approved_record_count": 0, + "carrier_sha256": "c345ddc89d879d93150851807656815c8139252ac799c139db8b6d24d3ec3a38", + "scope": "scbrown/bobbin repository pendant source", + "basis": "No verified approved artifact or pairs supplied at source review; empty carrier is explicit. This is not a global absence claim.", + "curation_contract": "https://github.com/scbrown/quipu/blob/main/docs/design/cross-graph-alignment.md" +} diff --git a/docs/knowledge/alignments.ttl b/docs/knowledge/alignments.ttl new file mode 100644 index 00000000..d251b87f --- /dev/null +++ b/docs/knowledge/alignments.ttl @@ -0,0 +1,6 @@ +# SSSOM/RDF carrier: explicitly no approved Bobbin cross-stack mappings. +# This is an empty mapping graph, not an absent/corrupt input or a claim that +# no mappings exist anywhere. Candidate records require the existing curator +# path before publication as accepted mappings. Dependencies remain dependencies. +@prefix sssom: . +@prefix owl: . diff --git a/docs/knowledge/repository-knowledge.json b/docs/knowledge/repository-knowledge.json new file mode 100644 index 00000000..2b78b784 --- /dev/null +++ b/docs/knowledge/repository-knowledge.json @@ -0,0 +1,138 @@ +{ + "source_revision": "2b1ba22cb0f2b8d26354507c3eb1b6fe6890f7f8", + "reviewed_at": "2026-10-20T22:30:00Z", + "max_age_seconds": 2592000, + "records": [ + { + "kind": "idea", + "source": "docs/book/src/architecture/overview.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/docs%2Fbook%2Fsrc%2Farchitecture%2Foverview.md#architecture-overview", + "summary": "Bobbin keeps code context local: structural parsing supplies file and symbol pointers, local embeddings supply semantic retrieval, and temporal coupling supplies change context.", + "source_sha256": "e44cb5a7fa67f01057da8566424a168abde6ce642b7c918baa785365d6805eb7", + "type": "Concept" + }, + { + "kind": "idea", + "source": "docs/book/src/architecture/overview.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/docs%2Fbook%2Fsrc%2Farchitecture%2Foverview.md#query-pipeline", + "summary": "Search combines semantic and keyword retrieval; context assembly adds related files rather than treating a search hit as the complete task context.", + "source_sha256": "e44cb5a7fa67f01057da8566424a168abde6ce642b7c918baa785365d6805eb7", + "type": "Concept" + }, + { + "kind": "interface", + "source": "src/cli/mod.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fcli%2Fmod.rs", + "summary": "The Commands enum defines the public command surface, including search, context and portable index pack operations.", + "source_sha256": "21c85d361c229d3c536c329ee4cb43ff8eb2b0679e4663ea2e5cd32a10fc5577", + "properties": { + "interfaceKind": "CLI", + "publicName": "bobbin Commands enum" + } + }, + { + "kind": "interface", + "source": "src/mcp/server.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fmcp%2Fserver.rs", + "summary": "The MCP server advertises the tool interface; defining source is the authority for tool names, inputs and feature conditions.", + "source_sha256": "458d3aeb36836d48788e1be3653a2e6ac515d55ea64bf07ddc388a22a79fa4c3", + "properties": { + "interfaceKind": "MCP", + "publicName": "Bobbin MCP tools" + } + }, + { + "kind": "interface", + "source": "src/http/mod.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fhttp%2Fmod.rs", + "summary": "The HTTP module defines the public REST service and route composition; it is distinct from the stdio MCP transport.", + "source_sha256": "24b6cc34852c375938cc9db7c35a77db3d8655935e80b4ad2946015eec7f4d6b", + "properties": { + "interfaceKind": "REST", + "publicName": "Bobbin HTTP service" + } + }, + { + "kind": "interface", + "source": "src/config.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fconfig.rs", + "summary": "Configuration types define keys and defaults, including embedding.backend, embedding.model and embedding.batch_size.", + "source_sha256": "cf69c283d9af17e4ccd1b9181dc6673b48cd4e7c7b7394bf4ba13af7c61958b0", + "properties": { + "interfaceKind": "config", + "publicName": "Bobbin configuration types" + } + }, + { + "kind": "interface", + "source": "src/cli/pack.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fcli%2Fpack.rs", + "summary": "Portable Bobbin index packs are a separate file format from Quipu repository pendants; pack operations are defined here.", + "source_sha256": "40afc73061ad6af9564df14d5ebdf2274d71c3e42101dec937c74cd9222f0406", + "properties": { + "interfaceKind": "file-format", + "publicName": "Bobbin portable index pack" + } + }, + { + "kind": "interface", + "source": "src/knowledge/mod.rs", + "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fknowledge%2Fmod.rs", + "summary": "The optional knowledge feature integrates with the Quipu crate. This dependency is not an identity assertion between the two repositories.", + "source_sha256": "fc41f9d18e06107ea72c335cc4b9fbd4114a8fc7dcb785f5e4eaa177d76a897c", + "properties": { + "interfaceKind": "library", + "publicName": "Optional Quipu knowledge integration" + }, + "depends_on": [ + "http://aegis.gastown.local/ontology/code/quipu/src%2Flib.rs" + ] + }, + { + "kind": "decision", + "source": "CONTRIBUTING.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/CONTRIBUTING.md#feature-integration-checklist", + "summary": "New retrieval signals must be considered for context assembly and its MCP and CLI interfaces. The published contributor checklist is the source of this active guidance.", + "source_sha256": "791ee33f6bc3f9dd9ca7248e2b19b4a154f1853ac1b4e2e14ab1d1b959bd7e5e", + "type": "DecisionRecord", + "properties": { + "status": "active", + "supersessionStatus": "not-recorded-in-source", + "glossaryTerm": "Decision" + }, + "see_also": [ + "https://scbrown.github.io/quechua/ns#Decision" + ] + }, + { + "kind": "start-here", + "source": "README.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/README.md", + "summary": "Start with installation and the first successful search, then read architecture and contributor guidance.", + "source_sha256": "b919f7e1618d9819ad6057cd3dfc296aff12fe82863bf2e49320bdebf0c55b1f", + "properties": { + "startHereOrder": "1" + } + }, + { + "kind": "start-here", + "source": "docs/book/src/architecture/overview.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/docs%2Fbook%2Fsrc%2Farchitecture%2Foverview.md", + "summary": "Read the architecture overview for the indexing, search and optional knowledge graph boundaries.", + "source_sha256": "e44cb5a7fa67f01057da8566424a168abde6ce642b7c918baa785365d6805eb7", + "properties": { + "startHereOrder": "2" + } + }, + { + "kind": "start-here", + "source": "CONTRIBUTING.md", + "subject": "http://aegis.gastown.local/ontology/doc/bobbin/CONTRIBUTING.md", + "summary": "Read contributor gates and the feature integration checklist before changing retrieval behavior.", + "source_sha256": "791ee33f6bc3f9dd9ca7248e2b19b4a154f1853ac1b4e2e14ab1d1b959bd7e5e", + "properties": { + "startHereOrder": "3" + } + } + ] +} diff --git a/docs/knowledge/repository-knowledge.shapes.ttl b/docs/knowledge/repository-knowledge.shapes.ttl new file mode 100644 index 00000000..7a59de4a --- /dev/null +++ b/docs/knowledge/repository-knowledge.shapes.ttl @@ -0,0 +1,52 @@ +@prefix a: . +@prefix sh: . +@prefix rdfs: . +@prefix xsd: . +@prefix sssom: . +@prefix owl: . +@prefix prov: . + +# This profile constrains only explicitly curated repository knowledge. It +# does not turn every imported Concept or CodeModule into a Bobbin record. +a:BobbinKnowledgeProfile a sh:NodeShape ; + sh:targetSubjectsOf a:knowledgeKind ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path rdfs:comment ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:knowledgeKind ; sh:minCount 1 ; + sh:in ( "idea" "interface" "decision" "start-here" ) ; ] ; + sh:property [ sh:path a:filePath ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:reviewAfter ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:dateTime ; ] ; + sh:property [ sh:path a:sourceHash ; sh:minCount 1 ; sh:maxCount 1 ; sh:pattern "^[0-9a-f]{64}$" ; ] ; + # maxAge is seconds since the explicit source review, not an eviction rule. + sh:property [ sh:path a:maxAge ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:integer ; sh:minExclusive 0 ; ] ; + sh:property [ sh:path prov:wasDerivedFrom ; sh:minCount 1 ; sh:nodeKind sh:IRI ; ] . + +a:BobbinInterfaceProfile a sh:NodeShape ; + sh:targetSubjectsOf a:interfaceKind ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path a:interfaceKind ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "CLI" "MCP" "REST" "config" "file-format" "library" ) ; ] ; + sh:property [ sh:path a:publicName ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:depends_on ; sh:nodeKind sh:IRI ; ] . + +a:BobbinDecisionProfile a sh:NodeShape ; + sh:targetSubjectsOf a:supersessionStatus ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path a:status ; sh:minCount 1 ; sh:maxCount 1 ; sh:in ( "active" "superseded" "proposed" ) ; ] ; + sh:property [ sh:path a:supersessionStatus ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "not-recorded-in-source" "recorded" ) ; ] . + +# Existing SSSOM/RDF is a record, not a second identity-assertion format. The +# native align decide/apply path remains the only way to derive identity knots. +a:BobbinMappingRecordProfile a sh:NodeShape ; + sh:targetSubjectsOf sssom:subject_id ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path sssom:subject_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:object_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:predicate_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:mapping_justification ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] . + +a:BobbinNoHandAuthoredIdentity a sh:NodeShape ; + sh:targetSubjectsOf owl:sameAs ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path owl:sameAs ; sh:maxCount 0 ; ] . diff --git a/docs/knowledge/stored-queries.json b/docs/knowledge/stored-queries.json new file mode 100644 index 00000000..cc46030a --- /dev/null +++ b/docs/knowledge/stored-queries.json @@ -0,0 +1,32 @@ +[ + { + "name": "repository_ideas", + "description": "Repository-owned repository ideas", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a a:Concept; rdfs:comment ?summary . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/bobbin/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/bobbin/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_interfaces", + "description": "Repository-owned repository interfaces", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:interfaceKind ?kind; a:publicName ?name . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/bobbin/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/bobbin/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_decisions", + "description": "Repository-owned repository decisions", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a a:DecisionRecord; a:status ?status . OPTIONAL { ?s ?successor } FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/bobbin/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/bobbin/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_start_here", + "description": "Repository-owned repository start here", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:startHereOrder ?order . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/bobbin/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/bobbin/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_freshness", + "description": "Repository-owned repository freshness", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:reviewAfter ?reviewAfter; a:maxAge ?maxAgeSeconds . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/bobbin/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/bobbin/\")) } ORDER BY ?s LIMIT 100", + "params": [] + } +] diff --git a/queries/repository-share-bobbin.rq b/queries/repository-share-bobbin.rq new file mode 100644 index 00000000..e855fe76 --- /dev/null +++ b/queries/repository-share-bobbin.rq @@ -0,0 +1,75 @@ +PREFIX aegis: +PREFIX rdf: +PREFIX prov: +PREFIX dct: +PREFIX sssom: +PREFIX owl: +PREFIX rdfs: + +# Public repository knowledge only. Bobbin publishes its replaceable snapshot +# into ROOT, so graph/group selectors cannot see it. Keep this predicate list +# aligned with Bobbin's generate_chunk_turtle output: selecting every outgoing +# edge also pulls private operational literals into a public release artifact. +CONSTRUCT { ?s ?p ?o } +WHERE { + { + ?s ?p ?o . + FILTER( + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/code/bobbin/") || + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/doc/bobbin/") + ) + VALUES ?p { + rdf:type rdfs:label + aegis:filePath aegis:repo aegis:language + aegis:inDocument aegis:chunkOrder aegis:nextChunk aegis:mentions + aegis:name aegis:definedIn aegis:symbolKind + aegis:heading aegis:headingDepth + rdfs:comment rdfs:seeAlso prov:wasDerivedFrom dct:modified dct:isReplacedBy + aegis:knowledgeKind aegis:interfaceKind aegis:publicName aegis:status + aegis:supersessionStatus aegis:startHereOrder aegis:glossaryTerm + aegis:reviewAfter aegis:maxAge aegis:depends_on aegis:sourceHash + } + # Bobbin emits weak mention names as strings. Quipu may additionally + # resolve those names to IRIs under the same predicate in its local store; + # those derived edges do not satisfy the shipped string-valued shape. + FILTER(?p != aegis:mentions || isLiteral(?o)) + } + UNION + { + ?s ?p ?o . + { ?s a sssom:MappingSet } UNION { ?s a owl:Axiom } + VALUES ?p { + rdf:type rdfs:label prov:wasDerivedFrom + sssom:mappings sssom:subject_id sssom:object_id sssom:predicate_id + sssom:subject_label sssom:object_label sssom:mapping_justification + sssom:predicate_modifier sssom:confidence sssom:author_id + } + } + UNION + { + VALUES ?s { + aegis:repo_bobbin + aegis:repo-embedded-knowledge-databases-directive-2026-09-03 + } + ?s ?p ?o . + } + # Versioned crew memory is internal source, including references attached to + # otherwise public context nodes. Internal backups use a separate explicit scope. + FILTER(!STRSTARTS(STR(?s), "urn:quipu:memory:")) + FILTER(!STRSTARTS(STR(?p), "https://quipu.dev/memory#")) + FILTER(!STRSTARTS(STR(?o), "urn:quipu:memory:")) + # Work tracking identifiers are not public repository knowledge. Filter + # both endpoints and literal mentions instead of renaming entity identity. + FILTER(!REGEX(STR(?s), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + FILTER(!REGEX(STR(?o), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + # Exclude the whole node when a required label contains an internal ID. + # Dropping only its label would leave a typed, SHACL-invalid fragment. + MINUS { + ?s ?internalPredicate ?internalValue . + FILTER(REGEX(STR(?internalValue), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + } + MINUS { + ?o ?targetPredicate ?targetValue . + FILTER(REGEX(STR(?targetValue), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + } +} diff --git a/scripts/build-repository-knowledge.py b/scripts/build-repository-knowledge.py new file mode 100644 index 00000000..34b860fb --- /dev/null +++ b/scripts/build-repository-knowledge.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Render reviewed summaries onto existing indexed pointers, never new identities.""" +import datetime +import hashlib +import json +from pathlib import Path +import re +from urllib.parse import quote + +A = 'http://aegis.gastown.local/ontology/' +RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' +RDFS = 'http://www.w3.org/2000/01/rdf-schema#' +PROV = 'http://www.w3.org/ns/prov#' +XSD = 'http://www.w3.org/2001/XMLSchema#' +KINDS = {'idea', 'interface', 'decision', 'start-here'} +PROPERTIES = {'interfaceKind', 'publicName', 'status', 'supersessionStatus', 'startHereOrder', 'glossaryTerm'} + + +def render(source, revision, lookup): + source = Path(source) + if not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('release revision must be explicit') + registry = json.loads((source / 'docs/knowledge/repository-knowledge.json').read_text()) + reviewed = datetime.datetime.fromisoformat(registry['reviewed_at'].replace('Z', '+00:00')) + age = registry['max_age_seconds'] + if reviewed.tzinfo is None or type(age) is not int or age <= 0: + raise ValueError('freshness requires an explicit UTC review time and positive seconds') + after = (reviewed + datetime.timedelta(seconds=age)).isoformat().replace('+00:00', 'Z') + records = registry['records'] + if {record['kind'] for record in records} != KINDS: + raise ValueError('ideas/interfaces/decisions/start-here inventory incomplete') + lines, seen = [], set() + def fact(subject, predicate, obj, iri=False, datatype=None): + if iri: + if not re.fullmatch(r'[^\s<>"{}|\\^`]+:[^\s<>"{}|\\^`]*', obj): + raise ValueError('invalid referenced IRI') + value = '<' + obj + '>' + else: + value = json.dumps(str(obj), ensure_ascii=False) + if datatype: + value += '^^<' + datatype + '>' + lines.append('<' + subject + '> <' + predicate + '> ' + value + ' .') + for record in records: + path = record['source'] + witness = source / path + if (Path(path).is_absolute() or '..' in Path(path).parts + or not witness.resolve().is_relative_to(source.resolve()) + or hashlib.sha256(witness.read_bytes()).hexdigest() != record['source_sha256']): + raise ValueError('source witness changed: ' + path) + subject = record['subject'] + bases = (A + 'doc/bobbin/' + quote(path, safe=''), A + 'code/bobbin/' + quote(path, safe='')) + if not any(subject == base or subject.startswith(base + '#') for base in bases): + raise ValueError('subject is not the source-backed indexed pointer') + # lookup must use the actual index: a deterministic-looking guessed IRI + # is not an identity proof. Never mint another node when it is absent. + label = lookup(subject) + if not isinstance(label, str) or not label: + raise ValueError('source pointer absent or ambiguous in actual index: ' + subject) + if (subject, record['kind']) in seen: + raise ValueError('duplicate knowledge record') + seen.add((subject, record['kind'])) + if record.get('type') not in ('Concept', 'DecisionRecord', None): + raise ValueError('unsupported knowledge type') + fact(subject, RDFS + 'label', label) + fact(subject, A + 'filePath', path) + if record.get('type'): + fact(subject, RDF + 'type', A + record['type'], iri=True) + summary = record['summary'] + if (not summary.strip() or len(summary) > 1000 + or re.search(r'\b[\w-]+\.(?:svc|lan)\b|/home/|\baegis-[A-Za-z0-9]', summary)): + raise ValueError('unreviewed or private summary') + fact(subject, RDFS + 'comment', summary) + fact(subject, A + 'knowledgeKind', record['kind']) + fact(subject, A + 'sourceHash', record['source_sha256']) + fact(subject, A + 'reviewAfter', after, datatype=XSD + 'dateTime') + fact(subject, A + 'maxAge', age, datatype=XSD + 'integer') + fact(subject, 'http://purl.org/dc/terms/modified', registry['reviewed_at'], datatype=XSD + 'dateTime') + fact(subject, PROV + 'wasDerivedFrom', 'https://github.com/scbrown/bobbin/blob/' + revision + '/' + path, iri=True) + for key, value in record.get('properties', {}).items(): + if key not in PROPERTIES: + raise ValueError('unknown content property') + if key == 'startHereOrder': + if not str(value).isdigit() or int(value) <= 0: + raise ValueError('start-here order must be a positive integer') + fact(subject, A + key, int(value), datatype=XSD + 'integer') + else: + fact(subject, A + key, value) + for dependency in record.get('depends_on', []): + fact(subject, A + 'depends_on', dependency, iri=True) + for term in record.get('see_also', []): + fact(subject, RDFS + 'seeAlso', term, iri=True) + return '\n'.join(sorted(set(lines))) + '\n' diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py new file mode 100644 index 00000000..e7bfd515 --- /dev/null +++ b/scripts/build-repository-share.py @@ -0,0 +1,235 @@ +"""Build one outward stack share using the existing Quipu/Bobbin boundary. + +Source and tooling are separate fresh, tag-verified checkouts. Index configuration, +live policy, databases and receiver probes stay in the private working directory. +Only the verified share directory leaves this function. +""" +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import subprocess + + +def command(args, *, env=None): + return subprocess.run([str(v) for v in args], check=True, capture_output=True, + text=True, env=env).stdout + + +def local_mcp(binary, database, calls): + """Short-lived native process over a private DB; no HTTP or live graph.""" + import selectors + requests = [ + {'jsonrpc': '2.0', 'id': 0, 'method': 'initialize', + 'params': {'protocolVersion': '2024-11-05', 'capabilities': {}, + 'clientInfo': {'name': 'repository-pendant-build', 'version': '1'}}}, + {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, + ] + requests.extend({'jsonrpc': '2.0', 'id': number, 'method': 'tools/call', + 'params': {'name': name, 'arguments': arguments}} + for number, (name, arguments) in enumerate(calls, 1)) + values = [] + with subprocess.Popen([str(binary), 'mcp', '--db', str(database)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) as process: + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + try: + for request in requests: + process.stdin.write(json.dumps(request) + '\n'); process.stdin.flush() + if 'id' not in request: + continue + if not selector.select(timeout=30): + raise ValueError('native MCP response timeout') + response = json.loads(process.stdout.readline()) + if (response.get('id') != request['id'] or 'result' not in response + or 'error' in response or response['result'].get('isError', False)): + raise ValueError('native MCP refused operation: ' + json.dumps(response)) + if request['id']: + content = response['result'].get('content', []) + if len(content) != 1 or content[0].get('type') != 'text': + raise ValueError('unexpected native MCP result envelope') + values.append(json.loads(content[0]['text'])) + finally: + selector.close(); process.stdin.close() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.terminate(); process.wait(timeout=10) + if process.returncode: + raise ValueError('native MCP process failed') + return values + + +def register_queries(binary, database, definitions): + if not isinstance(definitions, list) or not definitions: + raise ValueError('explicit repository stored query definitions required') + for definition in definitions: + if not isinstance(definition, dict) or definition.get('action', 'load') != 'load': + raise ValueError('stored query must be an explicit load definition') + results = local_mcp(binary, database, [('quipu_queries', {**d, 'action': 'load'}) for d in definitions]) + if [result.get('loaded') for result in results] != [d['name'] for d in definitions]: + raise ValueError('native registry did not return the exact loaded query names') + + +def indexed_labels(binary, database, subjects): + if any(not re.fullmatch(r'https?://[^\s<>"{}|\\^`]+', subject) for subject in subjects): + raise ValueError('invalid indexed subject IRI') + query = 'SELECT ?s ?label WHERE { VALUES ?s { ' + ' '.join('<' + s + '>' for s in subjects) + '} ?s ?label }' + result, = local_mcp(binary, database, [('quipu_query', {'query': query, 'verbose': True})]) + if result.get('truncated') is not False: + raise ValueError('indexed pointer lookup was incomplete') + labels = {} + for row in result['rows']: + subject, label = row['s'], row['label'] + if isinstance(subject, dict): subject = subject['value'] + if isinstance(label, dict): label = label['value'] + if subject in labels and labels[subject] != label: + raise ValueError('ambiguous indexed pointer label') + labels[subject] = label + return labels + + +def alignment_carrier(source): + """An explicit empty reviewed carrier differs from missing or corrupt data. + + Accepted nonempty mapping transport awaits an actual curated artifact. Do + not interpret a dependency, lexical match, or candidate as an identity. + """ + source = Path(source) + data = (source / 'docs/knowledge/alignments.ttl').read_bytes() + status = json.loads((source / 'docs/knowledge/alignment-status.json').read_text()) + if (status.get('status') != 'no-approved-mappings' + or type(status.get('approved_record_count')) is not int + or status['approved_record_count'] != 0 + or status.get('carrier_sha256') != hashlib.sha256(data).hexdigest()): + raise ValueError('unreviewed or corrupt alignment carrier') + # Only declarations/comments are permitted in the explicit-empty profile. + # The native data validator below still parses the complete Turtle input. + for line in data.decode().splitlines(): + if (line.strip() and not line.lstrip().startswith('#') + and line.strip() not in { + '@prefix sssom: .', + '@prefix owl: .'}): + raise ValueError('empty alignment profile contains mapping data') + return data.decode() + + +def build(revision, source, tooling, binary, bobbin, output): + repository = "scbrown/bobbin" + source, tooling, output = map(Path, (source, tooling, output)) + if output.exists(): + raise ValueError('repository share output already exists') + if not re.fullmatch(r'scbrown/[a-z][a-z0-9-]*', repository): + raise ValueError('invalid repository identity') + slug = repository.split('/')[1] + if (not re.fullmatch(r'[0-9a-f]{40}', revision) + or command(['git', '-C', source, 'rev-parse', 'HEAD']).strip() != revision + or command(['git', '-C', source, 'status', '--porcelain']).strip()): + raise ValueError('repository source is not the exact clean release') + paths = command(['git', '-C', source, 'ls-files', '-z']).split('\0') + control = next((p for p in ('README.md', 'README.rst') if p in paths), None) + if control is None: + raise ValueError('release lacks a supported public corpus positive control') + private = output.parent / 'build-private' + private.mkdir(mode=0o700) + policy = private / 'policy.ttl' + command(['python3', tooling / 'scripts/private-share-policy.py', policy]) + index = private / 'index'; (index / '.bobbin').mkdir(parents=True) + (index / '.bobbin/config.toml').write_text('''quipu_push_chunks = true + +[index] +include = ["**/*.rs", "**/*.md", "**/*.rst", "**/*.toml", "**/*.yml", "**/*.yaml", "**/*.py", "**/*.sh", "**/*.js", "**/*.mjs", "**/*.ts", "**/*.tsx", "**/*.json", "**/*.ttl", "**/*.go"] +exclude = ["**/target/**", "**/.git/**", "**/.bobbin/**", "**/.agents/**", "**/.claude/**", "**/.codex/**", "**/.mcp.json", "**/CLAUDE.local.md", "**/node_modules/**", "**/book/book/**", "**/vendor/**"] +use_gitignore = true +entities = false + +[embedding] +model = "all-MiniLM-L6-v2" +batch_size = 64 +''') + env = {k: v for k, v in os.environ.items() + if k not in ('BOBBIN_SERVER', 'BOBBIN_QUIPU_REMOTE')} + command([bobbin, 'index', index, '--source', source, '--repo', slug, + '--force', '--skip-calibrate', '--json'], env=env) + database = private / 'producer.db' + index_database = index / '.bobbin/quipu/quipu.db' + if not index_database.is_file(): + raise ValueError('indexer did not produce the repository graph') + with sqlite3.connect('file:' + str(index_database) + '?mode=ro', uri=True) as original: + with sqlite3.connect(database) as target: + original.backup(target) + shapes = sorted((tooling / 'shapes').glob('*.ttl')) + if not shapes: + raise ValueError('verified engine source has no receiver shapes') + for shape in shapes: + command([binary, 'shapes', 'load', shape.stem, shape, '--db', database]) + command([binary, 'shapes', 'load', 'publication-policy', + tooling / 'docs/knowledge/publication-policy.shapes.ttl', '--db', database]) + command([binary, 'knot', tooling / 'docs/knowledge/publication-policy.ttl', '--db', database]) + command([binary, 'knot', policy, '--graph', 'urn:quipu:private-release-policy', '--db', database]) + context = private / 'context.ttl' + context.write_text(f'''@prefix aegis: . +@prefix rdfs: . +aegis:repo_{slug} a aegis:GitRepo ; rdfs:label "repo_{slug}" ; + aegis:repositorySha "{revision}" . +aegis:repo-embedded-knowledge-databases-directive-2026-09-03 a aegis:Directive ; + rdfs:label "Repository-embedded knowledge databases" ; + aegis:issuedBy "Stiwi" ; aegis:applies_to aegis:repo_{slug} . +''') + command([binary, 'knot', context, '--db', database]) + # Curated repository content and validating shapes are repo-owned. Refuse + # an incomplete source proposal rather than minting a pointer-only stub. + registry_path = source / 'docs/knowledge/repository-knowledge.json' + knowledge = private / 'repository-knowledge.ttl' + constraints = source / 'docs/knowledge/repository-knowledge.shapes.ttl' + queries = source / 'docs/knowledge/stored-queries.json' + alignments = source / 'docs/knowledge/alignments.ttl' + for required in (registry_path, constraints, queries, alignments): + if not required.is_file() or not required.stat().st_size: + raise ValueError('required repository knowledge member missing: ' + required.name) + import importlib.util + spec = importlib.util.spec_from_file_location('repository_knowledge', source / 'scripts/build-repository-knowledge.py') + generator = importlib.util.module_from_spec(spec); spec.loader.exec_module(generator) + records = json.loads(registry_path.read_text())['records'] + labels = indexed_labels(binary, database, sorted({record['subject'] for record in records})) + knowledge.write_text(generator.render(source, revision, lambda subject: labels.get(subject)) + '\n' + alignment_carrier(source)) + command([binary, 'validate', '--shapes', constraints, '--data', knowledge]) + command([binary, 'shapes', 'load', 'repository-knowledge', constraints, '--db', database]) + command([binary, 'knot', knowledge, '--db', database]) + register_queries(binary, database, json.loads(queries.read_text())) + query = (source / 'queries/repository-share-bobbin.rq').read_text() + stage = private / 'share' + command([binary, 'share', '--construct', query, '--turtle', '--output', stage, '--db', database]) + if not (stage / 'shapes.ttl').is_file() or (stage / 'shapes.ttl').stat().st_size == 0: + raise ValueError('repository share omitted receiver shapes') + receiver = private / 'receiver.db' + command([binary, 'shapes', 'load', 'repository-share', stage / 'shapes.ttl', '--db', receiver]) + imported = json.loads(command([binary, 'import', stage, '--db', receiver])) + if (imported.get('outcome') != 'staged' + or imported.get('promotion', {}).get('eligible') is not True + or type(imported.get('triples', {}).get('accepted')) is not int + or imported['triples']['accepted'] <= 3): + raise ValueError('repository receiver did not admit a non-stub corpus') + command([binary, 'import', 'promote', imported['share_id'], '--db', receiver]) + positive = command([binary, 'query', + 'SELECT ?s WHERE { ?s ' + + json.dumps(control) + ' } LIMIT 1', '--db', receiver]) + if re.search(r'\n1 results\s*$', positive) is None: + raise ValueError('repository corpus positive control missing after import') + command(['python3', tooling / 'scripts/verify-repository-share-receiver.py', + binary, stage, receiver, private]) + if command(['git', '-C', source, 'status', '--porcelain']).strip(): + raise ValueError('indexing changed the exact repository source') + shutil.move(stage, output) + return {'triples': imported['triples']['accepted'], 'positive_file': control} + + +if __name__ == '__main__': + import sys + if len(sys.argv) != 7: + raise SystemExit('usage: build-repository-share.py ') + build(*sys.argv[1:]) diff --git a/scripts/publish-repository-share.sh b/scripts/publish-repository-share.sh new file mode 100644 index 00000000..3b0f0f94 --- /dev/null +++ b/scripts/publish-repository-share.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Repo-owned job, invoked only by the managed trusted-host release dispatcher. +set -euo pipefail +umask 077 +exec python3 "$(dirname -- "${BASH_SOURCE[0]}")/publish_repository_share.py" "$@" diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py new file mode 100644 index 00000000..97669f1e --- /dev/null +++ b/scripts/publish_repository_share.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Bobbin's owned trusted release job; hosted workflows only verify its output.""" +import gzip +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +REPOSITORY = 'scbrown/bobbin' +MEMBERS = {'export.nt', 'export.ttl', 'payload.nq', 'manifest.json', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'} + + +def command(args): + return subprocess.run([str(v) for v in args], check=True, capture_output=True, text=True).stdout + + +def digest(path): + result = hashlib.sha256() + with path.open('rb') as handle: + for block in iter(lambda: handle.read(1 << 20), b''): + result.update(block) + return result.hexdigest() + + +def check_share(stage): + files = list(stage.iterdir()) + if any(not p.is_file() or p.is_symlink() or p.name not in MEMBERS for p in files): + raise ValueError('unexpected or non-regular share member') + for name in ('export.nt', 'manifest.json', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'): + if not (stage / name).is_file() or not (stage / name).stat().st_size: + raise ValueError('required nonempty share member missing: ' + name) + recipe = json.loads((stage / 'manifest.json').read_text()).get('embedding') + if (not isinstance(recipe, dict) or not isinstance(recipe.get('model'), str) + or not recipe['model'] or Path(recipe['model']).name != recipe['model'] + or type(recipe.get('dimension')) is not int or recipe['dimension'] <= 0 + or any(not re.fullmatch('[0-9a-f]{64}', recipe.get(k, '')) + for k in ('model_sha256', 'tokenizer_sha256'))): + raise ValueError('share omitted a provider-byte embedding recipe') + # This repository pack is pointer-only. Curated idea summaries use + # rdfs:comment; source bodies and line spans are never release predicates. + export = (stage / 'export.nt').read_text() + for predicate in ('content', 'startLine', 'endLine'): + if '' in export: + raise ValueError('share duplicates source text or line-span data') + return recipe + + +def package(stage, archive): + check_share(stage) + with archive.open('xb') as raw: + with gzip.GzipFile(filename='', fileobj=raw, mode='wb', mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode='w') as tar: + for path in sorted(stage.iterdir()): + info = tar.gettarinfo(str(path), arcname=path.name) + info.uid = info.gid = 0 + info.uname = info.gname = '' + info.mtime = 0 + info.mode = 0o600 + with path.open('rb') as handle: + tar.addfile(info, handle) + + +def held(): + return Path(os.environ.get('QUIPU_SHARE_HOLD_FILE', str(Path.home() / '.quipu-share-hold'))).exists() + + +def publish(tag, revision, output): + if held(): + raise ValueError('publication held before any lookup/build') + if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag): + raise ValueError('unexpected Bobbin release tag') + if not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('invalid release revision') + source = Path(__file__).resolve().parents[1] + if (command(['git', '-C', source, 'rev-parse', 'HEAD']).strip() != revision + or command(['git', '-C', source, 'rev-parse', 'refs/tags/' + tag + '^{commit}']).strip() != revision + or command(['git', '-C', source, 'status', '--porcelain']).strip()): + raise ValueError('publisher source is not the exact clean release') + output = Path(output) + if output.exists(): + raise ValueError('retained output exists; reconcile before retrying') + binary = Path(os.environ['QUIPU_SHARE_ENGINE_BIN']).resolve(strict=True) + tooling = Path(os.environ['QUIPU_SHARE_ENGINE_SOURCE']).resolve(strict=True) + bobbin = Path(os.environ['QUIPU_SHARE_BOBBIN']).resolve(strict=True) + engine_revision = os.environ['QUIPU_SHARE_ENGINE_REVISION'] + if (not re.fullmatch('[0-9a-f]{40}', engine_revision) + or command(['git', '-C', tooling, 'rev-parse', 'HEAD']).strip() != engine_revision + or command(['git', '-C', tooling, 'status', '--porcelain']).strip() + or digest(binary) != os.environ['QUIPU_SHARE_ENGINE_BIN_SHA256'] + or digest(binary.with_name('quipu-server')) != os.environ['QUIPU_SHARE_ENGINE_SERVER_SHA256'] + or digest(bobbin) != os.environ['QUIPU_SHARE_BOBBIN_SHA256']): + raise ValueError('trusted engine/source/indexer pin mismatch') + output.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with tempfile.TemporaryDirectory(prefix='prepare-', dir=output.parent) as temp: + private = Path(temp) + stage = private / 'share' + spec = importlib.util.spec_from_file_location('bobbin_share_build', source / 'scripts/build-repository-share.py') + builder = importlib.util.module_from_spec(spec); spec.loader.exec_module(builder) + builder.build(revision, source, tooling, binary, bobbin, stage) + archive_name = 'bobbin-' + tag + '-repository.pendant.tar.gz' + prepared = private / 'assets'; prepared.mkdir(mode=0o700) + archive = prepared / archive_name + package(stage, archive) + (prepared / (archive_name + '.sha256')).write_text(digest(archive) + ' ' + archive_name + '\n') + proof = {'schema': 'quipu.repository-share-producer/v1', + 'source_repository': REPOSITORY, 'source_revision': revision, 'release_tag': tag, + 'archive_sha256': digest(archive), 'quipu_sha256': digest(binary), + 'engine_source_revision': engine_revision, 'bobbin_sha256': digest(bobbin), + 'quipu_server_sha256': digest(binary.with_name('quipu-server')), + 'destination': 'outward'} + (prepared / (archive_name + '.provenance.json')).write_text(json.dumps(proof, sort_keys=True, indent=2) + '\n') + names = sorted(p.name for p in prepared.iterdir()) + if held(): + raise ValueError('publication held after preparation') + available = json.loads(command(['gh', 'release', 'view', tag, '--repo', REPOSITORY, '--json', 'assets'])) + present = {a['name'] for a in available['assets']} & set(names) + if present and present != set(names): + raise ValueError('partial publication; explicit reconciliation required') + # Retain before the one-way upload. No --clobber and no automatic retry + # from an uncertain write. The managed dispatcher also retains STARTED. + shutil.copytree(prepared, output) + if not present: + if held(): + raise ValueError('publication held immediately before upload') + command(['gh', 'release', 'upload', tag, '--repo', REPOSITORY, + *(output / name for name in names)]) + downloaded = private / 'downloaded'; downloaded.mkdir() + command(['gh', 'release', 'download', tag, '--repo', REPOSITORY, '--dir', downloaded, + *(arg for name in names for arg in ('--pattern', name))]) + for name in names: + if (downloaded / name).read_bytes() != (output / name).read_bytes(): + raise ValueError('existing or newly published asset differs; never overwrite') + receipt = {**proof, 'published_and_read_back': True, 'assets': names, + 'asset_sha256': {name: digest(downloaded / name) for name in names}} + (output / 'delivery.json').write_text(json.dumps(receipt, indent=2) + '\n') + return receipt + + +if __name__ == '__main__': + if len(sys.argv) != 5 or sys.argv[1] != '--trusted-release': + raise SystemExit('usage: publish-repository-share.sh --trusted-release ') + print(json.dumps(publish(sys.argv[2], sys.argv[3], sys.argv[4]))) diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py new file mode 100644 index 00000000..02699297 --- /dev/null +++ b/scripts/test-repository-knowledge.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Curated-source refusal controls. Lookup fixtures are not index coverage proof. + +# arming: ci; release contract invokes this script. +""" +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('knowledge', ROOT / 'scripts/build-repository-knowledge.py') +module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + + +builder_spec = importlib.util.spec_from_file_location('share_builder', ROOT / 'scripts/build-repository-share.py') +builder = importlib.util.module_from_spec(builder_spec); builder_spec.loader.exec_module(builder) + + +class AlignmentCarrier(unittest.TestCase): + def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); target = root / 'docs/knowledge'; target.mkdir(parents=True) + for name in ('alignments.ttl', 'alignment-status.json'): + shutil.copyfile(ROOT / 'docs/knowledge' / name, target / name) + self.assertIn('@prefix sssom:', builder.alignment_carrier(root)) + data = target / 'alignments.ttl'; original = data.read_text() + data.unlink() + with self.assertRaises(FileNotFoundError): builder.alignment_carrier(root) + data.write_text(original + ' .\n') + with self.assertRaisesRegex(ValueError, 'corrupt'): builder.alignment_carrier(root) + import hashlib + status = json.loads((target / 'alignment-status.json').read_text()) + status['carrier_sha256'] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / 'alignment-status.json').write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, 'mapping data'): builder.alignment_carrier(root) + data.write_text('@prefix owl: . .\n') + status['carrier_sha256'] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / 'alignment-status.json').write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, 'mapping data'): builder.alignment_carrier(root) + + +class Knowledge(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.registry = json.loads((ROOT / 'docs/knowledge/repository-knowledge.json').read_text()) + for source in {r['source'] for r in self.registry['records']}: + target = self.root / source; target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / source, target) + (self.root / 'docs/knowledge').mkdir(parents=True) + self.save() + + def save(self): + (self.root / 'docs/knowledge/repository-knowledge.json').write_text(json.dumps(self.registry)) + + def render(self, lookup=lambda _: 'actual-index-label-fixture'): + return module.render(self.root, '1' * 40, lookup) + + def test_source_bound_nonempty_content(self): + text = self.render() + self.assertIn('2592000', text) + self.assertIn('reviewAfter', text) + self.assertIn('knowledgeKind> "idea"', text) + self.assertIn('interfaceKind> "REST"', text) + self.assertIn('supersessionStatus> "not-recorded-in-source"', text) + self.assertIn('startHereOrder> "3"^^', text) + self.assertNotIn('/content>', text) + self.assertNotIn('owl#sameAs', text) + + def test_absent_or_ambiguous_index_pointer_refuses(self): + for value in (None, False, ''): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'absent or ambiguous'): + self.render(lambda _: value) + + def test_changed_source_refuses(self): + path = self.root / self.registry['records'][0]['source'] + path.write_text(path.read_text() + '\nchanged\n') + with self.assertRaisesRegex(ValueError, 'witness changed'): + self.render() + + def test_guessed_identity_refuses(self): + self.registry['records'][0]['subject'] = 'https://example.org/guessed' + self.save() + with self.assertRaisesRegex(ValueError, 'source-backed'): + self.render() + + def test_missing_required_content_refuses(self): + self.registry['records'] = [r for r in self.registry['records'] if r['kind'] != 'decision'] + self.save() + with self.assertRaisesRegex(ValueError, 'inventory incomplete'): + self.render() + + def test_bad_freshness_and_private_summary_refuse(self): + self.registry['max_age_seconds'] = 0; self.save() + with self.assertRaisesRegex(ValueError, 'freshness'): + self.render() + self.registry['max_age_seconds'] = 30 + self.registry['records'][0]['summary'] = 'read /home/private/file' + self.save() + with self.assertRaisesRegex(ValueError, 'private summary'): + self.render() + + @unittest.skipUnless(os.environ.get('QUIPU_TEST_BIN'), 'native schema control requires explicit test artifact') + def test_actual_native_schema_positive_and_missing_freshness_negative(self): + data = self.root / 'curated.ttl'; data.write_text(self.render()) + args = [os.environ['QUIPU_TEST_BIN'], 'validate', '--shapes', + str(ROOT / 'docs/knowledge/repository-knowledge.shapes.ttl'), '--data', str(data), '--format', 'json'] + positive = subprocess.run(args, capture_output=True, text=True) + self.assertEqual(positive.returncode, 0, positive.stderr + positive.stdout) + data.write_text('\n'.join(line for line in data.read_text().splitlines() if '/reviewAfter>' not in line)) + negative = subprocess.run(args, capture_output=True, text=True) + self.assertNotEqual(negative.returncode, 0, negative.stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py new file mode 100644 index 00000000..c87b7a4c --- /dev/null +++ b/scripts/test-repository-share-publication.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Packaging/readback refusal controls; synthetic data is not native acceptance. + +# arming: ci; release asset publication contract invokes this script. +""" +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + +publisher = load('publisher_test', ROOT / 'publish_repository_share.py') +verifier = load('verifier_test', ROOT / 'verify-repository-share.py') + + +class Publication(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.stage = self.root / 'stage'; self.stage.mkdir() + recipe = {'model': 'model.onnx', 'model_sha256': 'a' * 64, + 'tokenizer_sha256': 'b' * 64, 'dimension': 384} + (self.stage / 'manifest.json').write_text(json.dumps({'embedding': recipe})) + for name in ('export.nt', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'): + (self.stage / name).write_text('# synthetic fixture only\n') + self.archive = self.root / 'bobbin-v1.2.3-repository.pendant.tar.gz' + self.revision = '1' * 40 + + def prepare(self): + publisher.package(self.stage, self.archive) + sha = publisher.digest(self.archive) + self.archive.with_name(self.archive.name + '.sha256').write_text(sha + ' ' + self.archive.name + '\n') + self.proof = {'schema': 'quipu.repository-share-producer/v1', 'source_repository': 'scbrown/bobbin', + 'source_revision': self.revision, 'release_tag': 'v1.2.3', 'archive_sha256': sha, + 'destination': 'outward', 'quipu_sha256': '2' * 64, 'quipu_server_sha256': '5' * 64, 'bobbin_sha256': '3' * 64, + 'engine_source_revision': '4' * 40} + self.proof_path = self.archive.with_name(self.archive.name + '.provenance.json') + self.proof_path.write_text(json.dumps(self.proof)) + + def test_deterministic_archive_and_exact_readback(self): + self.prepare() + other = self.root / 'second.tar.gz'; publisher.package(self.stage, other) + self.assertEqual(self.archive.read_bytes(), other.read_bytes()) + self.assertEqual(verifier.verify(self.root, 'v1.2.3', self.revision)['source_revision'], self.revision) + + def test_required_members_and_recipe(self): + (self.stage / 'queries.ttl').unlink() + with self.assertRaisesRegex(ValueError, 'required nonempty'): + publisher.check_share(self.stage) + (self.stage / 'queries.ttl').write_text('# query') + (self.stage / 'manifest.json').write_text('{}') + with self.assertRaisesRegex(ValueError, 'provider-byte'): + publisher.check_share(self.stage) + + def test_text_line_span_and_private_members_refuse(self): + for predicate in ('content', 'startLine', 'endLine'): + (self.stage / 'export.nt').write_text(' "body" .\n') + with self.subTest(predicate=predicate), self.assertRaisesRegex(ValueError, 'duplicates source'): + publisher.check_share(self.stage) + (self.stage / 'export.nt').write_text('# empty synthetic') + (self.stage / 'policy-token').write_text('not a real token') + with self.assertRaisesRegex(ValueError, 'unexpected'): + publisher.check_share(self.stage) + + def test_foreign_repository_and_wrong_source_refuse(self): + self.prepare() + for field, value in [('source_repository', 'scbrown/quipu'), ('source_revision', 'f' * 40), + ('release_tag', 'v9.9.9'), ('archive_sha256', 'f' * 64)]: + self.proof_path.write_text(json.dumps({**self.proof, field: value})) + with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'provenance mismatch'): + verifier.verify(self.root, 'v1.2.3', self.revision) + + def test_checksum_refuses(self): + self.prepare() + self.archive.with_name(self.archive.name + '.sha256').write_text('wrong') + with self.assertRaisesRegex(ValueError, 'checksum'): + verifier.verify(self.root, 'v1.2.3', self.revision) + + def test_wait_is_bounded(self): + calls, pauses = [], [] + class Result: + returncode = 1 + def run(args, **kwargs): + calls.append(args); return Result() + with self.assertRaisesRegex(ValueError, 'bounded'): + verifier.wait('v1.2.3', self.revision, attempts=2, pause=pauses.append, run=run) + self.assertEqual(len(calls), 2) + self.assertEqual(pauses, [30]) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test-repository-share-query.py b/scripts/test-repository-share-query.py new file mode 100644 index 00000000..be31d8b2 --- /dev/null +++ b/scripts/test-repository-share-query.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Private corpus controls for Bobbin's copied pointer-only release query. + +# arming: agent-invoked; run before publishing the repository-pendant source. +""" +from pathlib import Path +import unittest +from rdflib import Graph, Literal, Namespace, RDF, RDFS, URIRef + +A = Namespace('http://aegis.gastown.local/ontology/') +CODE = Namespace(str(A) + 'code/bobbin/') +DOC = Namespace(str(A) + 'doc/bobbin/') +QUERY = (Path(__file__).resolve().parents[1] / 'queries/repository-share-bobbin.rq').read_text() + + +class RepositoryQuery(unittest.TestCase): + def test_pointer_corpus_and_private_boundaries(self): + source = Graph() + doc, chunk = DOC['README.md'], DOC['README.md/chunk/0'] + expected = { + (doc, RDF.type, A.Document), + (doc, A.filePath, Literal('README.md')), + (chunk, RDF.type, A.Chunk), + (chunk, A.inDocument, doc), + (chunk, A.chunkOrder, Literal(0)), + (chunk, A.mentions, Literal('Store')), + (CODE['store.rs'], A.filePath, Literal('src/store.rs')), + } + for triple in expected: + source.add(triple) + blocked = { + (chunk, A.content, Literal('body must not be exported')), + (chunk, A.startLine, Literal(1)), + (chunk, A.endLine, Literal(30)), + (chunk, A.mentions, URIRef('https://example.org/resolved')), + (DOC['private'], RDFS.label, Literal('aegis-private123')), + (DOC['private'], RDF.type, A.Document), + (URIRef(str(A) + 'doc/other/README.md'), A.filePath, Literal('foreign')), + (URIRef('urn:quipu:memory:private'), RDFS.label, Literal('memory')), + } + for triple in blocked: + source.add(triple) + result = set(source.query(QUERY).graph) + self.assertEqual(result, expected) + + def test_ideas_and_sssom_records_survive_without_identity_knots(self): + source = Graph() + S = Namespace('https://w3id.org/sssom/') + OWL = Namespace('http://www.w3.org/2002/07/owl#') + record = URIRef('https://example.org/fixture/mapping') + pointer = DOC['guide.md#idea'] + other = URIRef('https://example.org/other/idea') + expected = { + (pointer, RDF.type, A.Concept), + (pointer, RDFS.comment, Literal('Curated high-level idea')), + (pointer, A.reviewAfter, Literal('2026-11-01T00:00:00Z')), + (record, RDF.type, OWL.Axiom), + (record, S.subject_id, pointer), + (record, S.object_id, other), + (record, S.predicate_id, URIRef('http://www.w3.org/2004/02/skos/core#closeMatch')), + (record, S.mapping_justification, URIRef('https://w3id.org/semapv/vocab/LexicalMatching')), + } + for triple in expected: source.add(triple) + source.add((pointer, OWL.sameAs, other)) + self.assertEqual(set(source.query(QUERY).graph), expected) + + def test_empty_is_empty_with_populated_positive(self): + self.assertEqual(len(Graph().query(QUERY).graph), 0) + source = Graph() + triple = (DOC['README.md'], A.filePath, Literal('README.md')) + source.add(triple) + self.assertEqual(set(source.query(QUERY).graph), {triple}) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/verify-repository-share.py b/scripts/verify-repository-share.py new file mode 100644 index 00000000..7cb71999 --- /dev/null +++ b/scripts/verify-repository-share.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Bounded hosted readback; build policy and credentials stay on trusted host.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import time + +spec = importlib.util.spec_from_file_location('publisher', Path(__file__).with_name('publish_repository_share.py')) +publisher = importlib.util.module_from_spec(spec); spec.loader.exec_module(publisher) + + +def verify(folder, tag, revision): + if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag) or not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('invalid release identity') + archive = folder / ('bobbin-' + tag + '-repository.pendant.tar.gz') + sha = hashlib.sha256(archive.read_bytes()).hexdigest() + if archive.with_name(archive.name + '.sha256').read_text() != sha + ' ' + archive.name + '\n': + raise ValueError('archive checksum mismatch') + proof = json.loads(archive.with_name(archive.name + '.provenance.json').read_text()) + expected = {'schema': 'quipu.repository-share-producer/v1', 'source_repository': 'scbrown/bobbin', + 'source_revision': revision, 'release_tag': tag, 'archive_sha256': sha, 'destination': 'outward'} + if any(proof.get(k) != v for k, v in expected.items()): + raise ValueError('repository/source/tag/provenance mismatch') + for key in ('quipu_sha256', 'quipu_server_sha256', 'bobbin_sha256'): + if not re.fullmatch('[0-9a-f]{64}', proof.get(key, '')): + raise ValueError('missing engine/indexer digest') + if not re.fullmatch('[0-9a-f]{40}', proof.get('engine_source_revision', '')): + raise ValueError('missing engine source pin') + with tempfile.TemporaryDirectory(prefix='verify-pendant-') as temp: + stage = Path(temp) + with tarfile.open(archive) as tar: + members = tar.getmembers() + if (len(members) != len({m.name for m in members}) + or any(not m.isfile() or m.name not in publisher.MEMBERS for m in members)): + raise ValueError('unexpected, duplicate, or unsafe archive member') + # Every member is a flat allowlisted regular file; no path, link or device extraction. + for member in members: + (stage / member.name).write_bytes(tar.extractfile(member).read()) + publisher.check_share(stage) + return expected + + +def wait(tag, revision, attempts=80, pause=time.sleep, run=subprocess.run): + if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag) or not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('invalid release identity before lookup') + stem = 'bobbin-' + tag + '-repository.pendant.tar.gz' + last_read = 'not attempted' + for number in range(attempts): + with tempfile.TemporaryDirectory(prefix='repository-pendant-readback-') as temp: + folder = Path(temp) + try: + result = run(['gh', 'release', 'download', tag, '--repo', 'scbrown/bobbin', '--dir', temp, + '--pattern', stem, '--pattern', stem + '.sha256', '--pattern', stem + '.provenance.json'], + capture_output=True, text=True, timeout=45) + last_read = 'exit=' + str(result.returncode) + if result.returncode == 0: + return verify(folder, tag, revision) + except subprocess.TimeoutExpired: + last_read = 'read timeout' + if number + 1 < attempts: + pause(30) + raise ValueError('repository pendant unverified after bounded trusted-producer wait; ' + last_read) + + +if __name__ == '__main__': + if len(sys.argv) != 3: + raise SystemExit('usage: verify-repository-share.py ') + print(json.dumps(wait(sys.argv[1], sys.argv[2]))) From a61cc39e456daae7f9434200c7e5ffecdd67386d Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 18:54:25 -0400 Subject: [PATCH 02/12] fix: refresh README source witness after signing-key guidance --- docs/knowledge/repository-knowledge.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/knowledge/repository-knowledge.json b/docs/knowledge/repository-knowledge.json index 2b78b784..71225a16 100644 --- a/docs/knowledge/repository-knowledge.json +++ b/docs/knowledge/repository-knowledge.json @@ -1,5 +1,5 @@ { - "source_revision": "2b1ba22cb0f2b8d26354507c3eb1b6fe6890f7f8", + "source_revision": "b036433f55b75ddb8520d5bf4ad0b681464187eb", "reviewed_at": "2026-10-20T22:30:00Z", "max_age_seconds": 2592000, "records": [ @@ -109,7 +109,7 @@ "source": "README.md", "subject": "http://aegis.gastown.local/ontology/doc/bobbin/README.md", "summary": "Start with installation and the first successful search, then read architecture and contributor guidance.", - "source_sha256": "b919f7e1618d9819ad6057cd3dfc296aff12fe82863bf2e49320bdebf0c55b1f", + "source_sha256": "77189862c0136aad56cfb5407ebb7ae99e913f6823105ffc146217a54653686d", "properties": { "startHereOrder": "1" } From 374c6d9254c266a7433815b185cfa20ed19e4a45 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 18:56:40 -0400 Subject: [PATCH 03/12] fix: keep native pendant fixture state private --- .gitignore | 3 ++ scripts/build-repository-share.py | 42 +++++++++++++++------------- scripts/test-repository-knowledge.py | 4 +-- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 0de68c95..880a1c19 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ book/_build/ *.db .beads-credential-key *.gate.lock* + +# Quipu local databases and signing state are never repository source. +/.quipu/ diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py index e7bfd515..6e019a80 100644 --- a/scripts/build-repository-share.py +++ b/scripts/build-repository-share.py @@ -14,9 +14,9 @@ import subprocess -def command(args, *, env=None): +def command(args, *, env=None, cwd=None): return subprocess.run([str(v) for v in args], check=True, capture_output=True, - text=True, env=env).stdout + text=True, env=env, cwd=cwd).stdout def local_mcp(binary, database, calls): @@ -34,7 +34,7 @@ def local_mcp(binary, database, calls): values = [] with subprocess.Popen([str(binary), 'mcp', '--db', str(database)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True) as process: + text=True, cwd=Path(database).parent) as process: selector = selectors.DefaultSelector() selector.register(process.stdout, selectors.EVENT_READ) try: @@ -136,8 +136,10 @@ def build(revision, source, tooling, binary, bobbin, output): raise ValueError('release lacks a supported public corpus positive control') private = output.parent / 'build-private' private.mkdir(mode=0o700) + def run(args, *, env=None): + return command(args, env=env, cwd=private) policy = private / 'policy.ttl' - command(['python3', tooling / 'scripts/private-share-policy.py', policy]) + run(['python3', tooling / 'scripts/private-share-policy.py', policy]) index = private / 'index'; (index / '.bobbin').mkdir(parents=True) (index / '.bobbin/config.toml').write_text('''quipu_push_chunks = true @@ -153,7 +155,7 @@ def build(revision, source, tooling, binary, bobbin, output): ''') env = {k: v for k, v in os.environ.items() if k not in ('BOBBIN_SERVER', 'BOBBIN_QUIPU_REMOTE')} - command([bobbin, 'index', index, '--source', source, '--repo', slug, + run([bobbin, 'index', index, '--source', source, '--repo', slug, '--force', '--skip-calibrate', '--json'], env=env) database = private / 'producer.db' index_database = index / '.bobbin/quipu/quipu.db' @@ -166,11 +168,11 @@ def build(revision, source, tooling, binary, bobbin, output): if not shapes: raise ValueError('verified engine source has no receiver shapes') for shape in shapes: - command([binary, 'shapes', 'load', shape.stem, shape, '--db', database]) - command([binary, 'shapes', 'load', 'publication-policy', + run([binary, 'shapes', 'load', shape.stem, shape, '--db', database]) + run([binary, 'shapes', 'load', 'publication-policy', tooling / 'docs/knowledge/publication-policy.shapes.ttl', '--db', database]) - command([binary, 'knot', tooling / 'docs/knowledge/publication-policy.ttl', '--db', database]) - command([binary, 'knot', policy, '--graph', 'urn:quipu:private-release-policy', '--db', database]) + run([binary, 'knot', tooling / 'docs/knowledge/publication-policy.ttl', '--db', database]) + run([binary, 'knot', policy, '--graph', 'urn:quipu:private-release-policy', '--db', database]) context = private / 'context.ttl' context.write_text(f'''@prefix aegis: . @prefix rdfs: . @@ -180,7 +182,7 @@ def build(revision, source, tooling, binary, bobbin, output): rdfs:label "Repository-embedded knowledge databases" ; aegis:issuedBy "Stiwi" ; aegis:applies_to aegis:repo_{slug} . ''') - command([binary, 'knot', context, '--db', database]) + run([binary, 'knot', context, '--db', database]) # Curated repository content and validating shapes are repo-owned. Refuse # an incomplete source proposal rather than minting a pointer-only stub. registry_path = source / 'docs/knowledge/repository-knowledge.json' @@ -197,32 +199,32 @@ def build(revision, source, tooling, binary, bobbin, output): records = json.loads(registry_path.read_text())['records'] labels = indexed_labels(binary, database, sorted({record['subject'] for record in records})) knowledge.write_text(generator.render(source, revision, lambda subject: labels.get(subject)) + '\n' + alignment_carrier(source)) - command([binary, 'validate', '--shapes', constraints, '--data', knowledge]) - command([binary, 'shapes', 'load', 'repository-knowledge', constraints, '--db', database]) - command([binary, 'knot', knowledge, '--db', database]) + run([binary, 'validate', '--shapes', constraints, '--data', knowledge]) + run([binary, 'shapes', 'load', 'repository-knowledge', constraints, '--db', database]) + run([binary, 'knot', knowledge, '--db', database]) register_queries(binary, database, json.loads(queries.read_text())) query = (source / 'queries/repository-share-bobbin.rq').read_text() stage = private / 'share' - command([binary, 'share', '--construct', query, '--turtle', '--output', stage, '--db', database]) + run([binary, 'share', '--construct', query, '--turtle', '--output', stage, '--db', database]) if not (stage / 'shapes.ttl').is_file() or (stage / 'shapes.ttl').stat().st_size == 0: raise ValueError('repository share omitted receiver shapes') receiver = private / 'receiver.db' - command([binary, 'shapes', 'load', 'repository-share', stage / 'shapes.ttl', '--db', receiver]) - imported = json.loads(command([binary, 'import', stage, '--db', receiver])) + run([binary, 'shapes', 'load', 'repository-share', stage / 'shapes.ttl', '--db', receiver]) + imported = json.loads(run([binary, 'import', stage, '--db', receiver])) if (imported.get('outcome') != 'staged' or imported.get('promotion', {}).get('eligible') is not True or type(imported.get('triples', {}).get('accepted')) is not int or imported['triples']['accepted'] <= 3): raise ValueError('repository receiver did not admit a non-stub corpus') - command([binary, 'import', 'promote', imported['share_id'], '--db', receiver]) - positive = command([binary, 'query', + run([binary, 'import', 'promote', imported['share_id'], '--db', receiver]) + positive = run([binary, 'query', 'SELECT ?s WHERE { ?s ' + json.dumps(control) + ' } LIMIT 1', '--db', receiver]) if re.search(r'\n1 results\s*$', positive) is None: raise ValueError('repository corpus positive control missing after import') - command(['python3', tooling / 'scripts/verify-repository-share-receiver.py', + run(['python3', tooling / 'scripts/verify-repository-share-receiver.py', binary, stage, receiver, private]) - if command(['git', '-C', source, 'status', '--porcelain']).strip(): + if run(['git', '-C', source, 'status', '--porcelain']).strip(): raise ValueError('indexing changed the exact repository source') shutil.move(stage, output) return {'triples': imported['triples']['accepted'], 'positive_file': control} diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index 02699297..d614ee23 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -110,10 +110,10 @@ def test_actual_native_schema_positive_and_missing_freshness_negative(self): data = self.root / 'curated.ttl'; data.write_text(self.render()) args = [os.environ['QUIPU_TEST_BIN'], 'validate', '--shapes', str(ROOT / 'docs/knowledge/repository-knowledge.shapes.ttl'), '--data', str(data), '--format', 'json'] - positive = subprocess.run(args, capture_output=True, text=True) + positive = subprocess.run(args, capture_output=True, text=True, cwd=self.root) self.assertEqual(positive.returncode, 0, positive.stderr + positive.stdout) data.write_text('\n'.join(line for line in data.read_text().splitlines() if '/reviewAfter>' not in line)) - negative = subprocess.run(args, capture_output=True, text=True) + negative = subprocess.run(args, capture_output=True, text=True, cwd=self.root) self.assertNotEqual(negative.returncode, 0, negative.stdout) From 8f9283d26a810b145cf1f5817816738fa93ea212 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:05:33 -0400 Subject: [PATCH 04/12] fix: isolate the native source graph during pendant indexing --- scripts/build-repository-share.py | 33 ++++++++++++++++++++++++++-- scripts/test-repository-knowledge.py | 18 +++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py index 6e019a80..cd8faebc 100644 --- a/scripts/build-repository-share.py +++ b/scripts/build-repository-share.py @@ -11,6 +11,8 @@ import re import shutil import sqlite3 +import io +import tarfile import subprocess @@ -93,6 +95,28 @@ def indexed_labels(binary, database, subjects): return labels +def materialize_release(source, revision, destination): + """Index only exact tracked release bytes in a new private source tree.""" + destination = Path(destination) + destination.mkdir(mode=0o700) + archive = subprocess.run(['git', '-C', str(source), 'archive', '--format=tar', revision], + check=True, capture_output=True).stdout + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:') as bundle: + for member in bundle.getmembers(): + relative = Path(member.name) + if (relative.is_absolute() or '..' in relative.parts + or not (member.isfile() or member.isdir())): + raise ValueError('unsupported tracked release member') + target = destination / relative + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with bundle.extractfile(member) as original, target.open('xb') as copied: + shutil.copyfileobj(original, copied) + return destination + + def alignment_carrier(source): """An explicit empty reviewed carrier differs from missing or corrupt data. @@ -140,6 +164,12 @@ def run(args, *, env=None): return command(args, env=env, cwd=private) policy = private / 'policy.ttl' run(['python3', tooling / 'scripts/private-share-policy.py', policy]) + corpus = materialize_release(source, revision, private / 'source') + (corpus / '.bobbin').mkdir(exist_ok=True) + index_database = private / 'indexed-graph.db' + # Bobbin resolves embedded graph state at --source, not at the vector + # index root. Explicit private configuration avoids global-store fallback. + (corpus / '.bobbin/config.toml').write_text('[quipu]\nstore_path = ' + json.dumps(str(index_database)) + '\n') index = private / 'index'; (index / '.bobbin').mkdir(parents=True) (index / '.bobbin/config.toml').write_text('''quipu_push_chunks = true @@ -155,10 +185,9 @@ def run(args, *, env=None): ''') env = {k: v for k, v in os.environ.items() if k not in ('BOBBIN_SERVER', 'BOBBIN_QUIPU_REMOTE')} - run([bobbin, 'index', index, '--source', source, '--repo', slug, + run([bobbin, 'index', index, '--source', corpus, '--repo', slug, '--force', '--skip-calibrate', '--json'], env=env) database = private / 'producer.db' - index_database = index / '.bobbin/quipu/quipu.db' if not index_database.is_file(): raise ValueError('indexer did not produce the repository graph') with sqlite3.connect('file:' + str(index_database) + '?mode=ro', uri=True) as original: diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index d614ee23..0841ad7e 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -21,6 +21,24 @@ builder = importlib.util.module_from_spec(builder_spec); builder_spec.loader.exec_module(builder) +class ReleaseSnapshot(unittest.TestCase): + def test_real_git_archive_excludes_untracked_key_and_preserves_source(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); source = root / 'source'; source.mkdir() + subprocess.run(['git', 'init', '-q', str(source)], check=True) + (source / 'README.md').write_text('tracked pointer source\n') + subprocess.run(['git', '-C', str(source), 'add', 'README.md'], check=True) + subprocess.run(['git', '-C', str(source), '-c', 'user.name=fixture', + '-c', 'user.email=fixture@example.org', 'commit', '-qm', 'fixture'], check=True) + (source / 'private.key').write_text('synthetic untracked sentinel') + revision = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() + copied = builder.materialize_release(source, revision, root / 'copied') + self.assertEqual((copied / 'README.md').read_bytes(), (source / 'README.md').read_bytes()) + self.assertFalse((copied / 'private.key').exists()) + self.assertEqual((source / 'private.key').read_text(), 'synthetic untracked sentinel') + self.assertFalse((source / '.bobbin').exists()) + + class AlignmentCarrier(unittest.TestCase): def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): with tempfile.TemporaryDirectory() as folder: From ce7e572ecdcfa39545488c27bef0b984867aa294 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:42:53 -0400 Subject: [PATCH 05/12] fix: retain native chunk pointer namespace in release query --- queries/repository-share-bobbin.rq | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/queries/repository-share-bobbin.rq b/queries/repository-share-bobbin.rq index e855fe76..6715a3a6 100644 --- a/queries/repository-share-bobbin.rq +++ b/queries/repository-share-bobbin.rq @@ -16,7 +16,8 @@ WHERE { ?s ?p ?o . FILTER( STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/code/bobbin/") || - STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/doc/bobbin/") + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/doc/bobbin/") || + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/chunk/bobbin/") ) VALUES ?p { rdf:type rdfs:label From ac15bd43a29877a06316650c031f97140edc9827 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:43:50 -0400 Subject: [PATCH 06/12] test: cover actual native chunk pointer identity --- scripts/test-repository-share-query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-repository-share-query.py b/scripts/test-repository-share-query.py index be31d8b2..bcf2d1a5 100644 --- a/scripts/test-repository-share-query.py +++ b/scripts/test-repository-share-query.py @@ -16,7 +16,7 @@ class RepositoryQuery(unittest.TestCase): def test_pointer_corpus_and_private_boundaries(self): source = Graph() - doc, chunk = DOC['README.md'], DOC['README.md/chunk/0'] + doc, chunk = DOC['README.md'], URIRef(str(A) + 'chunk/bobbin/README.md#C1') expected = { (doc, RDF.type, A.Document), (doc, A.filePath, Literal('README.md')), From 6254cb5d7287eaebc9070ce948d1298eb6e25fd2 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:50:45 -0400 Subject: [PATCH 07/12] fix: fail closed when publication hold cannot be cleared --- scripts/publish_repository_share.py | 9 ++++++++- scripts/test-repository-share-publication.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py index 97669f1e..8187668e 100644 --- a/scripts/publish_repository_share.py +++ b/scripts/publish_repository_share.py @@ -68,7 +68,14 @@ def package(stage, archive): def held(): - return Path(os.environ.get('QUIPU_SHARE_HOLD_FILE', str(Path.home() / '.quipu-share-hold'))).exists() + path = Path(os.environ.get("QUIPU_SHARE_HOLD_FILE", str(Path.home() / ".quipu-share-hold"))) + try: + path.lstat() + except FileNotFoundError: + return False + except OSError as error: + raise ValueError("cannot inspect publication hold") from error + return True def publish(tag, revision, output): diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py index c87b7a4c..5dfb5f9d 100644 --- a/scripts/test-repository-share-publication.py +++ b/scripts/test-repository-share-publication.py @@ -8,6 +8,7 @@ from pathlib import Path import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parent @@ -83,6 +84,21 @@ def test_checksum_refuses(self): with self.assertRaisesRegex(ValueError, 'checksum'): verifier.verify(self.root, 'v1.2.3', self.revision) + def test_hold_absent_and_dangling_symlink_before_any_build(self): + hold = self.root / "hold" + with patch.dict(publisher.os.environ, {"QUIPU_SHARE_HOLD_FILE": str(hold)}): + self.assertFalse(publisher.held()) + hold.symlink_to(self.root / "missing-target") + self.assertTrue(publisher.held()) + with self.assertRaisesRegex(ValueError, "held before"): + publisher.publish("invalid-tag", "invalid-revision", self.root / "output") + self.assertFalse((self.root / "output").exists()) + + def test_unreadable_hold_fails_closed(self): + with patch.object(Path, "lstat", side_effect=PermissionError("synthetic denied")): + with self.assertRaisesRegex(ValueError, "cannot inspect publication hold"): + publisher.held() + def test_wait_is_bounded(self): calls, pauses = [], [] class Result: From 7e296b9276effd207a676b990cf2a1a4279c8c58 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 20:32:35 -0400 Subject: [PATCH 08/12] fix: isolate pendant workers and preserve verifier checkout (aegis-ro425e.16) --- scripts/build-repository-share.py | 510 +++++++++++++------ scripts/publish_repository_share.py | 243 ++++++--- scripts/test-repository-knowledge.py | 245 ++++++--- scripts/test-repository-share-publication.py | 194 +++++-- scripts/verify-repository-share.py | 114 +++-- 5 files changed, 937 insertions(+), 369 deletions(-) diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py index cd8faebc..586b849f 100644 --- a/scripts/build-repository-share.py +++ b/scripts/build-repository-share.py @@ -4,93 +4,176 @@ live policy, databases and receiver probes stay in the private working directory. Only the verified share directory leaves this function. """ + import hashlib +import io import json import os -from pathlib import Path import re import shutil import sqlite3 -import io -import tarfile import subprocess +import sys +import tarfile +from pathlib import Path + + +def isolated_environment(private): + """Native workers must not inherit a host endpoint or credential catalogue.""" + private = Path(private) + env = dict(os.environ) + for key in list(env): + if key in { + "BOBBIN_SERVER", + "QUIPU_SERVER", + "QUIPU_DB", + "QUIPU_AUTH_TOKEN", + "QUIPU_AUTH_TOKEN_FILE", + "GH_TOKEN", + "GITHUB_TOKEN", + } or key.startswith("BOBBIN_QUIPU_"): + env.pop(key) + for key, name in ( + ("HOME", "native-home"), + ("XDG_CONFIG_HOME", "native-config"), + ("XDG_DATA_HOME", "native-data"), + ("XDG_CACHE_HOME", "native-cache"), + ): + directory = private / name + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + env[key] = str(directory) + return env def command(args, *, env=None, cwd=None): - return subprocess.run([str(v) for v in args], check=True, capture_output=True, - text=True, env=env, cwd=cwd).stdout + if env is None and cwd is not None: + env = isolated_environment(cwd) + return subprocess.run( + [str(v) for v in args], + check=True, + capture_output=True, + text=True, + env=env, + cwd=cwd, + ).stdout def local_mcp(binary, database, calls): """Short-lived native process over a private DB; no HTTP or live graph.""" import selectors + requests = [ - {'jsonrpc': '2.0', 'id': 0, 'method': 'initialize', - 'params': {'protocolVersion': '2024-11-05', 'capabilities': {}, - 'clientInfo': {'name': 'repository-pendant-build', 'version': '1'}}}, - {'jsonrpc': '2.0', 'method': 'notifications/initialized'}, + { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "repository-pendant-build", "version": "1"}, + }, + }, + {"jsonrpc": "2.0", "method": "notifications/initialized"}, ] - requests.extend({'jsonrpc': '2.0', 'id': number, 'method': 'tools/call', - 'params': {'name': name, 'arguments': arguments}} - for number, (name, arguments) in enumerate(calls, 1)) + requests.extend( + { + "jsonrpc": "2.0", + "id": number, + "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + } + for number, (name, arguments) in enumerate(calls, 1) + ) values = [] - with subprocess.Popen([str(binary), 'mcp', '--db', str(database)], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, cwd=Path(database).parent) as process: + with subprocess.Popen( + [str(binary), "mcp", "--db", str(database)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=Path(database).parent, + env=isolated_environment(Path(database).parent), + ) as process: selector = selectors.DefaultSelector() selector.register(process.stdout, selectors.EVENT_READ) try: for request in requests: - process.stdin.write(json.dumps(request) + '\n'); process.stdin.flush() - if 'id' not in request: + process.stdin.write(json.dumps(request) + "\n") + process.stdin.flush() + if "id" not in request: continue if not selector.select(timeout=30): - raise ValueError('native MCP response timeout') + raise ValueError("native MCP response timeout") response = json.loads(process.stdout.readline()) - if (response.get('id') != request['id'] or 'result' not in response - or 'error' in response or response['result'].get('isError', False)): - raise ValueError('native MCP refused operation: ' + json.dumps(response)) - if request['id']: - content = response['result'].get('content', []) - if len(content) != 1 or content[0].get('type') != 'text': - raise ValueError('unexpected native MCP result envelope') - values.append(json.loads(content[0]['text'])) + if ( + response.get("id") != request["id"] + or "result" not in response + or "error" in response + or response["result"].get("isError", False) + ): + raise ValueError( + "native MCP refused operation: " + json.dumps(response) + ) + if request["id"]: + content = response["result"].get("content", []) + if len(content) != 1 or content[0].get("type") != "text": + raise ValueError("unexpected native MCP result envelope") + values.append(json.loads(content[0]["text"])) finally: - selector.close(); process.stdin.close() + selector.close() + process.stdin.close() try: process.wait(timeout=10) except subprocess.TimeoutExpired: - process.terminate(); process.wait(timeout=10) + process.terminate() + process.wait(timeout=10) if process.returncode: - raise ValueError('native MCP process failed') + raise ValueError("native MCP process failed") return values def register_queries(binary, database, definitions): if not isinstance(definitions, list) or not definitions: - raise ValueError('explicit repository stored query definitions required') + raise ValueError("explicit repository stored query definitions required") for definition in definitions: - if not isinstance(definition, dict) or definition.get('action', 'load') != 'load': - raise ValueError('stored query must be an explicit load definition') - results = local_mcp(binary, database, [('quipu_queries', {**d, 'action': 'load'}) for d in definitions]) - if [result.get('loaded') for result in results] != [d['name'] for d in definitions]: - raise ValueError('native registry did not return the exact loaded query names') + if ( + not isinstance(definition, dict) + or definition.get("action", "load") != "load" + ): + raise ValueError("stored query must be an explicit load definition") + results = local_mcp( + binary, + database, + [("quipu_queries", {**d, "action": "load"}) for d in definitions], + ) + if [result.get("loaded") for result in results] != [d["name"] for d in definitions]: + raise ValueError("native registry did not return the exact loaded query names") def indexed_labels(binary, database, subjects): - if any(not re.fullmatch(r'https?://[^\s<>"{}|\\^`]+', subject) for subject in subjects): - raise ValueError('invalid indexed subject IRI') - query = 'SELECT ?s ?label WHERE { VALUES ?s { ' + ' '.join('<' + s + '>' for s in subjects) + '} ?s ?label }' - result, = local_mcp(binary, database, [('quipu_query', {'query': query, 'verbose': True})]) - if result.get('truncated') is not False: - raise ValueError('indexed pointer lookup was incomplete') + if any( + not re.fullmatch(r'https?://[^\s<>"{}|\\^`]+', subject) for subject in subjects + ): + raise ValueError("invalid indexed subject IRI") + query = ( + "SELECT ?s ?label WHERE { VALUES ?s { " + + " ".join("<" + s + ">" for s in subjects) + + "} ?s ?label }" + ) + (result,) = local_mcp( + binary, database, [("quipu_query", {"query": query, "verbose": True})] + ) + if result.get("truncated") is not False: + raise ValueError("indexed pointer lookup was incomplete") labels = {} - for row in result['rows']: - subject, label = row['s'], row['label'] - if isinstance(subject, dict): subject = subject['value'] - if isinstance(label, dict): label = label['value'] + for row in result["rows"]: + subject, label = row["s"], row["label"] + if isinstance(subject, dict): + subject = subject["value"] + if isinstance(label, dict): + label = label["value"] if subject in labels and labels[subject] != label: - raise ValueError('ambiguous indexed pointer label') + raise ValueError("ambiguous indexed pointer label") labels[subject] = label return labels @@ -99,20 +182,29 @@ def materialize_release(source, revision, destination): """Index only exact tracked release bytes in a new private source tree.""" destination = Path(destination) destination.mkdir(mode=0o700) - archive = subprocess.run(['git', '-C', str(source), 'archive', '--format=tar', revision], - check=True, capture_output=True).stdout - with tarfile.open(fileobj=io.BytesIO(archive), mode='r:') as bundle: + archive = subprocess.run( + ["git", "-C", str(source), "archive", "--format=tar", revision], + check=True, + capture_output=True, + ).stdout + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as bundle: for member in bundle.getmembers(): relative = Path(member.name) - if (relative.is_absolute() or '..' in relative.parts - or not (member.isfile() or member.isdir())): - raise ValueError('unsupported tracked release member') + if ( + relative.is_absolute() + or ".." in relative.parts + or not (member.isfile() or member.isdir()) + ): + raise ValueError("unsupported tracked release member") target = destination / relative if member.isdir(): target.mkdir(parents=True, exist_ok=True) else: target.parent.mkdir(parents=True, exist_ok=True) - with bundle.extractfile(member) as original, target.open('xb') as copied: + with ( + bundle.extractfile(member) as original, + target.open("xb") as copied, + ): shutil.copyfileobj(original, copied) return destination @@ -124,21 +216,28 @@ def alignment_carrier(source): not interpret a dependency, lexical match, or candidate as an identity. """ source = Path(source) - data = (source / 'docs/knowledge/alignments.ttl').read_bytes() - status = json.loads((source / 'docs/knowledge/alignment-status.json').read_text()) - if (status.get('status') != 'no-approved-mappings' - or type(status.get('approved_record_count')) is not int - or status['approved_record_count'] != 0 - or status.get('carrier_sha256') != hashlib.sha256(data).hexdigest()): - raise ValueError('unreviewed or corrupt alignment carrier') + data = (source / "docs/knowledge/alignments.ttl").read_bytes() + status = json.loads((source / "docs/knowledge/alignment-status.json").read_text()) + if ( + status.get("status") != "no-approved-mappings" + or type(status.get("approved_record_count")) is not int + or status["approved_record_count"] != 0 + or status.get("carrier_sha256") != hashlib.sha256(data).hexdigest() + ): + raise ValueError("unreviewed or corrupt alignment carrier") # Only declarations/comments are permitted in the explicit-empty profile. # The native data validator below still parses the complete Turtle input. for line in data.decode().splitlines(): - if (line.strip() and not line.lstrip().startswith('#') - and line.strip() not in { - '@prefix sssom: .', - '@prefix owl: .'}): - raise ValueError('empty alignment profile contains mapping data') + if ( + line.strip() + and not line.lstrip().startswith("#") + and line.strip() + not in { + "@prefix sssom: .", + "@prefix owl: .", + } + ): + raise ValueError("empty alignment profile contains mapping data") return data.decode() @@ -146,32 +245,42 @@ def build(revision, source, tooling, binary, bobbin, output): repository = "scbrown/bobbin" source, tooling, output = map(Path, (source, tooling, output)) if output.exists(): - raise ValueError('repository share output already exists') - if not re.fullmatch(r'scbrown/[a-z][a-z0-9-]*', repository): - raise ValueError('invalid repository identity') - slug = repository.split('/')[1] - if (not re.fullmatch(r'[0-9a-f]{40}', revision) - or command(['git', '-C', source, 'rev-parse', 'HEAD']).strip() != revision - or command(['git', '-C', source, 'status', '--porcelain']).strip()): - raise ValueError('repository source is not the exact clean release') - paths = command(['git', '-C', source, 'ls-files', '-z']).split('\0') - control = next((p for p in ('README.md', 'README.rst') if p in paths), None) + raise ValueError("repository share output already exists") + if not re.fullmatch(r"scbrown/[a-z][a-z0-9-]*", repository): + raise ValueError("invalid repository identity") + slug = repository.split("/")[1] + if ( + not re.fullmatch(r"[0-9a-f]{40}", revision) + or command(["git", "-C", source, "rev-parse", "HEAD"]).strip() != revision + or command(["git", "-C", source, "status", "--porcelain"]).strip() + ): + raise ValueError("repository source is not the exact clean release") + paths = command(["git", "-C", source, "ls-files", "-z"]).split("\0") + control = next((p for p in ("README.md", "README.rst") if p in paths), None) if control is None: - raise ValueError('release lacks a supported public corpus positive control') - private = output.parent / 'build-private' + raise ValueError("release lacks a supported public corpus positive control") + private = output.parent / "build-private" private.mkdir(mode=0o700) + def run(args, *, env=None): return command(args, env=env, cwd=private) - policy = private / 'policy.ttl' - run(['python3', tooling / 'scripts/private-share-policy.py', policy]) - corpus = materialize_release(source, revision, private / 'source') - (corpus / '.bobbin').mkdir(exist_ok=True) - index_database = private / 'indexed-graph.db' + + policy = private / "policy.ttl" + run( + ["python3", tooling / "scripts/private-share-policy.py", policy], + env=dict(os.environ), + ) + corpus = materialize_release(source, revision, private / "source") + (corpus / ".bobbin").mkdir(exist_ok=True) + index_database = private / "indexed-graph.db" # Bobbin resolves embedded graph state at --source, not at the vector # index root. Explicit private configuration avoids global-store fallback. - (corpus / '.bobbin/config.toml').write_text('[quipu]\nstore_path = ' + json.dumps(str(index_database)) + '\n') - index = private / 'index'; (index / '.bobbin').mkdir(parents=True) - (index / '.bobbin/config.toml').write_text('''quipu_push_chunks = true + (corpus / ".bobbin/config.toml").write_text( + "[quipu]\nstore_path = " + json.dumps(str(index_database)) + "\n" + ) + index = private / "index" + (index / ".bobbin").mkdir(parents=True) + (index / ".bobbin/config.toml").write_text("""quipu_push_chunks = true [index] include = ["**/*.rs", "**/*.md", "**/*.rst", "**/*.toml", "**/*.yml", "**/*.yaml", "**/*.py", "**/*.sh", "**/*.js", "**/*.mjs", "**/*.ts", "**/*.tsx", "**/*.json", "**/*.ttl", "**/*.go"] @@ -182,27 +291,67 @@ def run(args, *, env=None): [embedding] model = "all-MiniLM-L6-v2" batch_size = 64 -''') - env = {k: v for k, v in os.environ.items() - if k not in ('BOBBIN_SERVER', 'BOBBIN_QUIPU_REMOTE')} - run([bobbin, 'index', index, '--source', corpus, '--repo', slug, - '--force', '--skip-calibrate', '--json'], env=env) - database = private / 'producer.db' +""") + env = isolated_environment(private) + run( + [ + bobbin, + "index", + index, + "--source", + corpus, + "--repo", + slug, + "--force", + "--skip-calibrate", + "--json", + ], + env=env, + ) + database = private / "producer.db" if not index_database.is_file(): - raise ValueError('indexer did not produce the repository graph') - with sqlite3.connect('file:' + str(index_database) + '?mode=ro', uri=True) as original: - with sqlite3.connect(database) as target: - original.backup(target) - shapes = sorted((tooling / 'shapes').glob('*.ttl')) + raise ValueError("indexer did not produce the repository graph") + with sqlite3.connect( + "file:" + str(index_database) + "?mode=ro", uri=True + ) as original, sqlite3.connect(database) as target: + original.backup(target) + shapes = sorted((tooling / "shapes").glob("*.ttl")) if not shapes: - raise ValueError('verified engine source has no receiver shapes') + raise ValueError("verified engine source has no receiver shapes") for shape in shapes: - run([binary, 'shapes', 'load', shape.stem, shape, '--db', database]) - run([binary, 'shapes', 'load', 'publication-policy', - tooling / 'docs/knowledge/publication-policy.shapes.ttl', '--db', database]) - run([binary, 'knot', tooling / 'docs/knowledge/publication-policy.ttl', '--db', database]) - run([binary, 'knot', policy, '--graph', 'urn:quipu:private-release-policy', '--db', database]) - context = private / 'context.ttl' + run([binary, "shapes", "load", shape.stem, shape, "--db", database]) + run( + [ + binary, + "shapes", + "load", + "publication-policy", + tooling / "docs/knowledge/publication-policy.shapes.ttl", + "--db", + database, + ] + ) + run( + [ + binary, + "knot", + tooling / "docs/knowledge/publication-policy.ttl", + "--db", + database, + ] + ) + run( + [ + binary, + "knot", + policy, + "--graph", + "urn:quipu:private-release-policy", + "--db", + database, + ] + ) + context = private / "context.ttl" context.write_text(f'''@prefix aegis: . @prefix rdfs: . aegis:repo_{slug} a aegis:GitRepo ; rdfs:label "repo_{slug}" ; @@ -211,56 +360,129 @@ def run(args, *, env=None): rdfs:label "Repository-embedded knowledge databases" ; aegis:issuedBy "Stiwi" ; aegis:applies_to aegis:repo_{slug} . ''') - run([binary, 'knot', context, '--db', database]) + run([binary, "knot", context, "--db", database]) # Curated repository content and validating shapes are repo-owned. Refuse # an incomplete source proposal rather than minting a pointer-only stub. - registry_path = source / 'docs/knowledge/repository-knowledge.json' - knowledge = private / 'repository-knowledge.ttl' - constraints = source / 'docs/knowledge/repository-knowledge.shapes.ttl' - queries = source / 'docs/knowledge/stored-queries.json' - alignments = source / 'docs/knowledge/alignments.ttl' + registry_path = source / "docs/knowledge/repository-knowledge.json" + knowledge = private / "repository-knowledge.ttl" + constraints = source / "docs/knowledge/repository-knowledge.shapes.ttl" + queries = source / "docs/knowledge/stored-queries.json" + alignments = source / "docs/knowledge/alignments.ttl" for required in (registry_path, constraints, queries, alignments): if not required.is_file() or not required.stat().st_size: - raise ValueError('required repository knowledge member missing: ' + required.name) + raise ValueError( + "required repository knowledge member missing: " + required.name + ) import importlib.util - spec = importlib.util.spec_from_file_location('repository_knowledge', source / 'scripts/build-repository-knowledge.py') - generator = importlib.util.module_from_spec(spec); spec.loader.exec_module(generator) - records = json.loads(registry_path.read_text())['records'] - labels = indexed_labels(binary, database, sorted({record['subject'] for record in records})) - knowledge.write_text(generator.render(source, revision, lambda subject: labels.get(subject)) + '\n' + alignment_carrier(source)) - run([binary, 'validate', '--shapes', constraints, '--data', knowledge]) - run([binary, 'shapes', 'load', 'repository-knowledge', constraints, '--db', database]) - run([binary, 'knot', knowledge, '--db', database]) + + spec = importlib.util.spec_from_file_location( + "repository_knowledge", source / "scripts/build-repository-knowledge.py" + ) + generator = importlib.util.module_from_spec(spec) + _previous_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + spec.loader.exec_module(generator) + finally: + sys.dont_write_bytecode = _previous_bytecode + records = json.loads(registry_path.read_text())["records"] + labels = indexed_labels( + binary, database, sorted({record["subject"] for record in records}) + ) + knowledge.write_text( + generator.render(source, revision, lambda subject: labels.get(subject)) + + "\n" + + alignment_carrier(source) + ) + run([binary, "validate", "--shapes", constraints, "--data", knowledge]) + run( + [ + binary, + "shapes", + "load", + "repository-knowledge", + constraints, + "--db", + database, + ] + ) + run([binary, "knot", knowledge, "--db", database]) register_queries(binary, database, json.loads(queries.read_text())) - query = (source / 'queries/repository-share-bobbin.rq').read_text() - stage = private / 'share' - run([binary, 'share', '--construct', query, '--turtle', '--output', stage, '--db', database]) - if not (stage / 'shapes.ttl').is_file() or (stage / 'shapes.ttl').stat().st_size == 0: - raise ValueError('repository share omitted receiver shapes') - receiver = private / 'receiver.db' - run([binary, 'shapes', 'load', 'repository-share', stage / 'shapes.ttl', '--db', receiver]) - imported = json.loads(run([binary, 'import', stage, '--db', receiver])) - if (imported.get('outcome') != 'staged' - or imported.get('promotion', {}).get('eligible') is not True - or type(imported.get('triples', {}).get('accepted')) is not int - or imported['triples']['accepted'] <= 3): - raise ValueError('repository receiver did not admit a non-stub corpus') - run([binary, 'import', 'promote', imported['share_id'], '--db', receiver]) - positive = run([binary, 'query', - 'SELECT ?s WHERE { ?s ' - + json.dumps(control) + ' } LIMIT 1', '--db', receiver]) - if re.search(r'\n1 results\s*$', positive) is None: - raise ValueError('repository corpus positive control missing after import') - run(['python3', tooling / 'scripts/verify-repository-share-receiver.py', - binary, stage, receiver, private]) - if run(['git', '-C', source, 'status', '--porcelain']).strip(): - raise ValueError('indexing changed the exact repository source') + query = (source / "queries/repository-share-bobbin.rq").read_text() + stage = private / "share" + run( + [ + binary, + "share", + "--construct", + query, + "--turtle", + "--output", + stage, + "--db", + database, + ] + ) + if ( + not (stage / "shapes.ttl").is_file() + or (stage / "shapes.ttl").stat().st_size == 0 + ): + raise ValueError("repository share omitted receiver shapes") + receiver = private / "receiver.db" + run( + [ + binary, + "shapes", + "load", + "repository-share", + stage / "shapes.ttl", + "--db", + receiver, + ] + ) + imported = json.loads(run([binary, "import", stage, "--db", receiver])) + if ( + imported.get("outcome") != "staged" + or imported.get("promotion", {}).get("eligible") is not True + or type(imported.get("triples", {}).get("accepted")) is not int + or imported["triples"]["accepted"] <= 3 + ): + raise ValueError("repository receiver did not admit a non-stub corpus") + run([binary, "import", "promote", imported["share_id"], "--db", receiver]) + positive = run( + [ + binary, + "query", + "SELECT ?s WHERE { ?s " + + json.dumps(control) + + " } LIMIT 1", + "--db", + receiver, + ] + ) + if re.search(r"\n1 results\s*$", positive) is None: + raise ValueError("repository corpus positive control missing after import") + run( + [ + "python3", + tooling / "scripts/verify-repository-share-receiver.py", + binary, + stage, + receiver, + private, + ] + ) + if run(["git", "-C", source, "status", "--porcelain"]).strip(): + raise ValueError("indexing changed the exact repository source") shutil.move(stage, output) - return {'triples': imported['triples']['accepted'], 'positive_file': control} + return {"triples": imported["triples"]["accepted"], "positive_file": control} -if __name__ == '__main__': +if __name__ == "__main__": import sys + if len(sys.argv) != 7: - raise SystemExit('usage: build-repository-share.py ') + raise SystemExit( + "usage: build-repository-share.py " + ) build(*sys.argv[1:]) diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py index 8187668e..29c27d08 100644 --- a/scripts/publish_repository_share.py +++ b/scripts/publish_repository_share.py @@ -1,30 +1,41 @@ #!/usr/bin/env python3 """Bobbin's owned trusted release job; hosted workflows only verify its output.""" + import gzip import hashlib import importlib.util import json import os -from pathlib import Path import re import shutil import subprocess import sys import tarfile import tempfile +from pathlib import Path -REPOSITORY = 'scbrown/bobbin' -MEMBERS = {'export.nt', 'export.ttl', 'payload.nq', 'manifest.json', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'} +REPOSITORY = "scbrown/bobbin" +MEMBERS = { + "export.nt", + "export.ttl", + "payload.nq", + "manifest.json", + "manifest.ttl", + "shapes.ttl", + "queries.ttl", +} def command(args): - return subprocess.run([str(v) for v in args], check=True, capture_output=True, text=True).stdout + return subprocess.run( + [str(v) for v in args], check=True, capture_output=True, text=True + ).stdout def digest(path): result = hashlib.sha256() - with path.open('rb') as handle: - for block in iter(lambda: handle.read(1 << 20), b''): + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): result.update(block) return result.hexdigest() @@ -32,43 +43,58 @@ def digest(path): def check_share(stage): files = list(stage.iterdir()) if any(not p.is_file() or p.is_symlink() or p.name not in MEMBERS for p in files): - raise ValueError('unexpected or non-regular share member') - for name in ('export.nt', 'manifest.json', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'): + raise ValueError("unexpected or non-regular share member") + for name in ( + "export.nt", + "manifest.json", + "manifest.ttl", + "shapes.ttl", + "queries.ttl", + ): if not (stage / name).is_file() or not (stage / name).stat().st_size: - raise ValueError('required nonempty share member missing: ' + name) - recipe = json.loads((stage / 'manifest.json').read_text()).get('embedding') - if (not isinstance(recipe, dict) or not isinstance(recipe.get('model'), str) - or not recipe['model'] or Path(recipe['model']).name != recipe['model'] - or type(recipe.get('dimension')) is not int or recipe['dimension'] <= 0 - or any(not re.fullmatch('[0-9a-f]{64}', recipe.get(k, '')) - for k in ('model_sha256', 'tokenizer_sha256'))): - raise ValueError('share omitted a provider-byte embedding recipe') + raise ValueError("required nonempty share member missing: " + name) + recipe = json.loads((stage / "manifest.json").read_text()).get("embedding") + if ( + not isinstance(recipe, dict) + or not isinstance(recipe.get("model"), str) + or not recipe["model"] + or Path(recipe["model"]).name != recipe["model"] + or type(recipe.get("dimension")) is not int + or recipe["dimension"] <= 0 + or any( + not re.fullmatch("[0-9a-f]{64}", recipe.get(k, "")) + for k in ("model_sha256", "tokenizer_sha256") + ) + ): + raise ValueError("share omitted a provider-byte embedding recipe") # This repository pack is pointer-only. Curated idea summaries use # rdfs:comment; source bodies and line spans are never release predicates. - export = (stage / 'export.nt').read_text() - for predicate in ('content', 'startLine', 'endLine'): - if '' in export: - raise ValueError('share duplicates source text or line-span data') + export = (stage / "export.nt").read_text() + for predicate in ("content", "startLine", "endLine"): + if "" in export: + raise ValueError("share duplicates source text or line-span data") return recipe def package(stage, archive): check_share(stage) - with archive.open('xb') as raw: - with gzip.GzipFile(filename='', fileobj=raw, mode='wb', mtime=0) as compressed: - with tarfile.open(fileobj=compressed, mode='w') as tar: + with archive.open("xb") as raw: + with gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w") as tar: for path in sorted(stage.iterdir()): info = tar.gettarinfo(str(path), arcname=path.name) info.uid = info.gid = 0 - info.uname = info.gname = '' + info.uname = info.gname = "" info.mtime = 0 info.mode = 0o600 - with path.open('rb') as handle: + with path.open("rb") as handle: tar.addfile(info, handle) def held(): - path = Path(os.environ.get("QUIPU_SHARE_HOLD_FILE", str(Path.home() / ".quipu-share-hold"))) + path = Path( + os.environ.get("QUIPU_SHARE_HOLD_FILE", str(Path.home() / ".quipu-share-hold")) + ) try: path.lstat() except FileNotFoundError: @@ -80,77 +106,138 @@ def held(): def publish(tag, revision, output): if held(): - raise ValueError('publication held before any lookup/build') - if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag): - raise ValueError('unexpected Bobbin release tag') - if not re.fullmatch('[0-9a-f]{40}', revision): - raise ValueError('invalid release revision') + raise ValueError("publication held before any lookup/build") + if not re.fullmatch(r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag): + raise ValueError("unexpected Bobbin release tag") + if not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release revision") source = Path(__file__).resolve().parents[1] - if (command(['git', '-C', source, 'rev-parse', 'HEAD']).strip() != revision - or command(['git', '-C', source, 'rev-parse', 'refs/tags/' + tag + '^{commit}']).strip() != revision - or command(['git', '-C', source, 'status', '--porcelain']).strip()): - raise ValueError('publisher source is not the exact clean release') + if ( + command(["git", "-C", source, "rev-parse", "HEAD"]).strip() != revision + or command( + ["git", "-C", source, "rev-parse", "refs/tags/" + tag + "^{commit}"] + ).strip() + != revision + or command(["git", "-C", source, "status", "--porcelain"]).strip() + ): + raise ValueError("publisher source is not the exact clean release") output = Path(output) if output.exists(): - raise ValueError('retained output exists; reconcile before retrying') - binary = Path(os.environ['QUIPU_SHARE_ENGINE_BIN']).resolve(strict=True) - tooling = Path(os.environ['QUIPU_SHARE_ENGINE_SOURCE']).resolve(strict=True) - bobbin = Path(os.environ['QUIPU_SHARE_BOBBIN']).resolve(strict=True) - engine_revision = os.environ['QUIPU_SHARE_ENGINE_REVISION'] - if (not re.fullmatch('[0-9a-f]{40}', engine_revision) - or command(['git', '-C', tooling, 'rev-parse', 'HEAD']).strip() != engine_revision - or command(['git', '-C', tooling, 'status', '--porcelain']).strip() - or digest(binary) != os.environ['QUIPU_SHARE_ENGINE_BIN_SHA256'] - or digest(binary.with_name('quipu-server')) != os.environ['QUIPU_SHARE_ENGINE_SERVER_SHA256'] - or digest(bobbin) != os.environ['QUIPU_SHARE_BOBBIN_SHA256']): - raise ValueError('trusted engine/source/indexer pin mismatch') + raise ValueError("retained output exists; reconcile before retrying") + binary = Path(os.environ["QUIPU_SHARE_ENGINE_BIN"]).resolve(strict=True) + tooling = Path(os.environ["QUIPU_SHARE_ENGINE_SOURCE"]).resolve(strict=True) + bobbin = Path(os.environ["QUIPU_SHARE_BOBBIN"]).resolve(strict=True) + engine_revision = os.environ["QUIPU_SHARE_ENGINE_REVISION"] + if ( + not re.fullmatch("[0-9a-f]{40}", engine_revision) + or command(["git", "-C", tooling, "rev-parse", "HEAD"]).strip() + != engine_revision + or command(["git", "-C", tooling, "status", "--porcelain"]).strip() + or digest(binary) != os.environ["QUIPU_SHARE_ENGINE_BIN_SHA256"] + or digest(binary.with_name("quipu-server")) + != os.environ["QUIPU_SHARE_ENGINE_SERVER_SHA256"] + or digest(bobbin) != os.environ["QUIPU_SHARE_BOBBIN_SHA256"] + ): + raise ValueError("trusted engine/source/indexer pin mismatch") output.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - with tempfile.TemporaryDirectory(prefix='prepare-', dir=output.parent) as temp: + with tempfile.TemporaryDirectory(prefix="prepare-", dir=output.parent) as temp: private = Path(temp) - stage = private / 'share' - spec = importlib.util.spec_from_file_location('bobbin_share_build', source / 'scripts/build-repository-share.py') - builder = importlib.util.module_from_spec(spec); spec.loader.exec_module(builder) + stage = private / "share" + spec = importlib.util.spec_from_file_location( + "bobbin_share_build", source / "scripts/build-repository-share.py" + ) + builder = importlib.util.module_from_spec(spec) + _previous_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + spec.loader.exec_module(builder) + finally: + sys.dont_write_bytecode = _previous_bytecode builder.build(revision, source, tooling, binary, bobbin, stage) - archive_name = 'bobbin-' + tag + '-repository.pendant.tar.gz' - prepared = private / 'assets'; prepared.mkdir(mode=0o700) + archive_name = "bobbin-" + tag + "-repository.pendant.tar.gz" + prepared = private / "assets" + prepared.mkdir(mode=0o700) archive = prepared / archive_name package(stage, archive) - (prepared / (archive_name + '.sha256')).write_text(digest(archive) + ' ' + archive_name + '\n') - proof = {'schema': 'quipu.repository-share-producer/v1', - 'source_repository': REPOSITORY, 'source_revision': revision, 'release_tag': tag, - 'archive_sha256': digest(archive), 'quipu_sha256': digest(binary), - 'engine_source_revision': engine_revision, 'bobbin_sha256': digest(bobbin), - 'quipu_server_sha256': digest(binary.with_name('quipu-server')), - 'destination': 'outward'} - (prepared / (archive_name + '.provenance.json')).write_text(json.dumps(proof, sort_keys=True, indent=2) + '\n') + (prepared / (archive_name + ".sha256")).write_text( + digest(archive) + " " + archive_name + "\n" + ) + proof = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": REPOSITORY, + "source_revision": revision, + "release_tag": tag, + "archive_sha256": digest(archive), + "quipu_sha256": digest(binary), + "engine_source_revision": engine_revision, + "bobbin_sha256": digest(bobbin), + "quipu_server_sha256": digest(binary.with_name("quipu-server")), + "destination": "outward", + } + (prepared / (archive_name + ".provenance.json")).write_text( + json.dumps(proof, sort_keys=True, indent=2) + "\n" + ) names = sorted(p.name for p in prepared.iterdir()) if held(): - raise ValueError('publication held after preparation') - available = json.loads(command(['gh', 'release', 'view', tag, '--repo', REPOSITORY, '--json', 'assets'])) - present = {a['name'] for a in available['assets']} & set(names) + raise ValueError("publication held after preparation") + available = json.loads( + command( + ["gh", "release", "view", tag, "--repo", REPOSITORY, "--json", "assets"] + ) + ) + present = {a["name"] for a in available["assets"]} & set(names) if present and present != set(names): - raise ValueError('partial publication; explicit reconciliation required') + raise ValueError("partial publication; explicit reconciliation required") # Retain before the one-way upload. No --clobber and no automatic retry # from an uncertain write. The managed dispatcher also retains STARTED. shutil.copytree(prepared, output) if not present: if held(): - raise ValueError('publication held immediately before upload') - command(['gh', 'release', 'upload', tag, '--repo', REPOSITORY, - *(output / name for name in names)]) - downloaded = private / 'downloaded'; downloaded.mkdir() - command(['gh', 'release', 'download', tag, '--repo', REPOSITORY, '--dir', downloaded, - *(arg for name in names for arg in ('--pattern', name))]) + raise ValueError("publication held immediately before upload") + command( + [ + "gh", + "release", + "upload", + tag, + "--repo", + REPOSITORY, + *(output / name for name in names), + ] + ) + downloaded = private / "downloaded" + downloaded.mkdir() + command( + [ + "gh", + "release", + "download", + tag, + "--repo", + REPOSITORY, + "--dir", + downloaded, + *(arg for name in names for arg in ("--pattern", name)), + ] + ) for name in names: if (downloaded / name).read_bytes() != (output / name).read_bytes(): - raise ValueError('existing or newly published asset differs; never overwrite') - receipt = {**proof, 'published_and_read_back': True, 'assets': names, - 'asset_sha256': {name: digest(downloaded / name) for name in names}} - (output / 'delivery.json').write_text(json.dumps(receipt, indent=2) + '\n') + raise ValueError( + "existing or newly published asset differs; never overwrite" + ) + receipt = { + **proof, + "published_and_read_back": True, + "assets": names, + "asset_sha256": {name: digest(downloaded / name) for name in names}, + } + (output / "delivery.json").write_text(json.dumps(receipt, indent=2) + "\n") return receipt -if __name__ == '__main__': - if len(sys.argv) != 5 or sys.argv[1] != '--trusted-release': - raise SystemExit('usage: publish-repository-share.sh --trusted-release ') +if __name__ == "__main__": + if len(sys.argv) != 5 or sys.argv[1] != "--trusted-release": + raise SystemExit( + "usage: publish-repository-share.sh --trusted-release " + ) print(json.dumps(publish(sys.argv[2], sys.argv[3], sys.argv[4]))) diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index 0841ad7e..e7e69639 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -3,137 +3,250 @@ # arming: ci; release contract invokes this script. """ + import importlib.util import json import os -from pathlib import Path import shutil import subprocess +import sys import tempfile import unittest +from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location('knowledge', ROOT / 'scripts/build-repository-knowledge.py') -module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) +spec = importlib.util.spec_from_file_location( + "knowledge", ROOT / "scripts/build-repository-knowledge.py" +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) -builder_spec = importlib.util.spec_from_file_location('share_builder', ROOT / 'scripts/build-repository-share.py') -builder = importlib.util.module_from_spec(builder_spec); builder_spec.loader.exec_module(builder) +builder_spec = importlib.util.spec_from_file_location( + "share_builder", ROOT / "scripts/build-repository-share.py" +) +builder = importlib.util.module_from_spec(builder_spec) +builder_spec.loader.exec_module(builder) class ReleaseSnapshot(unittest.TestCase): def test_real_git_archive_excludes_untracked_key_and_preserves_source(self): with tempfile.TemporaryDirectory() as folder: - root = Path(folder); source = root / 'source'; source.mkdir() - subprocess.run(['git', 'init', '-q', str(source)], check=True) - (source / 'README.md').write_text('tracked pointer source\n') - subprocess.run(['git', '-C', str(source), 'add', 'README.md'], check=True) - subprocess.run(['git', '-C', str(source), '-c', 'user.name=fixture', - '-c', 'user.email=fixture@example.org', 'commit', '-qm', 'fixture'], check=True) - (source / 'private.key').write_text('synthetic untracked sentinel') - revision = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() - copied = builder.materialize_release(source, revision, root / 'copied') - self.assertEqual((copied / 'README.md').read_bytes(), (source / 'README.md').read_bytes()) - self.assertFalse((copied / 'private.key').exists()) - self.assertEqual((source / 'private.key').read_text(), 'synthetic untracked sentinel') - self.assertFalse((source / '.bobbin').exists()) + root = Path(folder) + source = root / "source" + source.mkdir() + subprocess.run(["git", "init", "-q", str(source)], check=True) + (source / "README.md").write_text("tracked pointer source\n") + subprocess.run(["git", "-C", str(source), "add", "README.md"], check=True) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=fixture", + "-c", + "user.email=fixture@example.org", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + (source / "private.key").write_text("synthetic untracked sentinel") + revision = subprocess.check_output( + ["git", "-C", str(source), "rev-parse", "HEAD"], text=True + ).strip() + copied = builder.materialize_release(source, revision, root / "copied") + self.assertEqual( + (copied / "README.md").read_bytes(), (source / "README.md").read_bytes() + ) + self.assertFalse((copied / "private.key").exists()) + self.assertEqual( + (source / "private.key").read_text(), "synthetic untracked sentinel" + ) + self.assertFalse((source / ".bobbin").exists()) class AlignmentCarrier(unittest.TestCase): def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): with tempfile.TemporaryDirectory() as folder: - root = Path(folder); target = root / 'docs/knowledge'; target.mkdir(parents=True) - for name in ('alignments.ttl', 'alignment-status.json'): - shutil.copyfile(ROOT / 'docs/knowledge' / name, target / name) - self.assertIn('@prefix sssom:', builder.alignment_carrier(root)) - data = target / 'alignments.ttl'; original = data.read_text() + root = Path(folder) + target = root / "docs/knowledge" + target.mkdir(parents=True) + for name in ("alignments.ttl", "alignment-status.json"): + shutil.copyfile(ROOT / "docs/knowledge" / name, target / name) + self.assertIn("@prefix sssom:", builder.alignment_carrier(root)) + data = target / "alignments.ttl" + original = data.read_text() data.unlink() - with self.assertRaises(FileNotFoundError): builder.alignment_carrier(root) - data.write_text(original + ' .\n') - with self.assertRaisesRegex(ValueError, 'corrupt'): builder.alignment_carrier(root) + with self.assertRaises(FileNotFoundError): + builder.alignment_carrier(root) + data.write_text(original + " .\n") + with self.assertRaisesRegex(ValueError, "corrupt"): + builder.alignment_carrier(root) import hashlib - status = json.loads((target / 'alignment-status.json').read_text()) - status['carrier_sha256'] = hashlib.sha256(data.read_bytes()).hexdigest() - (target / 'alignment-status.json').write_text(json.dumps(status)) - with self.assertRaisesRegex(ValueError, 'mapping data'): builder.alignment_carrier(root) - data.write_text('@prefix owl: . .\n') - status['carrier_sha256'] = hashlib.sha256(data.read_bytes()).hexdigest() - (target / 'alignment-status.json').write_text(json.dumps(status)) - with self.assertRaisesRegex(ValueError, 'mapping data'): builder.alignment_carrier(root) + + status = json.loads((target / "alignment-status.json").read_text()) + status["carrier_sha256"] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / "alignment-status.json").write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, "mapping data"): + builder.alignment_carrier(root) + data.write_text( + "@prefix owl: . .\n" + ) + status["carrier_sha256"] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / "alignment-status.json").write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, "mapping data"): + builder.alignment_carrier(root) + + +class NativeIsolation(unittest.TestCase): + def test_real_subprocess_cannot_see_synthetic_global_config_or_endpoints(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + host, private = root / "host", root / "private" + private.mkdir() + config = host / ".config/bobbin" + config.mkdir(parents=True) + (config / "config.toml").write_text('[server]\nurl="http://127.0.0.1:9"\n') + self.assertTrue((config / "config.toml").exists()) + code = ( + "import os,json;from pathlib import Path;" + 'visible=(Path(os.environ["HOME"])/".config/bobbin/config.toml").exists();' + "remote=any(k in os.environ for k in " + '("BOBBIN_SERVER","BOBBIN_QUIPU_REMOTE","QUIPU_SERVER","QUIPU_AUTH_TOKEN","GH_TOKEN"));' + 'print(json.dumps({"global_config_visible":visible,"remote_or_upload_env_visible":remote}));' + "raise SystemExit(3 if visible or remote else 0)" + ) + environment = { + "HOME": str(host), + "XDG_CONFIG_HOME": str(host / ".config"), + "BOBBIN_SERVER": "http://127.0.0.1:9", + "BOBBIN_QUIPU_REMOTE": "http://127.0.0.1:9", + "QUIPU_SERVER": "http://127.0.0.1:9", + "QUIPU_AUTH_TOKEN": "synthetic-only", + "GH_TOKEN": "synthetic-only", + } + with patch.dict(os.environ, environment): + output = builder.command([sys.executable, "-c", code], cwd=private) + policy_code = code.replace( + "3 if visible or remote else 0", "0 if visible and remote else 3" + ) + policy = builder.command( + [sys.executable, "-c", policy_code], + env=dict(os.environ), + cwd=private, + ) + self.assertIn('"global_config_visible": false', output) + self.assertIn('"remote_or_upload_env_visible": false', output) + self.assertIn('"global_config_visible": true', policy) + self.assertIn('"remote_or_upload_env_visible": true', policy) class Knowledge(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - self.registry = json.loads((ROOT / 'docs/knowledge/repository-knowledge.json').read_text()) - for source in {r['source'] for r in self.registry['records']}: - target = self.root / source; target.parent.mkdir(parents=True, exist_ok=True) + self.registry = json.loads( + (ROOT / "docs/knowledge/repository-knowledge.json").read_text() + ) + for source in {r["source"] for r in self.registry["records"]}: + target = self.root / source + target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(ROOT / source, target) - (self.root / 'docs/knowledge').mkdir(parents=True) + (self.root / "docs/knowledge").mkdir(parents=True) self.save() def save(self): - (self.root / 'docs/knowledge/repository-knowledge.json').write_text(json.dumps(self.registry)) + (self.root / "docs/knowledge/repository-knowledge.json").write_text( + json.dumps(self.registry) + ) - def render(self, lookup=lambda _: 'actual-index-label-fixture'): - return module.render(self.root, '1' * 40, lookup) + def render(self, lookup=lambda _: "actual-index-label-fixture"): + return module.render(self.root, "1" * 40, lookup) def test_source_bound_nonempty_content(self): text = self.render() - self.assertIn('2592000', text) - self.assertIn('reviewAfter', text) + self.assertIn("2592000", text) + self.assertIn("reviewAfter", text) self.assertIn('knowledgeKind> "idea"', text) self.assertIn('interfaceKind> "REST"', text) self.assertIn('supersessionStatus> "not-recorded-in-source"', text) self.assertIn('startHereOrder> "3"^^', text) - self.assertNotIn('/content>', text) - self.assertNotIn('owl#sameAs', text) + self.assertNotIn("/content>", text) + self.assertNotIn("owl#sameAs", text) def test_absent_or_ambiguous_index_pointer_refuses(self): - for value in (None, False, ''): - with self.subTest(value=value), self.assertRaisesRegex(ValueError, 'absent or ambiguous'): + for value in (None, False, ""): + with ( + self.subTest(value=value), + self.assertRaisesRegex(ValueError, "absent or ambiguous"), + ): self.render(lambda _: value) def test_changed_source_refuses(self): - path = self.root / self.registry['records'][0]['source'] - path.write_text(path.read_text() + '\nchanged\n') - with self.assertRaisesRegex(ValueError, 'witness changed'): + path = self.root / self.registry["records"][0]["source"] + path.write_text(path.read_text() + "\nchanged\n") + with self.assertRaisesRegex(ValueError, "witness changed"): self.render() def test_guessed_identity_refuses(self): - self.registry['records'][0]['subject'] = 'https://example.org/guessed' + self.registry["records"][0]["subject"] = "https://example.org/guessed" self.save() - with self.assertRaisesRegex(ValueError, 'source-backed'): + with self.assertRaisesRegex(ValueError, "source-backed"): self.render() def test_missing_required_content_refuses(self): - self.registry['records'] = [r for r in self.registry['records'] if r['kind'] != 'decision'] + self.registry["records"] = [ + r for r in self.registry["records"] if r["kind"] != "decision" + ] self.save() - with self.assertRaisesRegex(ValueError, 'inventory incomplete'): + with self.assertRaisesRegex(ValueError, "inventory incomplete"): self.render() def test_bad_freshness_and_private_summary_refuse(self): - self.registry['max_age_seconds'] = 0; self.save() - with self.assertRaisesRegex(ValueError, 'freshness'): + self.registry["max_age_seconds"] = 0 + self.save() + with self.assertRaisesRegex(ValueError, "freshness"): self.render() - self.registry['max_age_seconds'] = 30 - self.registry['records'][0]['summary'] = 'read /home/private/file' + self.registry["max_age_seconds"] = 30 + self.registry["records"][0]["summary"] = "read /home/private/file" self.save() - with self.assertRaisesRegex(ValueError, 'private summary'): + with self.assertRaisesRegex(ValueError, "private summary"): self.render() - @unittest.skipUnless(os.environ.get('QUIPU_TEST_BIN'), 'native schema control requires explicit test artifact') + @unittest.skipUnless( + os.environ.get("QUIPU_TEST_BIN"), + "native schema control requires explicit test artifact", + ) def test_actual_native_schema_positive_and_missing_freshness_negative(self): - data = self.root / 'curated.ttl'; data.write_text(self.render()) - args = [os.environ['QUIPU_TEST_BIN'], 'validate', '--shapes', - str(ROOT / 'docs/knowledge/repository-knowledge.shapes.ttl'), '--data', str(data), '--format', 'json'] + data = self.root / "curated.ttl" + data.write_text(self.render()) + args = [ + os.environ["QUIPU_TEST_BIN"], + "validate", + "--shapes", + str(ROOT / "docs/knowledge/repository-knowledge.shapes.ttl"), + "--data", + str(data), + "--format", + "json", + ] positive = subprocess.run(args, capture_output=True, text=True, cwd=self.root) self.assertEqual(positive.returncode, 0, positive.stderr + positive.stdout) - data.write_text('\n'.join(line for line in data.read_text().splitlines() if '/reviewAfter>' not in line)) + data.write_text( + "\n".join( + line + for line in data.read_text().splitlines() + if "/reviewAfter>" not in line + ) + ) negative = subprocess.run(args, capture_output=True, text=True, cwd=self.root) self.assertNotEqual(negative.returncode, 0, negative.stdout) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py index 5dfb5f9d..c79dd3ac 100644 --- a/scripts/test-repository-share-publication.py +++ b/scripts/test-repository-share-publication.py @@ -3,86 +3,128 @@ # arming: ci; release asset publication contract invokes this script. """ + import importlib.util import json -from pathlib import Path +import os +import shutil +import subprocess +import sys import tempfile import unittest +from pathlib import Path from unittest.mock import patch ROOT = Path(__file__).resolve().parent + def load(name, path): spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) return module -publisher = load('publisher_test', ROOT / 'publish_repository_share.py') -verifier = load('verifier_test', ROOT / 'verify-repository-share.py') + +publisher = load("publisher_test", ROOT / "publish_repository_share.py") +verifier = load("verifier_test", ROOT / "verify-repository-share.py") class Publication(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - self.stage = self.root / 'stage'; self.stage.mkdir() - recipe = {'model': 'model.onnx', 'model_sha256': 'a' * 64, - 'tokenizer_sha256': 'b' * 64, 'dimension': 384} - (self.stage / 'manifest.json').write_text(json.dumps({'embedding': recipe})) - for name in ('export.nt', 'manifest.ttl', 'shapes.ttl', 'queries.ttl'): - (self.stage / name).write_text('# synthetic fixture only\n') - self.archive = self.root / 'bobbin-v1.2.3-repository.pendant.tar.gz' - self.revision = '1' * 40 + self.stage = self.root / "stage" + self.stage.mkdir() + recipe = { + "model": "model.onnx", + "model_sha256": "a" * 64, + "tokenizer_sha256": "b" * 64, + "dimension": 384, + } + (self.stage / "manifest.json").write_text(json.dumps({"embedding": recipe})) + for name in ("export.nt", "manifest.ttl", "shapes.ttl", "queries.ttl"): + (self.stage / name).write_text("# synthetic fixture only\n") + self.archive = self.root / "bobbin-v1.2.3-repository.pendant.tar.gz" + self.revision = "1" * 40 def prepare(self): publisher.package(self.stage, self.archive) sha = publisher.digest(self.archive) - self.archive.with_name(self.archive.name + '.sha256').write_text(sha + ' ' + self.archive.name + '\n') - self.proof = {'schema': 'quipu.repository-share-producer/v1', 'source_repository': 'scbrown/bobbin', - 'source_revision': self.revision, 'release_tag': 'v1.2.3', 'archive_sha256': sha, - 'destination': 'outward', 'quipu_sha256': '2' * 64, 'quipu_server_sha256': '5' * 64, 'bobbin_sha256': '3' * 64, - 'engine_source_revision': '4' * 40} - self.proof_path = self.archive.with_name(self.archive.name + '.provenance.json') + self.archive.with_name(self.archive.name + ".sha256").write_text( + sha + " " + self.archive.name + "\n" + ) + self.proof = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": "scbrown/bobbin", + "source_revision": self.revision, + "release_tag": "v1.2.3", + "archive_sha256": sha, + "destination": "outward", + "quipu_sha256": "2" * 64, + "quipu_server_sha256": "5" * 64, + "bobbin_sha256": "3" * 64, + "engine_source_revision": "4" * 40, + } + self.proof_path = self.archive.with_name(self.archive.name + ".provenance.json") self.proof_path.write_text(json.dumps(self.proof)) def test_deterministic_archive_and_exact_readback(self): self.prepare() - other = self.root / 'second.tar.gz'; publisher.package(self.stage, other) + other = self.root / "second.tar.gz" + publisher.package(self.stage, other) self.assertEqual(self.archive.read_bytes(), other.read_bytes()) - self.assertEqual(verifier.verify(self.root, 'v1.2.3', self.revision)['source_revision'], self.revision) + self.assertEqual( + verifier.verify(self.root, "v1.2.3", self.revision)["source_revision"], + self.revision, + ) def test_required_members_and_recipe(self): - (self.stage / 'queries.ttl').unlink() - with self.assertRaisesRegex(ValueError, 'required nonempty'): + (self.stage / "queries.ttl").unlink() + with self.assertRaisesRegex(ValueError, "required nonempty"): publisher.check_share(self.stage) - (self.stage / 'queries.ttl').write_text('# query') - (self.stage / 'manifest.json').write_text('{}') - with self.assertRaisesRegex(ValueError, 'provider-byte'): + (self.stage / "queries.ttl").write_text("# query") + (self.stage / "manifest.json").write_text("{}") + with self.assertRaisesRegex(ValueError, "provider-byte"): publisher.check_share(self.stage) def test_text_line_span_and_private_members_refuse(self): - for predicate in ('content', 'startLine', 'endLine'): - (self.stage / 'export.nt').write_text(' "body" .\n') - with self.subTest(predicate=predicate), self.assertRaisesRegex(ValueError, 'duplicates source'): + for predicate in ("content", "startLine", "endLine"): + (self.stage / "export.nt").write_text( + " "body" .\n' + ) + with ( + self.subTest(predicate=predicate), + self.assertRaisesRegex(ValueError, "duplicates source"), + ): publisher.check_share(self.stage) - (self.stage / 'export.nt').write_text('# empty synthetic') - (self.stage / 'policy-token').write_text('not a real token') - with self.assertRaisesRegex(ValueError, 'unexpected'): + (self.stage / "export.nt").write_text("# empty synthetic") + (self.stage / "policy-token").write_text("not a real token") + with self.assertRaisesRegex(ValueError, "unexpected"): publisher.check_share(self.stage) def test_foreign_repository_and_wrong_source_refuse(self): self.prepare() - for field, value in [('source_repository', 'scbrown/quipu'), ('source_revision', 'f' * 40), - ('release_tag', 'v9.9.9'), ('archive_sha256', 'f' * 64)]: + for field, value in [ + ("source_repository", "scbrown/quipu"), + ("source_revision", "f" * 40), + ("release_tag", "v9.9.9"), + ("archive_sha256", "f" * 64), + ]: self.proof_path.write_text(json.dumps({**self.proof, field: value})) - with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'provenance mismatch'): - verifier.verify(self.root, 'v1.2.3', self.revision) + with ( + self.subTest(field=field), + self.assertRaisesRegex(ValueError, "provenance mismatch"), + ): + verifier.verify(self.root, "v1.2.3", self.revision) def test_checksum_refuses(self): self.prepare() - self.archive.with_name(self.archive.name + '.sha256').write_text('wrong') - with self.assertRaisesRegex(ValueError, 'checksum'): - verifier.verify(self.root, 'v1.2.3', self.revision) + self.archive.with_name(self.archive.name + ".sha256").write_text("wrong") + with self.assertRaisesRegex(ValueError, "checksum"): + verifier.verify(self.root, "v1.2.3", self.revision) def test_hold_absent_and_dangling_symlink_before_any_build(self): hold = self.root / "hold" @@ -91,25 +133,83 @@ def test_hold_absent_and_dangling_symlink_before_any_build(self): hold.symlink_to(self.root / "missing-target") self.assertTrue(publisher.held()) with self.assertRaisesRegex(ValueError, "held before"): - publisher.publish("invalid-tag", "invalid-revision", self.root / "output") + publisher.publish( + "invalid-tag", "invalid-revision", self.root / "output" + ) self.assertFalse((self.root / "output").exists()) def test_unreadable_hold_fails_closed(self): - with patch.object(Path, "lstat", side_effect=PermissionError("synthetic denied")): - with self.assertRaisesRegex(ValueError, "cannot inspect publication hold"): - publisher.held() + with patch.object( + Path, "lstat", side_effect=PermissionError("synthetic denied") + ), self.assertRaisesRegex(ValueError, "cannot inspect publication hold"): + publisher.held() + + def test_readonly_verifier_keeps_nonpython_checkout_clean(self): + source = self.root / "clean-source" + (source / "scripts").mkdir(parents=True) + for name in ( + "verify-repository-share.py", + "publish_repository_share.py", + "build-repository-share.py", + "build-repository-knowledge.py", + ): + shutil.copyfile(ROOT / name, source / "scripts" / name) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.org", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + environment = dict(os.environ) + environment.pop("PYTHONDONTWRITEBYTECODE", None) + result = subprocess.run( + [ + sys.executable, + str(source / "scripts/verify-repository-share.py"), + "not-a-release", + "a" * 40, + ], + cwd=source, + env=environment, + capture_output=True, + text=True, + timeout=10, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid release identity", result.stderr) + status = subprocess.check_output( + ["git", "-C", str(source), "status", "--porcelain"], text=True + ) + self.assertEqual(status, "") def test_wait_is_bounded(self): calls, pauses = [], [] + class Result: returncode = 1 + def run(args, **kwargs): - calls.append(args); return Result() - with self.assertRaisesRegex(ValueError, 'bounded'): - verifier.wait('v1.2.3', self.revision, attempts=2, pause=pauses.append, run=run) + calls.append(args) + return Result() + + with self.assertRaisesRegex(ValueError, "bounded"): + verifier.wait( + "v1.2.3", self.revision, attempts=2, pause=pauses.append, run=run + ) self.assertEqual(len(calls), 2) self.assertEqual(pauses, [30]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/scripts/verify-repository-share.py b/scripts/verify-repository-share.py index 7cb71999..353b02e2 100644 --- a/scripts/verify-repository-share.py +++ b/scripts/verify-repository-share.py @@ -1,44 +1,65 @@ #!/usr/bin/env python3 """Bounded hosted readback; build policy and credentials stay on trusted host.""" + import hashlib import importlib.util import json -from pathlib import Path import re import subprocess import sys import tarfile import tempfile import time +from pathlib import Path -spec = importlib.util.spec_from_file_location('publisher', Path(__file__).with_name('publish_repository_share.py')) -publisher = importlib.util.module_from_spec(spec); spec.loader.exec_module(publisher) +spec = importlib.util.spec_from_file_location( + "publisher", Path(__file__).with_name("publish_repository_share.py") +) +publisher = importlib.util.module_from_spec(spec) +_previous_bytecode = sys.dont_write_bytecode +try: + sys.dont_write_bytecode = True + spec.loader.exec_module(publisher) +finally: + sys.dont_write_bytecode = _previous_bytecode def verify(folder, tag, revision): - if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag) or not re.fullmatch('[0-9a-f]{40}', revision): - raise ValueError('invalid release identity') - archive = folder / ('bobbin-' + tag + '-repository.pendant.tar.gz') + if not re.fullmatch( + r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag + ) or not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release identity") + archive = folder / ("bobbin-" + tag + "-repository.pendant.tar.gz") sha = hashlib.sha256(archive.read_bytes()).hexdigest() - if archive.with_name(archive.name + '.sha256').read_text() != sha + ' ' + archive.name + '\n': - raise ValueError('archive checksum mismatch') - proof = json.loads(archive.with_name(archive.name + '.provenance.json').read_text()) - expected = {'schema': 'quipu.repository-share-producer/v1', 'source_repository': 'scbrown/bobbin', - 'source_revision': revision, 'release_tag': tag, 'archive_sha256': sha, 'destination': 'outward'} + if ( + archive.with_name(archive.name + ".sha256").read_text() + != sha + " " + archive.name + "\n" + ): + raise ValueError("archive checksum mismatch") + proof = json.loads(archive.with_name(archive.name + ".provenance.json").read_text()) + expected = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": "scbrown/bobbin", + "source_revision": revision, + "release_tag": tag, + "archive_sha256": sha, + "destination": "outward", + } if any(proof.get(k) != v for k, v in expected.items()): - raise ValueError('repository/source/tag/provenance mismatch') - for key in ('quipu_sha256', 'quipu_server_sha256', 'bobbin_sha256'): - if not re.fullmatch('[0-9a-f]{64}', proof.get(key, '')): - raise ValueError('missing engine/indexer digest') - if not re.fullmatch('[0-9a-f]{40}', proof.get('engine_source_revision', '')): - raise ValueError('missing engine source pin') - with tempfile.TemporaryDirectory(prefix='verify-pendant-') as temp: + raise ValueError("repository/source/tag/provenance mismatch") + for key in ("quipu_sha256", "quipu_server_sha256", "bobbin_sha256"): + if not re.fullmatch("[0-9a-f]{64}", proof.get(key, "")): + raise ValueError("missing engine/indexer digest") + if not re.fullmatch("[0-9a-f]{40}", proof.get("engine_source_revision", "")): + raise ValueError("missing engine source pin") + with tempfile.TemporaryDirectory(prefix="verify-pendant-") as temp: stage = Path(temp) with tarfile.open(archive) as tar: members = tar.getmembers() - if (len(members) != len({m.name for m in members}) - or any(not m.isfile() or m.name not in publisher.MEMBERS for m in members)): - raise ValueError('unexpected, duplicate, or unsafe archive member') + if len(members) != len({m.name for m in members}) or any( + not m.isfile() or m.name not in publisher.MEMBERS for m in members + ): + raise ValueError("unexpected, duplicate, or unsafe archive member") # Every member is a flat allowlisted regular file; no path, link or device extraction. for member in members: (stage / member.name).write_bytes(tar.extractfile(member).read()) @@ -47,28 +68,53 @@ def verify(folder, tag, revision): def wait(tag, revision, attempts=80, pause=time.sleep, run=subprocess.run): - if not re.fullmatch(r'v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?', tag) or not re.fullmatch('[0-9a-f]{40}', revision): - raise ValueError('invalid release identity before lookup') - stem = 'bobbin-' + tag + '-repository.pendant.tar.gz' - last_read = 'not attempted' + if not re.fullmatch( + r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag + ) or not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release identity before lookup") + stem = "bobbin-" + tag + "-repository.pendant.tar.gz" + last_read = "not attempted" for number in range(attempts): - with tempfile.TemporaryDirectory(prefix='repository-pendant-readback-') as temp: + with tempfile.TemporaryDirectory(prefix="repository-pendant-readback-") as temp: folder = Path(temp) try: - result = run(['gh', 'release', 'download', tag, '--repo', 'scbrown/bobbin', '--dir', temp, - '--pattern', stem, '--pattern', stem + '.sha256', '--pattern', stem + '.provenance.json'], - capture_output=True, text=True, timeout=45) - last_read = 'exit=' + str(result.returncode) + result = run( + [ + "gh", + "release", + "download", + tag, + "--repo", + "scbrown/bobbin", + "--dir", + temp, + "--pattern", + stem, + "--pattern", + stem + ".sha256", + "--pattern", + stem + ".provenance.json", + ], + capture_output=True, + text=True, + timeout=45, + ) + last_read = "exit=" + str(result.returncode) if result.returncode == 0: return verify(folder, tag, revision) except subprocess.TimeoutExpired: - last_read = 'read timeout' + last_read = "read timeout" if number + 1 < attempts: pause(30) - raise ValueError('repository pendant unverified after bounded trusted-producer wait; ' + last_read) + raise ValueError( + "repository pendant unverified after bounded trusted-producer wait; " + + last_read + ) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) != 3: - raise SystemExit('usage: verify-repository-share.py ') + raise SystemExit( + "usage: verify-repository-share.py " + ) print(json.dumps(wait(sys.argv[1], sys.argv[2]))) From 4fa241820b2077e14298bbf839d81f29bd480808 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 21:25:18 -0400 Subject: [PATCH 09/12] feat: carry reviewed alignments through native share when supported (aegis-ro425e.16) Engines whose share usage advertises --alignments now receive the reviewed physical carrier, and the builder refuses a manifest whose alignments_hash, files.alignments member or copied bytes differ from it. Older engines keep the existing inline empty carrier and never emit an undeclared member. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BkdpDkAkbTvXvNbNacVpJb --- scripts/build-repository-share.py | 46 +++++++++++++++++++++------- scripts/test-repository-knowledge.py | 45 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py index 586b849f..ea9848bc 100644 --- a/scripts/build-repository-share.py +++ b/scripts/build-repository-share.py @@ -241,6 +241,35 @@ def alignment_carrier(source): return data.decode() +def native_alignments(binary, private): + """Engines that advertise --alignments carry the reviewed physical file. + + Older engines keep the inline empty carrier and never emit an undeclared + archive member that an older receiver would refuse. + """ + usage = subprocess.run( + [str(binary), "share", "--help"], + capture_output=True, + text=True, + env=isolated_environment(private), + cwd=private, + ) + return "--alignments " in usage.stdout + usage.stderr + + +def verify_carried_alignments(stage, alignments): + data = Path(alignments).read_bytes() + carried = json.loads((Path(stage) / "manifest.json").read_text()) + copied = Path(stage) / "alignments.ttl" + if ( + carried.get("alignments_hash") != "sha256:" + hashlib.sha256(data).hexdigest() + or carried.get("files", {}).get("alignments") != "alignments.ttl" + or not copied.is_file() + or copied.read_bytes() != data + ): + raise ValueError("native share did not carry the reviewed alignments") + + def build(revision, source, tooling, binary, bobbin, output): repository = "scbrown/bobbin" source, tooling, output = map(Path, (source, tooling, output)) @@ -410,19 +439,14 @@ def run(args, *, env=None): register_queries(binary, database, json.loads(queries.read_text())) query = (source / "queries/repository-share-bobbin.rq").read_text() stage = private / "share" + native = native_alignments(binary, private) run( - [ - binary, - "share", - "--construct", - query, - "--turtle", - "--output", - stage, - "--db", - database, - ] + [binary, "share", "--construct", query, "--turtle", "--output", stage] + + (["--alignments", alignments] if native else []) + + ["--db", database] ) + if native: + verify_carried_alignments(stage, alignments) if ( not (stage / "shapes.ttl").is_file() or (stage / "shapes.ttl").stat().st_size == 0 diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index e7e69639..73a26eb0 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -102,6 +102,51 @@ def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): builder.alignment_carrier(root) + def test_native_alignment_transport_is_advertised_and_hash_bound(self): + import hashlib + + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + private = root / "private" + private.mkdir() + for name, usage in ( + ("current", "[--queries ]... [--alignments ] [--turtle]"), + ("older", "[--queries ]... [--turtle]"), + ): + engine = root / name + # Real engines print share usage to stderr and exit nonzero. + engine.write_text(f"#!/bin/sh\necho 'usage: quipu share {usage}' >&2\nexit 1\n") + engine.chmod(0o755) + self.assertTrue(builder.native_alignments(root / "current", private)) + self.assertFalse(builder.native_alignments(root / "older", private)) + reviewed = ROOT / "docs/knowledge/alignments.ttl" + stage = root / "share" + stage.mkdir() + shutil.copyfile(reviewed, stage / "alignments.ttl") + manifest = { + "alignments_hash": "sha256:" + + hashlib.sha256(reviewed.read_bytes()).hexdigest(), + "files": {"alignments": "alignments.ttl"}, + } + (stage / "manifest.json").write_text(json.dumps(manifest)) + builder.verify_carried_alignments(stage, reviewed) + for corrupt in ( + {**manifest, "alignments_hash": manifest["alignments_hash"][7:]}, + {**manifest, "files": {}}, + ): + (stage / "manifest.json").write_text(json.dumps(corrupt)) + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + (stage / "manifest.json").write_text(json.dumps(manifest)) + with (stage / "alignments.ttl").open("a") as copied: + copied.write(" .\n") + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + (stage / "alignments.ttl").unlink() + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + + class NativeIsolation(unittest.TestCase): def test_real_subprocess_cannot_see_synthetic_global_config_or_endpoints(self): with tempfile.TemporaryDirectory() as folder: From 5586f67cc3f63d84863479bb463f7be63b03af20 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Thu, 22 Oct 2026 19:25:29 -0400 Subject: [PATCH 10/12] test: use a placeholder account in the private-summary fixture The public-repository identifier guard refuses any /home// path outside its placeholder list. The private-summary negative only needs a /home/ path, so a placeholder account keeps that refusal under test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BkdpDkAkbTvXvNbNacVpJb --- scripts/test-repository-knowledge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index 73a26eb0..c07d615f 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -258,7 +258,7 @@ def test_bad_freshness_and_private_summary_refuse(self): with self.assertRaisesRegex(ValueError, "freshness"): self.render() self.registry["max_age_seconds"] = 30 - self.registry["records"][0]["summary"] = "read /home/private/file" + self.registry["records"][0]["summary"] = "read /home/user/file" self.save() with self.assertRaisesRegex(ValueError, "private summary"): self.render() From 60733f1f1a603146bec74db399f2b4b7389af324 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Thu, 22 Oct 2026 19:54:16 -0400 Subject: [PATCH 11/12] docs: refresh source witnesses after main integration --- docs/knowledge/repository-knowledge.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/knowledge/repository-knowledge.json b/docs/knowledge/repository-knowledge.json index 71225a16..f59aba43 100644 --- a/docs/knowledge/repository-knowledge.json +++ b/docs/knowledge/repository-knowledge.json @@ -35,7 +35,7 @@ "source": "src/mcp/server.rs", "subject": "http://aegis.gastown.local/ontology/code/bobbin/src%2Fmcp%2Fserver.rs", "summary": "The MCP server advertises the tool interface; defining source is the authority for tool names, inputs and feature conditions.", - "source_sha256": "458d3aeb36836d48788e1be3653a2e6ac515d55ea64bf07ddc388a22a79fa4c3", + "source_sha256": "80696a61fd9220297d45e3ac65f791a18911de0db0add6b0030cec4dd76c443f", "properties": { "interfaceKind": "MCP", "publicName": "Bobbin MCP tools" @@ -109,7 +109,7 @@ "source": "README.md", "subject": "http://aegis.gastown.local/ontology/doc/bobbin/README.md", "summary": "Start with installation and the first successful search, then read architecture and contributor guidance.", - "source_sha256": "77189862c0136aad56cfb5407ebb7ae99e913f6823105ffc146217a54653686d", + "source_sha256": "8c2e3973751a8dd7e983cd1a35174b098d5c39bcf74146cd07114d718337737c", "properties": { "startHereOrder": "1" } From 5eb7b225011ee72a074ba731d81ae055087e0d15 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Thu, 22 Oct 2026 20:22:06 -0400 Subject: [PATCH 12/12] fix(share): carry the public policy catalogue and declared alignments The share query never projected the public publication-policy catalogue, so the receiver verifier, which requires all four reviewed rules, could not pass. Add a bounded branch for publication-policy rule IRIs with only their label, regex and tier, leaving out any IRI the private release policy also names, so a private pattern cannot ride along and the receiver fails closed. The publisher's member allowlist rejected alignments.ttl, which a native engine with --alignments carries and the builder already hash-verifies. Admit it only when the manifest declares it and binds its exact bytes; an undeclared, missing, altered or symlinked carrier still refuses. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BkdpDkAkbTvXvNbNacVpJb --- queries/repository-share-bobbin.rq | 13 ++++++ scripts/publish_repository_share.py | 15 ++++++- scripts/test-repository-share-publication.py | 44 ++++++++++++++++++++ scripts/test-repository-share-query.py | 34 ++++++++++++++- 4 files changed, 104 insertions(+), 2 deletions(-) diff --git a/queries/repository-share-bobbin.rq b/queries/repository-share-bobbin.rq index 6715a3a6..8ba1f57d 100644 --- a/queries/repository-share-bobbin.rq +++ b/queries/repository-share-bobbin.rq @@ -47,6 +47,19 @@ WHERE { } } UNION + { + # The public publication-policy catalogue the receiver verifier requires. + # Only reviewed public rule IRIs and their label/regex/tier; a rule IRI that + # also appears in the producer's private policy graph is left out entirely, + # so a private regex can never ride along and the receiver fails closed. + ?s a aegis:InternalIdentifierPattern ; ?p ?o . + FILTER(STRSTARTS(STR(?s), "https://quipu.dev/knowledge/publication-policy/")) + VALUES ?p { rdf:type rdfs:label aegis:regex aegis:enforcementTier } + FILTER NOT EXISTS { + GRAPH { ?s ?privatePredicate ?privateValue } + } + } + UNION { VALUES ?s { aegis:repo_bobbin diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py index 29c27d08..aff9d40b 100644 --- a/scripts/publish_repository_share.py +++ b/scripts/publish_repository_share.py @@ -23,6 +23,7 @@ "manifest.ttl", "shapes.ttl", "queries.ttl", + "alignments.ttl", } @@ -53,7 +54,19 @@ def check_share(stage): ): if not (stage / name).is_file() or not (stage / name).stat().st_size: raise ValueError("required nonempty share member missing: " + name) - recipe = json.loads((stage / "manifest.json").read_text()).get("embedding") + manifest = json.loads((stage / "manifest.json").read_text()) + # A native engine with --alignments carries the reviewed file; it is only + # admitted when the manifest declares it and binds its exact bytes. + carried = stage / "alignments.ttl" + declared = manifest.get("files", {}).get("alignments") + if carried.exists() or declared is not None or "alignments_hash" in manifest: + if ( + declared != "alignments.ttl" + or not carried.is_file() + or manifest.get("alignments_hash") != "sha256:" + digest(carried) + ): + raise ValueError("alignments member is undeclared, missing or not hash-bound") + recipe = manifest.get("embedding") if ( not isinstance(recipe, dict) or not isinstance(recipe.get("model"), str) diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py index c79dd3ac..05506339 100644 --- a/scripts/test-repository-share-publication.py +++ b/scripts/test-repository-share-publication.py @@ -88,6 +88,50 @@ def test_required_members_and_recipe(self): with self.assertRaisesRegex(ValueError, "provider-byte"): publisher.check_share(self.stage) + def carry_alignments(self, data=b"# reviewed empty alignment carrier\n"): + (self.stage / "alignments.ttl").write_bytes(data) + manifest = json.loads((self.stage / "manifest.json").read_text()) + manifest["files"] = {"alignments": "alignments.ttl"} + manifest["alignments_hash"] = "sha256:" + publisher.digest( + self.stage / "alignments.ttl" + ) + (self.stage / "manifest.json").write_text(json.dumps(manifest)) + return manifest + + def test_declared_hash_bound_alignments_publish_and_read_back(self): + self.carry_alignments() + publisher.check_share(self.stage) + self.prepare() + self.assertEqual( + verifier.verify(self.root, "v1.2.3", self.revision)["source_revision"], + self.revision, + ) + + def test_undeclared_missing_corrupt_or_symlinked_alignments_refuse(self): + manifest = self.carry_alignments() + cases = {} + undeclared = dict(manifest) + del undeclared["files"], undeclared["alignments_hash"] + cases["undeclared"] = (undeclared, b"# reviewed empty alignment carrier\n") + cases["corrupt"] = (manifest, b"# altered bytes\n") + cases["missing"] = (manifest, None) + for name, (declared, data) in cases.items(): + with self.subTest(case=name): + (self.stage / "manifest.json").write_text(json.dumps(declared)) + carried = self.stage / "alignments.ttl" + carried.unlink(missing_ok=True) + if data is not None: + carried.write_bytes(data) + with self.assertRaisesRegex(ValueError, "alignments member"): + publisher.check_share(self.stage) + target = self.root / "outside.ttl" + target.write_bytes(b"# reviewed empty alignment carrier\n") + (self.stage / "manifest.json").write_text(json.dumps(manifest)) + (self.stage / "alignments.ttl").unlink(missing_ok=True) + (self.stage / "alignments.ttl").symlink_to(target) + with self.assertRaisesRegex(ValueError, "unexpected or non-regular"): + publisher.check_share(self.stage) + def test_text_line_span_and_private_members_refuse(self): for predicate in ("content", "startLine", "endLine"): (self.stage / "export.nt").write_text( diff --git a/scripts/test-repository-share-query.py b/scripts/test-repository-share-query.py index bcf2d1a5..25399c1c 100644 --- a/scripts/test-repository-share-query.py +++ b/scripts/test-repository-share-query.py @@ -5,7 +5,7 @@ """ from pathlib import Path import unittest -from rdflib import Graph, Literal, Namespace, RDF, RDFS, URIRef +from rdflib import Dataset, Graph, Literal, Namespace, RDF, RDFS, URIRef A = Namespace('http://aegis.gastown.local/ontology/') CODE = Namespace(str(A) + 'code/bobbin/') @@ -64,6 +64,38 @@ def test_ideas_and_sssom_records_survive_without_identity_knots(self): source.add((pointer, OWL.sameAs, other)) self.assertEqual(set(source.query(QUERY).graph), expected) + def test_public_policy_catalogue_survives_and_private_policy_does_not(self): + POLICY = 'https://quipu.dev/knowledge/publication-policy/' + public = Graph() + for name in ('access-tokens', 'private-keys', 'user-directories', 'demo-private-domain'): + rule = URIRef(POLICY + name) + public.add((rule, RDF.type, A.InternalIdentifierPattern)) + public.add((rule, RDFS.label, Literal(name + ' label'))) + public.add((rule, A.regex, Literal(name + '-pattern'))) + public.add((rule, A.enforcementTier, Literal('block'))) + rules = set(public.subjects(RDF.type, A.InternalIdentifierPattern)) + source = Dataset(default_union=True) + for triple in public: + source.add(triple) + shadowed = URIRef(POLICY + 'private-keys') + private = source.graph(URIRef('urn:quipu:private-release-policy')) + # A private rule, and a private regex attached to a public rule IRI. + private.add((URIRef(POLICY + 'internal-hosts'), RDF.type, A.InternalIdentifierPattern)) + private.add((URIRef(POLICY + 'internal-hosts'), A.regex, Literal('corp[.]internal'))) + private.add((shadowed, A.regex, Literal('secret-host-name'))) + # A public rule IRI cannot carry an unreviewed predicate. + source.add((URIRef(POLICY + 'access-tokens'), RDFS.comment, Literal('extra'))) + result = set(source.query(QUERY).graph) + expected = { + t for t in public + if t[0] != shadowed and t[1] in (RDF.type, RDFS.label, A.regex, A.enforcementTier) + } + self.assertEqual(result, expected) + self.assertEqual( + {s for s, _, _ in result}, rules - {shadowed} + ) + self.assertFalse(any('secret-host-name' in str(o) or 'corp' in str(o) for _, _, o in result)) + def test_empty_is_empty_with_populated_positive(self): self.assertEqual(len(Graph().query(QUERY).graph), 0) source = Graph()