From d8114e0a4c9c1d0d034864309a5f9e8ee85a0a38 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 20:08:36 -0400 Subject: [PATCH 1/3] feat: ship source-bound Shuttle repository pendants --- .github/workflows/ci.yml | 16 + .github/workflows/release.yml | 18 + docs/knowledge/README.md | 35 ++ docs/knowledge/alignment-status.json | 8 + docs/knowledge/alignments.ttl | 6 + docs/knowledge/repository-knowledge.json | 130 +++++++ .../knowledge/repository-knowledge.shapes.ttl | 52 +++ docs/knowledge/stored-queries.json | 32 ++ queries/repository-share-shuttle.rq | 68 ++++ scripts/build-repository-knowledge.py | 106 ++++++ scripts/build-repository-share.py | 336 ++++++++++++++++++ scripts/publish-repository-share.sh | 5 + scripts/publish_repository_share.py | 190 ++++++++++ scripts/test-repository-knowledge.py | 191 ++++++++++ scripts/test-repository-share-publication.py | 139 ++++++++ scripts/test-repository-share-query.py | 78 ++++ scripts/verify-repository-share.py | 101 ++++++ 17 files changed, 1511 insertions(+) create mode 100644 docs/knowledge/README.md create mode 100644 docs/knowledge/alignment-status.json create mode 100644 docs/knowledge/alignments.ttl create mode 100644 docs/knowledge/repository-knowledge.json create mode 100644 docs/knowledge/repository-knowledge.shapes.ttl create mode 100644 docs/knowledge/stored-queries.json create mode 100644 queries/repository-share-shuttle.rq create mode 100644 scripts/build-repository-knowledge.py create mode 100644 scripts/build-repository-share.py create mode 100755 scripts/publish-repository-share.sh create mode 100644 scripts/publish_repository_share.py create mode 100644 scripts/test-repository-knowledge.py create mode 100644 scripts/test-repository-share-publication.py create mode 100644 scripts/test-repository-share-query.py create mode 100644 scripts/verify-repository-share.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 724a47f..2ee7a63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,3 +50,19 @@ jobs: # so a change to it fails in CI rather than in an unattended deploy. test "$(printf '%s' "$line" | awk '{print $1}')" = "shuttle" test -n "$(printf '%s' "$line" | awk '{print $2}')" + + repository-pendant-source: + name: Repository pendant source controls + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.11" + - run: python -m pip install 'rdflib==7.1.4' + - run: | + python scripts/test-repository-knowledge.py + python scripts/test-repository-share-publication.py + python scripts/test-repository-share-query.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40d789e..5322385 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -113,3 +113,21 @@ jobs: --notes-file "$RUNNER_TEMP/notes.md" \ dist/shuttle-*.whl dist/shuttle-*.tar.gz dist/checksums.txt fi + + repository-pendant: + name: Verify repository pendant + needs: release + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + env: + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: refs/tags/${{ inputs.tag || github.ref_name }} + fetch-depth: 0 + - name: Verify the bounded public pointer corpus + run: python3 scripts/verify-repository-share.py "$RELEASE_TAG" "$(git rev-parse HEAD)" diff --git a/docs/knowledge/README.md b/docs/knowledge/README.md new file mode 100644 index 0000000..a7ac3df --- /dev/null +++ b/docs/knowledge/README.md @@ -0,0 +1,35 @@ +# Repository pendant source + +The repository-owned release job exports code and document **pointers**, plus +reviewed summaries of ideas, interfaces, decisions, freshness and starting points. +It does not export source chunk text or line spans. Source witnesses bind each +summary to its defining tracked file. The native index must resolve every +curated pointer before the build can proceed. Stored queries and validating +shapes travel with the share. + +`alignments.ttl` is currently a valid empty SSSOM/RDF carrier. Its accompanying +status file explicitly records zero approved mappings and seals those bytes. +This is not evidence of global absence. Missing, changed or malformed input +refuses; an explicit empty carrier does not. Dependencies and glossary references +are not identity alignments. Code/commit deduplication and unrelated exactMatch +records are not cross-repository mapping approval. + +Candidate mappings must go through Quipu's existing `align propose` and +`align decide` curation path. Keep their source graphs and source witnesses; +do not fill an approval author from a lexical score. Only the reviewed mapping +record may produce derived identity through the engine's existing apply path. +The current source profile deliberately does not import arbitrary nonempty +mapping data until an actual curated artifact and its transport are reviewed. + +The copied release workflow verifies the three public assets only after the +ordinary release finishes. The managed trusted host runs this repository's own +publisher, keeps its publication policy private, pins both native engine +executables and the indexer, and refuses retained uncertain upload outcomes. +Default configuration still publishes only the existing Quipu corpus. Enabling +another repository requires its actual trusted host configuration and a clean +release containing this job; committing this source does not publish an asset. + +The publisher requires the engine's model/digest/dimension manifest recipe. +Loading and vector rebuilding belong to the native engine. No hosted job gets +private publication policy or a new credential route. These source controls do +not establish public asset availability, live loading or service acceptance. diff --git a/docs/knowledge/alignment-status.json b/docs/knowledge/alignment-status.json new file mode 100644 index 0000000..88aca2f --- /dev/null +++ b/docs/knowledge/alignment-status.json @@ -0,0 +1,8 @@ +{ + "status": "no-approved-mappings", + "approved_record_count": 0, + "carrier_sha256": "a8f3d420900dce866437e3ecfaba792b4ac4c301ae4702966f9147d4b7d49e3b", + "scope": "scbrown/shuttle repository pendant source", + "basis": "No verified approved artifact or pairs supplied at source review; empty carrier is explicit. This is not a global absence claim.", + "curation_contract": "https://github.com/scbrown/quipu/blob/main/docs/design/cross-graph-alignment.md" +} diff --git a/docs/knowledge/alignments.ttl b/docs/knowledge/alignments.ttl new file mode 100644 index 0000000..b0f73f1 --- /dev/null +++ b/docs/knowledge/alignments.ttl @@ -0,0 +1,6 @@ +# SSSOM/RDF carrier: explicitly no approved Shuttle cross-stack mappings. +# This is an empty mapping graph, not an absent/corrupt input or a claim that +# no mappings exist anywhere. Candidate records require the existing curator +# path before publication as accepted mappings. Dependencies remain dependencies. +@prefix sssom: . +@prefix owl: . diff --git a/docs/knowledge/repository-knowledge.json b/docs/knowledge/repository-knowledge.json new file mode 100644 index 0000000..2d67fe9 --- /dev/null +++ b/docs/knowledge/repository-knowledge.json @@ -0,0 +1,130 @@ +{ + "source_revision": "67ec0d108ce9d51207fdff1ce10b1d1241cad82a", + "reviewed_at": "2026-10-21T00:05:39.939993Z", + "max_age_seconds": 2592000, + "records": [ + { + "kind": "idea", + "source": "README.md", + "subject": "http://aegis.gastown.local/ontology/doc/shuttle/README.md", + "summary": "Shuttle records workflow transitions in a local append-only JSONL log and exports signed history into governed, time-windowed Quipu graphs.", + "source_sha256": "d5ae89b2dda09cd6a7d292b34f4335737088845372924630a1c510d8b455086d", + "type": "Concept" + }, + { + "kind": "interface", + "source": "shuttle/cli.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fcli.py", + "summary": "The CLI defines and advances workflows, exports history, verifies signatures and freezes windows. It has no HTTP or MCP server.", + "source_sha256": "c4255c23dc66050d2a8f8e928123db08ae9b7d1dddcb27018db464b8efbc6931", + "properties": { + "interfaceKind": "CLI", + "publicName": "shuttle" + } + }, + { + "kind": "interface", + "source": "shuttle/state.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fstate.py", + "summary": "SHUTTLE_STATE_DIR selects local append-only workflow state; otherwise the user state directory is used.", + "source_sha256": "d1f0b8428753c569725b925d22e08d25d8590dc1d2a9f2106d172c7d44b74a5b", + "properties": { + "interfaceKind": "config", + "publicName": "SHUTTLE_STATE_DIR" + } + }, + { + "kind": "interface", + "source": "shuttle/signing.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fsigning.py", + "summary": "SHUTTLE_KEY_DIR selects agent key storage; require-existing-key mode refuses missing or unsafe key files and does not create replacements.", + "source_sha256": "4e9a268d068030989e5131bb8d4d9e83e671b3c9e497aaa8fbd10449acc246e3", + "properties": { + "interfaceKind": "config", + "publicName": "SHUTTLE_KEY_DIR / existing_keys_only" + } + }, + { + "kind": "interface", + "source": "shuttle/model.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fmodel.py", + "summary": "Workflow definitions and transition events are pure data models; replay folds validated transitions to reconstruct state.", + "source_sha256": "b6be4c052d4abda4965815da0e9a44d85dc19df2224e8cc29e1bb117f5126198", + "properties": { + "interfaceKind": "file-format", + "publicName": "Workflow definition JSON and transition records" + } + }, + { + "kind": "interface", + "source": "shuttle/quipu_client.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fquipu_client.py", + "summary": "The client uses Quipu HTTP graph, knot and episode APIs. Capability and write refusals are distinct from empty results; it does not serve REST itself.", + "source_sha256": "9037e275caf804b0e9a5a0234631d92f85558b1c5cb7c9f09e2a5b541f622df7", + "properties": { + "interfaceKind": "library", + "publicName": "Quipu client" + }, + "depends_on": [ + "https://github.com/scbrown/quipu" + ] + }, + { + "kind": "interface", + "source": "shuttle/seeds.py", + "subject": "http://aegis.gastown.local/ontology/code/shuttle/shuttle%2Fseeds.py", + "summary": "Seed steps create and reconcile linked work through the configured sd executable; backend routing belongs to that client.", + "source_sha256": "b40c0c128697d92bcfcf326768fab2c9e545b8a291618f2362d068e2c9bbc7f6", + "properties": { + "interfaceKind": "library", + "publicName": "Seed-step reconciliation" + }, + "depends_on": [ + "https://github.com/scbrown/seeds" + ] + }, + { + "kind": "decision", + "source": "docs/design/shuttle.md", + "subject": "http://aegis.gastown.local/ontology/doc/shuttle/docs%2Fdesign%2Fshuttle.md", + "summary": "The v1 design deliberately exposes a CLI and library rather than a daemon, HTTP or MCP server. Consumers read the governed graph instead.", + "source_sha256": "959dd5719f22d0997e7438520eccacc3907fa260fa904f14903d2954a85ad166", + "type": "DecisionRecord", + "properties": { + "status": "active", + "supersessionStatus": "not-recorded-in-source" + } + }, + { + "kind": "start-here", + "source": "README.md", + "subject": "http://aegis.gastown.local/ontology/doc/shuttle/README.md", + "summary": "Begin with the workflow quick start and role in the stack; registration and production writes remain explicit operator actions.", + "source_sha256": "d5ae89b2dda09cd6a7d292b34f4335737088845372924630a1c510d8b455086d", + "properties": { + "startHereOrder": 1, + "glossaryTerm": "https://scbrown.github.io/quechua/ns#Decision" + } + }, + { + "kind": "start-here", + "source": "docs/operating.md", + "subject": "http://aegis.gastown.local/ontology/doc/shuttle/docs%2Foperating.md", + "summary": "Read the operational command and environment contract before exporting or freezing workflow history.", + "source_sha256": "d5634302171e1787507a4e065b46126fd4e3daa328954209981719817382bc64", + "properties": { + "startHereOrder": 2 + } + }, + { + "kind": "start-here", + "source": "docs/seed-steps.md", + "subject": "http://aegis.gastown.local/ontology/doc/shuttle/docs%2Fseed-steps.md", + "summary": "Read seed-step visit, replay and outcome semantics before connecting work-item completion to workflow transitions.", + "source_sha256": "af2b562361814de135058f64047c0a36d0e8b40ec490d2bc26fd1b6374071ed9", + "properties": { + "startHereOrder": 3 + } + } + ] +} diff --git a/docs/knowledge/repository-knowledge.shapes.ttl b/docs/knowledge/repository-knowledge.shapes.ttl new file mode 100644 index 0000000..4c81ee9 --- /dev/null +++ b/docs/knowledge/repository-knowledge.shapes.ttl @@ -0,0 +1,52 @@ +@prefix a: . +@prefix sh: . +@prefix rdfs: . +@prefix xsd: . +@prefix sssom: . +@prefix owl: . +@prefix prov: . + +# This profile constrains only explicitly curated repository knowledge. It +# does not turn every imported Concept or CodeModule into a Bobbin record. +a:ShuttleKnowledgeProfile a sh:NodeShape ; + sh:targetSubjectsOf a:knowledgeKind ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path rdfs:comment ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:knowledgeKind ; sh:minCount 1 ; + sh:in ( "idea" "interface" "decision" "start-here" ) ; ] ; + sh:property [ sh:path a:filePath ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:reviewAfter ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:dateTime ; ] ; + sh:property [ sh:path a:sourceHash ; sh:minCount 1 ; sh:maxCount 1 ; sh:pattern "^[0-9a-f]{64}$" ; ] ; + # maxAge is seconds since the explicit source review, not an eviction rule. + sh:property [ sh:path a:maxAge ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:integer ; sh:minExclusive 0 ; ] ; + sh:property [ sh:path prov:wasDerivedFrom ; sh:minCount 1 ; sh:nodeKind sh:IRI ; ] . + +a:ShuttleInterfaceProfile a sh:NodeShape ; + sh:targetSubjectsOf a:interfaceKind ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path a:interfaceKind ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "CLI" "MCP" "REST" "config" "file-format" "library" ) ; ] ; + sh:property [ sh:path a:publicName ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:depends_on ; sh:nodeKind sh:IRI ; ] . + +a:ShuttleDecisionProfile a sh:NodeShape ; + sh:targetSubjectsOf a:supersessionStatus ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path a:status ; sh:minCount 1 ; sh:maxCount 1 ; sh:in ( "active" "superseded" "proposed" ) ; ] ; + sh:property [ sh:path a:supersessionStatus ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "not-recorded-in-source" "recorded" ) ; ] . + +# Existing SSSOM/RDF is a record, not a second identity-assertion format. The +# native align decide/apply path remains the only way to derive identity knots. +a:ShuttleMappingRecordProfile a sh:NodeShape ; + sh:targetSubjectsOf sssom:subject_id ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path sssom:subject_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:object_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:predicate_id ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] ; + sh:property [ sh:path sssom:mapping_justification ; sh:minCount 1 ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; ] . + +a:ShuttleNoHandAuthoredIdentity a sh:NodeShape ; + sh:targetSubjectsOf owl:sameAs ; + sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; + sh:property [ sh:path owl:sameAs ; sh:maxCount 0 ; ] . diff --git a/docs/knowledge/stored-queries.json b/docs/knowledge/stored-queries.json new file mode 100644 index 0000000..2757a4c --- /dev/null +++ b/docs/knowledge/stored-queries.json @@ -0,0 +1,32 @@ +[ + { + "name": "repository_ideas", + "description": "Repository-owned repository ideas", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a a:Concept; rdfs:comment ?summary . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/shuttle/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/shuttle/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_interfaces", + "description": "Repository-owned repository interfaces", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:interfaceKind ?kind; a:publicName ?name . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/shuttle/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/shuttle/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_decisions", + "description": "Repository-owned repository decisions", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a a:DecisionRecord; a:status ?status . OPTIONAL { ?s ?successor } FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/shuttle/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/shuttle/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_start_here", + "description": "Repository-owned repository start here", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:startHereOrder ?order . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/shuttle/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/shuttle/\")) } ORDER BY ?s LIMIT 100", + "params": [] + }, + { + "name": "repository_freshness", + "description": "Repository-owned repository freshness", + "template": "PREFIX a: PREFIX rdfs: SELECT * WHERE { ?s a:reviewAfter ?reviewAfter; a:maxAge ?maxAgeSeconds . FILTER(STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/doc/shuttle/\") || STRSTARTS(STR(?s), \"http://aegis.gastown.local/ontology/code/shuttle/\")) } ORDER BY ?s LIMIT 100", + "params": [] + } +] diff --git a/queries/repository-share-shuttle.rq b/queries/repository-share-shuttle.rq new file mode 100644 index 0000000..cd36df7 --- /dev/null +++ b/queries/repository-share-shuttle.rq @@ -0,0 +1,68 @@ +PREFIX aegis: +PREFIX rdf: +PREFIX prov: +PREFIX dct: +PREFIX sssom: +PREFIX owl: +PREFIX rdfs: + +# Public repository knowledge only. Bobbin publishes its replaceable snapshot +# into ROOT, so graph/group selectors cannot see it. Keep this predicate list +# aligned with Bobbin's generate_chunk_turtle output: selecting every outgoing +# edge also pulls private operational literals into a public release artifact. +CONSTRUCT { ?s ?p ?o } +WHERE { + { + ?s ?p ?o . + FILTER( + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/code/shuttle/") || + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/doc/shuttle/") || + STRSTARTS(STR(?s), "http://aegis.gastown.local/ontology/chunk/shuttle/") + ) + VALUES ?p { + rdf:type rdfs:label + aegis:filePath aegis:repo aegis:language + aegis:inDocument aegis:chunkOrder aegis:nextChunk aegis:mentions + aegis:name aegis:definedIn aegis:symbolKind + aegis:heading aegis:headingDepth + rdfs:comment rdfs:seeAlso prov:wasDerivedFrom dct:modified dct:isReplacedBy + aegis:knowledgeKind aegis:interfaceKind aegis:publicName aegis:status + aegis:supersessionStatus aegis:startHereOrder aegis:glossaryTerm + aegis:reviewAfter aegis:maxAge aegis:depends_on aegis:sourceHash + } + # Bobbin emits weak mention names as strings. Quipu may additionally + # resolve those names to IRIs under the same predicate in its local store; + # those derived edges do not satisfy the shipped string-valued shape. + FILTER(?p != aegis:mentions || isLiteral(?o)) + } + UNION + { + ?s ?p ?o . + { ?s a sssom:MappingSet } UNION { ?s a owl:Axiom } + VALUES ?p { + rdf:type rdfs:label prov:wasDerivedFrom + sssom:mappings sssom:subject_id sssom:object_id sssom:predicate_id + sssom:subject_label sssom:object_label sssom:mapping_justification + sssom:predicate_modifier sssom:confidence sssom:author_id + } + } + # Versioned crew memory is internal source, including references attached to + # otherwise public context nodes. Internal backups use a separate explicit scope. + FILTER(!STRSTARTS(STR(?s), "urn:quipu:memory:")) + FILTER(!STRSTARTS(STR(?p), "https://quipu.dev/memory#")) + FILTER(!STRSTARTS(STR(?o), "urn:quipu:memory:")) + # Work tracking identifiers are not public repository knowledge. Filter + # both endpoints and literal mentions instead of renaming entity identity. + FILTER(!REGEX(STR(?s), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + FILTER(!REGEX(STR(?o), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + # Exclude the whole node when a required label contains an internal ID. + # Dropping only its label would leave a typed, SHACL-invalid fragment. + MINUS { + ?s ?internalPredicate ?internalValue . + FILTER(REGEX(STR(?internalValue), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + } + MINUS { + ?o ?targetPredicate ?targetValue . + FILTER(REGEX(STR(?targetValue), "(^|[^A-Za-z0-9_])aegis-[A-Za-z0-9]")) + } +} diff --git a/scripts/build-repository-knowledge.py b/scripts/build-repository-knowledge.py new file mode 100644 index 0000000..1ebacf6 --- /dev/null +++ b/scripts/build-repository-knowledge.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Render reviewed summaries onto existing indexed pointers, never new identities.""" + +import datetime +import hashlib +import json +from pathlib import Path +import re +from urllib.parse import quote + +A = "http://aegis.gastown.local/ontology/" +RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" +RDFS = "http://www.w3.org/2000/01/rdf-schema#" +PROV = "http://www.w3.org/ns/prov#" +XSD = "http://www.w3.org/2001/XMLSchema#" +KINDS = {"idea", "interface", "decision", "start-here"} +PROPERTIES = {"interfaceKind", "publicName", "status", "supersessionStatus", "startHereOrder", "glossaryTerm"} + + +def render(source, revision, lookup): + source = Path(source) + if not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("release revision must be explicit") + registry = json.loads((source / "docs/knowledge/repository-knowledge.json").read_text()) + reviewed = datetime.datetime.fromisoformat(registry["reviewed_at"].replace("Z", "+00:00")) + age = registry["max_age_seconds"] + if reviewed.tzinfo is None or type(age) is not int or age <= 0: + raise ValueError("freshness requires an explicit UTC review time and positive seconds") + after = (reviewed + datetime.timedelta(seconds=age)).isoformat().replace("+00:00", "Z") + records = registry["records"] + if {record["kind"] for record in records} != KINDS: + raise ValueError("ideas/interfaces/decisions/start-here inventory incomplete") + lines, seen = [], set() + + def fact(subject, predicate, obj, iri=False, datatype=None): + if iri: + if not re.fullmatch(r'[^\s<>"{}|\\^`]+:[^\s<>"{}|\\^`]*', obj): + raise ValueError("invalid referenced IRI") + value = "<" + obj + ">" + else: + value = json.dumps(str(obj), ensure_ascii=False) + if datatype: + value += "^^<" + datatype + ">" + lines.append("<" + subject + "> <" + predicate + "> " + value + " .") + + for record in records: + path = record["source"] + witness = source / path + if ( + Path(path).is_absolute() + or ".." in Path(path).parts + or not witness.resolve().is_relative_to(source.resolve()) + or hashlib.sha256(witness.read_bytes()).hexdigest() != record["source_sha256"] + ): + raise ValueError("source witness changed: " + path) + subject = record["subject"] + bases = (A + "doc/shuttle/" + quote(path, safe=""), A + "code/shuttle/" + quote(path, safe="")) + if not any(subject == base or subject.startswith(base + "#") for base in bases): + raise ValueError("subject is not the source-backed indexed pointer") + # lookup must use the actual index: a deterministic-looking guessed IRI + # is not an identity proof. Never mint another node when it is absent. + label = lookup(subject) + if not isinstance(label, str) or not label: + raise ValueError("source pointer absent or ambiguous in actual index: " + subject) + if (subject, record["kind"]) in seen: + raise ValueError("duplicate knowledge record") + seen.add((subject, record["kind"])) + if record.get("type") not in ("Concept", "DecisionRecord", None): + raise ValueError("unsupported knowledge type") + fact(subject, RDFS + "label", label) + fact(subject, A + "filePath", path) + if record.get("type"): + fact(subject, RDF + "type", A + record["type"], iri=True) + summary = record["summary"] + if ( + not summary.strip() + or len(summary) > 1000 + or re.search(r"\b[\w-]+\.(?:svc|lan)\b|/home/|\baegis-[A-Za-z0-9]", summary) + ): + raise ValueError("unreviewed or private summary") + fact(subject, RDFS + "comment", summary) + fact(subject, A + "knowledgeKind", record["kind"]) + fact(subject, A + "sourceHash", record["source_sha256"]) + fact(subject, A + "reviewAfter", after, datatype=XSD + "dateTime") + fact(subject, A + "maxAge", age, datatype=XSD + "integer") + fact(subject, "http://purl.org/dc/terms/modified", registry["reviewed_at"], datatype=XSD + "dateTime") + fact( + subject, + PROV + "wasDerivedFrom", + "https://github.com/scbrown/shuttle/blob/" + revision + "/" + path, + iri=True, + ) + for key, value in record.get("properties", {}).items(): + if key not in PROPERTIES: + raise ValueError("unknown content property") + if key == "startHereOrder": + if not str(value).isdigit() or int(value) <= 0: + raise ValueError("start-here order must be a positive integer") + fact(subject, A + key, int(value), datatype=XSD + "integer") + else: + fact(subject, A + key, value) + for dependency in record.get("depends_on", []): + fact(subject, A + "depends_on", dependency, iri=True) + for term in record.get("see_also", []): + fact(subject, RDFS + "seeAlso", term, iri=True) + return "\n".join(sorted(set(lines))) + "\n" diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py new file mode 100644 index 0000000..19da8ab --- /dev/null +++ b/scripts/build-repository-share.py @@ -0,0 +1,336 @@ +"""Build one outward stack share using the existing Quipu/Bobbin boundary. + +Source and tooling are separate fresh, tag-verified checkouts. Index configuration, +live policy, databases and receiver probes stay in the private working directory. +Only the verified share directory leaves this function. +""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import io +import tarfile +import subprocess + + +def command(args, *, env=None, cwd=None): + return subprocess.run([str(v) for v in args], check=True, capture_output=True, text=True, env=env, cwd=cwd).stdout + + +def index_environment(private): + """Keep user-global endpoints and credentials outside private indexing.""" + env = {k: v for k, v in os.environ.items() + if k not in ("BOBBIN_SERVER", "BOBBIN_QUIPU_REMOTE") and not k.startswith("QUIPU_")} + for key, name in (("HOME", "index-home"), ("XDG_CONFIG_HOME", "index-config")): + folder = Path(private) / name + folder.mkdir(mode=0o700) + env[key] = str(folder) + # Models are public cached artifacts, not host configuration. Preserve an + # explicitly configured cache; otherwise resolve its original default. + env.setdefault("XDG_CACHE_HOME", str(Path.home() / ".cache")) + return env + + +def local_mcp(binary, database, calls): + """Short-lived native process over a private DB; no HTTP or live graph.""" + import selectors + + requests = [ + { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "repository-pendant-build", "version": "1"}, + }, + }, + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + ] + requests.extend( + {"jsonrpc": "2.0", "id": number, "method": "tools/call", "params": {"name": name, "arguments": arguments}} + for number, (name, arguments) in enumerate(calls, 1) + ) + values = [] + with subprocess.Popen( + [str(binary), "mcp", "--db", str(database)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=Path(database).parent, + ) as process: + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + try: + for request in requests: + process.stdin.write(json.dumps(request) + "\n") + process.stdin.flush() + if "id" not in request: + continue + if not selector.select(timeout=30): + raise ValueError("native MCP response timeout") + response = json.loads(process.stdout.readline()) + if ( + response.get("id") != request["id"] + or "result" not in response + or "error" in response + or response["result"].get("isError", False) + ): + raise ValueError("native MCP refused operation: " + json.dumps(response)) + if request["id"]: + content = response["result"].get("content", []) + if len(content) != 1 or content[0].get("type") != "text": + raise ValueError("unexpected native MCP result envelope") + values.append(json.loads(content[0]["text"])) + finally: + selector.close() + process.stdin.close() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=10) + if process.returncode: + raise ValueError("native MCP process failed") + return values + + +def register_queries(binary, database, definitions): + if not isinstance(definitions, list) or not definitions: + raise ValueError("explicit repository stored query definitions required") + for definition in definitions: + if not isinstance(definition, dict) or definition.get("action", "load") != "load": + raise ValueError("stored query must be an explicit load definition") + results = local_mcp(binary, database, [("quipu_queries", {**d, "action": "load"}) for d in definitions]) + if [result.get("loaded") for result in results] != [d["name"] for d in definitions]: + raise ValueError("native registry did not return the exact loaded query names") + + +def indexed_labels(binary, database, subjects): + if any(not re.fullmatch(r'https?://[^\s<>"{}|\\^`]+', subject) for subject in subjects): + raise ValueError("invalid indexed subject IRI") + query = ( + "SELECT ?s ?label WHERE { VALUES ?s { " + + " ".join("<" + s + ">" for s in subjects) + + "} ?s ?label }" + ) + (result,) = local_mcp(binary, database, [("quipu_query", {"query": query, "verbose": True})]) + if result.get("truncated") is not False: + raise ValueError("indexed pointer lookup was incomplete") + labels = {} + for row in result["rows"]: + subject, label = row["s"], row["label"] + if isinstance(subject, dict): + subject = subject["value"] + if isinstance(label, dict): + label = label["value"] + if subject in labels and labels[subject] != label: + raise ValueError("ambiguous indexed pointer label") + labels[subject] = label + return labels + + +def materialize_release(source, revision, destination): + """Index only exact tracked release bytes in a new private source tree.""" + destination = Path(destination) + destination.mkdir(mode=0o700) + archive = subprocess.run( + ["git", "-C", str(source), "archive", "--format=tar", revision], check=True, capture_output=True + ).stdout + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as bundle: + for member in bundle.getmembers(): + relative = Path(member.name) + if relative.is_absolute() or ".." in relative.parts or not (member.isfile() or member.isdir()): + raise ValueError("unsupported tracked release member") + target = destination / relative + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with bundle.extractfile(member) as original, target.open("xb") as copied: + shutil.copyfileobj(original, copied) + return destination + + +def alignment_carrier(source): + """An explicit empty reviewed carrier differs from missing or corrupt data. + + Accepted nonempty mapping transport awaits an actual curated artifact. Do + not interpret a dependency, lexical match, or candidate as an identity. + """ + source = Path(source) + data = (source / "docs/knowledge/alignments.ttl").read_bytes() + status = json.loads((source / "docs/knowledge/alignment-status.json").read_text()) + if ( + status.get("status") != "no-approved-mappings" + or type(status.get("approved_record_count")) is not int + or status["approved_record_count"] != 0 + or status.get("carrier_sha256") != hashlib.sha256(data).hexdigest() + ): + raise ValueError("unreviewed or corrupt alignment carrier") + # Only declarations/comments are permitted in the explicit-empty profile. + # The native data validator below still parses the complete Turtle input. + for line in data.decode().splitlines(): + if ( + line.strip() + and not line.lstrip().startswith("#") + and line.strip() + not in {"@prefix sssom: .", "@prefix owl: ."} + ): + raise ValueError("empty alignment profile contains mapping data") + return data.decode() + + +def build(revision, source, tooling, binary, bobbin, output): + repository = "scbrown/shuttle" + source, tooling, output = map(Path, (source, tooling, output)) + if output.exists(): + raise ValueError("repository share output already exists") + if not re.fullmatch(r"scbrown/[a-z][a-z0-9-]*", repository): + raise ValueError("invalid repository identity") + slug = repository.split("/")[1] + if ( + not re.fullmatch(r"[0-9a-f]{40}", revision) + or command(["git", "-C", source, "rev-parse", "HEAD"]).strip() != revision + or command(["git", "-C", source, "status", "--porcelain"]).strip() + ): + raise ValueError("repository source is not the exact clean release") + paths = command(["git", "-C", source, "ls-files", "-z"]).split("\0") + control = next((p for p in ("README.md", "README.rst") if p in paths), None) + if control is None: + raise ValueError("release lacks a supported public corpus positive control") + private = output.parent / "build-private" + private.mkdir(mode=0o700) + + def run(args, *, env=None): + return command(args, env=env, cwd=private) + + policy = private / "policy.ttl" + run(["python3", tooling / "scripts/private-share-policy.py", policy]) + corpus = materialize_release(source, revision, private / "source") + (corpus / ".bobbin").mkdir(exist_ok=True) + index_database = private / "indexed-graph.db" + # Bobbin resolves embedded graph state at --source, not at the vector + # index root. Explicit private configuration avoids global-store fallback. + (corpus / ".bobbin/config.toml").write_text("[quipu]\nstore_path = " + json.dumps(str(index_database)) + "\n") + index = private / "index" + (index / ".bobbin").mkdir(parents=True) + (index / ".bobbin/config.toml").write_text("""quipu_push_chunks = true + +[index] +include = [ + "**/*.rs", "**/*.md", "**/*.rst", "**/*.toml", "**/*.yml", "**/*.yaml", "**/*.py", + "**/*.sh", "**/*.js", "**/*.mjs", "**/*.ts", "**/*.tsx", "**/*.json", "**/*.ttl", "**/*.go", +] +exclude = [ + "**/target/**", "**/.git/**", "**/.bobbin/**", "**/.agents/**", "**/.claude/**", "**/.codex/**", + "**/.mcp.json", "**/CLAUDE.local.md", "**/node_modules/**", "**/book/book/**", "**/vendor/**", +] +use_gitignore = true +entities = false + +[embedding] +model = "all-MiniLM-L6-v2" +batch_size = 64 +""") + env = index_environment(private) + run([bobbin, "index", index, "--source", corpus, "--repo", slug, "--force", "--skip-calibrate", "--json"], env=env) + database = private / "producer.db" + if not index_database.is_file(): + raise ValueError("indexer did not produce the repository graph") + with sqlite3.connect("file:" + str(index_database) + "?mode=ro", uri=True) as original: + with sqlite3.connect(database) as target: + original.backup(target) + shapes = sorted((tooling / "shapes").glob("*.ttl")) + if not shapes: + raise ValueError("verified engine source has no receiver shapes") + for shape in shapes: + run([binary, "shapes", "load", shape.stem, shape, "--db", database]) + run( + [ + binary, + "shapes", + "load", + "publication-policy", + tooling / "docs/knowledge/publication-policy.shapes.ttl", + "--db", + database, + ] + ) + run([binary, "knot", tooling / "docs/knowledge/publication-policy.ttl", "--db", database]) + run([binary, "knot", policy, "--graph", "urn:quipu:private-release-policy", "--db", database]) + # Actual indexed pointers and sealed repository/source provenance identify this corpus. + # Curated repository content and validating shapes are repo-owned. Refuse + # an incomplete source proposal rather than minting a pointer-only stub. + registry_path = source / "docs/knowledge/repository-knowledge.json" + knowledge = private / "repository-knowledge.ttl" + constraints = source / "docs/knowledge/repository-knowledge.shapes.ttl" + queries = source / "docs/knowledge/stored-queries.json" + alignments = source / "docs/knowledge/alignments.ttl" + for required in (registry_path, constraints, queries, alignments): + if not required.is_file() or not required.stat().st_size: + raise ValueError("required repository knowledge member missing: " + required.name) + import importlib.util + + spec = importlib.util.spec_from_file_location( + "repository_knowledge", source / "scripts/build-repository-knowledge.py" + ) + generator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generator) + records = json.loads(registry_path.read_text())["records"] + labels = indexed_labels(binary, database, sorted({record["subject"] for record in records})) + knowledge.write_text( + generator.render(source, revision, lambda subject: labels.get(subject)) + "\n" + alignment_carrier(source) + ) + run([binary, "validate", "--shapes", constraints, "--data", knowledge]) + run([binary, "shapes", "load", "repository-knowledge", constraints, "--db", database]) + run([binary, "knot", knowledge, "--db", database]) + register_queries(binary, database, json.loads(queries.read_text())) + query = (source / "queries/repository-share-shuttle.rq").read_text() + stage = private / "share" + run([binary, "share", "--construct", query, "--turtle", "--output", stage, "--db", database]) + if not (stage / "shapes.ttl").is_file() or (stage / "shapes.ttl").stat().st_size == 0: + raise ValueError("repository share omitted receiver shapes") + receiver = private / "receiver.db" + run([binary, "shapes", "load", "repository-share", stage / "shapes.ttl", "--db", receiver]) + imported = json.loads(run([binary, "import", stage, "--db", receiver])) + if ( + imported.get("outcome") != "staged" + or imported.get("promotion", {}).get("eligible") is not True + or type(imported.get("triples", {}).get("accepted")) is not int + or imported["triples"]["accepted"] <= 3 + ): + raise ValueError("repository receiver did not admit a non-stub corpus") + run([binary, "import", "promote", imported["share_id"], "--db", receiver]) + positive = run( + [ + binary, + "query", + "SELECT ?s WHERE { ?s " + json.dumps(control) + " } LIMIT 1", + "--db", + receiver, + ] + ) + if re.search(r"\n1 results\s*$", positive) is None: + raise ValueError("repository corpus positive control missing after import") + run(["python3", tooling / "scripts/verify-repository-share-receiver.py", binary, stage, receiver, private]) + if run(["git", "-C", source, "status", "--porcelain"]).strip(): + raise ValueError("indexing changed the exact repository source") + shutil.move(stage, output) + return {"triples": imported["triples"]["accepted"], "positive_file": control} + + +if __name__ == "__main__": + import sys + + if len(sys.argv) != 7: + raise SystemExit( + "usage: build-repository-share.py " + ) + build(*sys.argv[1:]) diff --git a/scripts/publish-repository-share.sh b/scripts/publish-repository-share.sh new file mode 100755 index 0000000..3b0f0f9 --- /dev/null +++ b/scripts/publish-repository-share.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Repo-owned job, invoked only by the managed trusted-host release dispatcher. +set -euo pipefail +umask 077 +exec python3 "$(dirname -- "${BASH_SOURCE[0]}")/publish_repository_share.py" "$@" diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py new file mode 100644 index 0000000..29eee46 --- /dev/null +++ b/scripts/publish_repository_share.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Shuttle's owned trusted release job; hosted workflows only verify its output.""" + +import gzip +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +REPOSITORY = "scbrown/shuttle" +MEMBERS = {"export.nt", "export.ttl", "payload.nq", "manifest.json", "manifest.ttl", "shapes.ttl", "queries.ttl"} + + +def command(args): + return subprocess.run([str(v) for v in args], check=True, capture_output=True, text=True).stdout + + +def digest(path): + result = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + result.update(block) + return result.hexdigest() + + +def check_share(stage): + files = list(stage.iterdir()) + if any(not p.is_file() or p.is_symlink() or p.name not in MEMBERS for p in files): + raise ValueError("unexpected or non-regular share member") + for name in ("export.nt", "manifest.json", "manifest.ttl", "shapes.ttl", "queries.ttl"): + if not (stage / name).is_file() or not (stage / name).stat().st_size: + raise ValueError("required nonempty share member missing: " + name) + recipe = json.loads((stage / "manifest.json").read_text()).get("embedding") + if ( + not isinstance(recipe, dict) + or not isinstance(recipe.get("model"), str) + or not recipe["model"] + or Path(recipe["model"]).name != recipe["model"] + or type(recipe.get("dimension")) is not int + or recipe["dimension"] <= 0 + or any(not re.fullmatch("[0-9a-f]{64}", recipe.get(k, "")) for k in ("model_sha256", "tokenizer_sha256")) + ): + raise ValueError("share omitted a provider-byte embedding recipe") + # This repository pack is pointer-only. Curated idea summaries use + # rdfs:comment; source bodies and line spans are never release predicates. + export = (stage / "export.nt").read_text() + for predicate in ("content", "startLine", "endLine"): + if "" in export: + raise ValueError("share duplicates source text or line-span data") + return recipe + + +def package(stage, archive): + check_share(stage) + with archive.open("xb") as raw: + with gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w") as tar: + for path in sorted(stage.iterdir()): + info = tar.gettarinfo(str(path), arcname=path.name) + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = 0 + info.mode = 0o600 + with path.open("rb") as handle: + tar.addfile(info, handle) + + +def held(): + path = Path(os.environ.get("QUIPU_SHARE_HOLD_FILE", str(Path.home() / ".quipu-share-hold"))) + try: + path.lstat() + except FileNotFoundError: + return False + except OSError as error: + raise ValueError("cannot inspect publication hold") from error + return True + + +def publish(tag, revision, output): + if held(): + raise ValueError("publication held before any lookup/build") + if not re.fullmatch(r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag): + raise ValueError("unexpected Bobbin release tag") + if not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release revision") + source = Path(__file__).resolve().parents[1] + if ( + command(["git", "-C", source, "rev-parse", "HEAD"]).strip() != revision + or command(["git", "-C", source, "rev-parse", "refs/tags/" + tag + "^{commit}"]).strip() != revision + or command(["git", "-C", source, "status", "--porcelain"]).strip() + ): + raise ValueError("publisher source is not the exact clean release") + output = Path(output) + if output.exists(): + raise ValueError("retained output exists; reconcile before retrying") + binary = Path(os.environ["QUIPU_SHARE_ENGINE_BIN"]).resolve(strict=True) + tooling = Path(os.environ["QUIPU_SHARE_ENGINE_SOURCE"]).resolve(strict=True) + bobbin = Path(os.environ["QUIPU_SHARE_BOBBIN"]).resolve(strict=True) + engine_revision = os.environ["QUIPU_SHARE_ENGINE_REVISION"] + if ( + not re.fullmatch("[0-9a-f]{40}", engine_revision) + or command(["git", "-C", tooling, "rev-parse", "HEAD"]).strip() != engine_revision + or command(["git", "-C", tooling, "status", "--porcelain"]).strip() + or digest(binary) != os.environ["QUIPU_SHARE_ENGINE_BIN_SHA256"] + or digest(binary.with_name("quipu-server")) != os.environ["QUIPU_SHARE_ENGINE_SERVER_SHA256"] + or digest(bobbin) != os.environ["QUIPU_SHARE_BOBBIN_SHA256"] + ): + raise ValueError("trusted engine/source/indexer pin mismatch") + output.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with tempfile.TemporaryDirectory(prefix="prepare-", dir=output.parent) as temp: + private = Path(temp) + stage = private / "share" + spec = importlib.util.spec_from_file_location( + "bobbin_share_build", source / "scripts/build-repository-share.py" + ) + builder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(builder) + builder.build(revision, source, tooling, binary, bobbin, stage) + archive_name = "shuttle-" + tag + "-repository.pendant.tar.gz" + prepared = private / "assets" + prepared.mkdir(mode=0o700) + archive = prepared / archive_name + package(stage, archive) + (prepared / (archive_name + ".sha256")).write_text(digest(archive) + " " + archive_name + "\n") + proof = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": REPOSITORY, + "source_revision": revision, + "release_tag": tag, + "archive_sha256": digest(archive), + "quipu_sha256": digest(binary), + "engine_source_revision": engine_revision, + "bobbin_sha256": digest(bobbin), + "quipu_server_sha256": digest(binary.with_name("quipu-server")), + "destination": "outward", + } + (prepared / (archive_name + ".provenance.json")).write_text(json.dumps(proof, sort_keys=True, indent=2) + "\n") + names = sorted(p.name for p in prepared.iterdir()) + if held(): + raise ValueError("publication held after preparation") + available = json.loads(command(["gh", "release", "view", tag, "--repo", REPOSITORY, "--json", "assets"])) + present = {a["name"] for a in available["assets"]} & set(names) + if present and present != set(names): + raise ValueError("partial publication; explicit reconciliation required") + # Retain before the one-way upload. No --clobber and no automatic retry + # from an uncertain write. The managed dispatcher also retains STARTED. + shutil.copytree(prepared, output) + if not present: + if held(): + raise ValueError("publication held immediately before upload") + command(["gh", "release", "upload", tag, "--repo", REPOSITORY, *(output / name for name in names)]) + downloaded = private / "downloaded" + downloaded.mkdir() + command( + [ + "gh", + "release", + "download", + tag, + "--repo", + REPOSITORY, + "--dir", + downloaded, + *(arg for name in names for arg in ("--pattern", name)), + ] + ) + for name in names: + if (downloaded / name).read_bytes() != (output / name).read_bytes(): + raise ValueError("existing or newly published asset differs; never overwrite") + receipt = { + **proof, + "published_and_read_back": True, + "assets": names, + "asset_sha256": {name: digest(downloaded / name) for name in names}, + } + (output / "delivery.json").write_text(json.dumps(receipt, indent=2) + "\n") + return receipt + + +if __name__ == "__main__": + if len(sys.argv) != 5 or sys.argv[1] != "--trusted-release": + raise SystemExit("usage: publish-repository-share.sh --trusted-release ") + print(json.dumps(publish(sys.argv[2], sys.argv[3], sys.argv[4]))) diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py new file mode 100644 index 0000000..49e49c1 --- /dev/null +++ b/scripts/test-repository-knowledge.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Curated-source refusal controls. Lookup fixtures are not index coverage proof. + +# arming: ci; release contract invokes this script. +""" + +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("knowledge", ROOT / "scripts/build-repository-knowledge.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +builder_spec = importlib.util.spec_from_file_location("share_builder", ROOT / "scripts/build-repository-share.py") +builder = importlib.util.module_from_spec(builder_spec) +builder_spec.loader.exec_module(builder) + + +class ReleaseSnapshot(unittest.TestCase): + def test_real_git_archive_excludes_untracked_key_and_preserves_source(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + source = root / "source" + source.mkdir() + subprocess.run(["git", "init", "-q", str(source)], check=True) + (source / "README.md").write_text("tracked pointer source\n") + subprocess.run(["git", "-C", str(source), "add", "README.md"], check=True) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=fixture", + "-c", + "user.email=fixture@example.org", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + (source / "private.key").write_text("synthetic untracked sentinel") + revision = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() + copied = builder.materialize_release(source, revision, root / "copied") + self.assertEqual((copied / "README.md").read_bytes(), (source / "README.md").read_bytes()) + self.assertFalse((copied / "private.key").exists()) + self.assertEqual((source / "private.key").read_text(), "synthetic untracked sentinel") + self.assertFalse((source / ".bobbin").exists()) + + +class IndexIsolation(unittest.TestCase): + def test_private_configuration_and_credentials_do_not_follow_parent(self): + from unittest.mock import patch + with tempfile.TemporaryDirectory() as folder: + with patch.dict(os.environ, {"HOME": "/synthetic-parent", "XDG_CONFIG_HOME": "/synthetic-global", + "BOBBIN_SERVER": "https://invalid.example", "QUIPU_AUTH_TOKEN": "synthetic-only"}): + env = builder.index_environment(Path(folder)) + self.assertEqual(env["HOME"], str(Path(folder) / "index-home")) + self.assertEqual(env["XDG_CONFIG_HOME"], str(Path(folder) / "index-config")) + self.assertNotIn("BOBBIN_SERVER", env) + self.assertNotIn("QUIPU_AUTH_TOKEN", env) + self.assertEqual(list((Path(folder) / "index-config").iterdir()), []) + + +class AlignmentCarrier(unittest.TestCase): + def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + target = root / "docs/knowledge" + target.mkdir(parents=True) + for name in ("alignments.ttl", "alignment-status.json"): + shutil.copyfile(ROOT / "docs/knowledge" / name, target / name) + self.assertIn("@prefix sssom:", builder.alignment_carrier(root)) + data = target / "alignments.ttl" + original = data.read_text() + data.unlink() + with self.assertRaises(FileNotFoundError): + builder.alignment_carrier(root) + data.write_text(original + " .\n") + with self.assertRaisesRegex(ValueError, "corrupt"): + builder.alignment_carrier(root) + import hashlib + + status = json.loads((target / "alignment-status.json").read_text()) + status["carrier_sha256"] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / "alignment-status.json").write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, "mapping data"): + builder.alignment_carrier(root) + data.write_text("@prefix owl: . .\n") + status["carrier_sha256"] = hashlib.sha256(data.read_bytes()).hexdigest() + (target / "alignment-status.json").write_text(json.dumps(status)) + with self.assertRaisesRegex(ValueError, "mapping data"): + builder.alignment_carrier(root) + + +class Knowledge(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.registry = json.loads((ROOT / "docs/knowledge/repository-knowledge.json").read_text()) + for source in {r["source"] for r in self.registry["records"]}: + target = self.root / source + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / source, target) + (self.root / "docs/knowledge").mkdir(parents=True) + self.save() + + def save(self): + (self.root / "docs/knowledge/repository-knowledge.json").write_text(json.dumps(self.registry)) + + def render(self, lookup=lambda _: "actual-index-label-fixture"): + return module.render(self.root, "1" * 40, lookup) + + def test_source_bound_nonempty_content(self): + text = self.render() + self.assertIn("2592000", text) + self.assertIn("reviewAfter", text) + self.assertIn('knowledgeKind> "idea"', text) + self.assertIn('interfaceKind> "file-format"', text) + self.assertIn('supersessionStatus> "not-recorded-in-source"', text) + self.assertIn('startHereOrder> "3"^^', text) + self.assertNotIn("/content>", text) + self.assertNotIn("owl#sameAs", text) + + def test_absent_or_ambiguous_index_pointer_refuses(self): + for value in (None, False, ""): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "absent or ambiguous"): + self.render(lambda _: value) + + def test_changed_source_refuses(self): + path = self.root / self.registry["records"][0]["source"] + path.write_text(path.read_text() + "\nchanged\n") + with self.assertRaisesRegex(ValueError, "witness changed"): + self.render() + + def test_guessed_identity_refuses(self): + self.registry["records"][0]["subject"] = "https://example.org/guessed" + self.save() + with self.assertRaisesRegex(ValueError, "source-backed"): + self.render() + + def test_missing_required_content_refuses(self): + self.registry["records"] = [r for r in self.registry["records"] if r["kind"] != "decision"] + self.save() + with self.assertRaisesRegex(ValueError, "inventory incomplete"): + self.render() + + def test_bad_freshness_and_private_summary_refuse(self): + self.registry["max_age_seconds"] = 0 + self.save() + with self.assertRaisesRegex(ValueError, "freshness"): + self.render() + self.registry["max_age_seconds"] = 30 + self.registry["records"][0]["summary"] = "read /home/private/file" + self.save() + with self.assertRaisesRegex(ValueError, "private summary"): + self.render() + + @unittest.skipUnless(os.environ.get("QUIPU_TEST_BIN"), "native schema control requires explicit test artifact") + def test_actual_native_schema_positive_and_missing_freshness_negative(self): + data = self.root / "curated.ttl" + data.write_text(self.render()) + args = [ + os.environ["QUIPU_TEST_BIN"], + "validate", + "--shapes", + str(ROOT / "docs/knowledge/repository-knowledge.shapes.ttl"), + "--data", + str(data), + "--format", + "json", + ] + positive = subprocess.run(args, capture_output=True, text=True, cwd=self.root) + self.assertEqual(positive.returncode, 0, positive.stderr + positive.stdout) + data.write_text("\n".join(line for line in data.read_text().splitlines() if "/reviewAfter>" not in line)) + negative = subprocess.run(args, capture_output=True, text=True, cwd=self.root) + self.assertNotEqual(negative.returncode, 0, negative.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py new file mode 100644 index 0000000..24fc534 --- /dev/null +++ b/scripts/test-repository-share-publication.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Packaging/readback refusal controls; synthetic data is not native acceptance. + +# arming: ci; release asset publication contract invokes this script. +""" + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parent + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +publisher = load("publisher_test", ROOT / "publish_repository_share.py") +verifier = load("verifier_test", ROOT / "verify-repository-share.py") + + +class Publication(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.stage = self.root / "stage" + self.stage.mkdir() + recipe = {"model": "model.onnx", "model_sha256": "a" * 64, "tokenizer_sha256": "b" * 64, "dimension": 384} + (self.stage / "manifest.json").write_text(json.dumps({"embedding": recipe})) + for name in ("export.nt", "manifest.ttl", "shapes.ttl", "queries.ttl"): + (self.stage / name).write_text("# synthetic fixture only\n") + self.archive = self.root / "shuttle-v1.2.3-repository.pendant.tar.gz" + self.revision = "1" * 40 + + def prepare(self): + publisher.package(self.stage, self.archive) + sha = publisher.digest(self.archive) + self.archive.with_name(self.archive.name + ".sha256").write_text(sha + " " + self.archive.name + "\n") + self.proof = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": "scbrown/shuttle", + "source_revision": self.revision, + "release_tag": "v1.2.3", + "archive_sha256": sha, + "destination": "outward", + "quipu_sha256": "2" * 64, + "quipu_server_sha256": "5" * 64, + "bobbin_sha256": "3" * 64, + "engine_source_revision": "4" * 40, + } + self.proof_path = self.archive.with_name(self.archive.name + ".provenance.json") + self.proof_path.write_text(json.dumps(self.proof)) + + def test_deterministic_archive_and_exact_readback(self): + self.prepare() + other = self.root / "second.tar.gz" + publisher.package(self.stage, other) + self.assertEqual(self.archive.read_bytes(), other.read_bytes()) + self.assertEqual(verifier.verify(self.root, "v1.2.3", self.revision)["source_revision"], self.revision) + + def test_required_members_and_recipe(self): + (self.stage / "queries.ttl").unlink() + with self.assertRaisesRegex(ValueError, "required nonempty"): + publisher.check_share(self.stage) + (self.stage / "queries.ttl").write_text("# query") + (self.stage / "manifest.json").write_text("{}") + with self.assertRaisesRegex(ValueError, "provider-byte"): + publisher.check_share(self.stage) + + def test_text_line_span_and_private_members_refuse(self): + for predicate in ("content", "startLine", "endLine"): + (self.stage / "export.nt").write_text( + " "body" .\n' + ) + with self.subTest(predicate=predicate), self.assertRaisesRegex(ValueError, "duplicates source"): + publisher.check_share(self.stage) + (self.stage / "export.nt").write_text("# empty synthetic") + (self.stage / "policy-token").write_text("not a real token") + with self.assertRaisesRegex(ValueError, "unexpected"): + publisher.check_share(self.stage) + + def test_foreign_repository_and_wrong_source_refuse(self): + self.prepare() + for field, value in [ + ("source_repository", "scbrown/quipu"), + ("source_revision", "f" * 40), + ("release_tag", "v9.9.9"), + ("archive_sha256", "f" * 64), + ]: + self.proof_path.write_text(json.dumps({**self.proof, field: value})) + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "provenance mismatch"): + verifier.verify(self.root, "v1.2.3", self.revision) + + def test_checksum_refuses(self): + self.prepare() + self.archive.with_name(self.archive.name + ".sha256").write_text("wrong") + with self.assertRaisesRegex(ValueError, "checksum"): + verifier.verify(self.root, "v1.2.3", self.revision) + + def test_hold_absent_and_dangling_symlink_before_any_build(self): + hold = self.root / "hold" + with patch.dict(publisher.os.environ, {"QUIPU_SHARE_HOLD_FILE": str(hold)}): + self.assertFalse(publisher.held()) + hold.symlink_to(self.root / "missing-target") + self.assertTrue(publisher.held()) + with self.assertRaisesRegex(ValueError, "held before"): + publisher.publish("invalid-tag", "invalid-revision", self.root / "output") + self.assertFalse((self.root / "output").exists()) + + def test_unreadable_hold_fails_closed(self): + with patch.object(Path, "lstat", side_effect=PermissionError("synthetic denied")): + with self.assertRaisesRegex(ValueError, "cannot inspect publication hold"): + publisher.held() + + def test_wait_is_bounded(self): + calls, pauses = [], [] + + class Result: + returncode = 1 + + def run(args, **kwargs): + calls.append(args) + return Result() + + with self.assertRaisesRegex(ValueError, "bounded"): + verifier.wait("v1.2.3", self.revision, attempts=2, pause=pauses.append, run=run) + self.assertEqual(len(calls), 2) + self.assertEqual(pauses, [30]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-repository-share-query.py b/scripts/test-repository-share-query.py new file mode 100644 index 0000000..e1da5c4 --- /dev/null +++ b/scripts/test-repository-share-query.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Private corpus controls for Shuttle's copied pointer-only release query. + +# arming: agent-invoked; run before publishing the repository-pendant source. +""" + +from pathlib import Path +import unittest +from rdflib import Graph, Literal, Namespace, RDF, RDFS, URIRef + +A = Namespace("http://aegis.gastown.local/ontology/") +CODE = Namespace(str(A) + "code/shuttle/") +DOC = Namespace(str(A) + "doc/shuttle/") +QUERY = (Path(__file__).resolve().parents[1] / "queries/repository-share-shuttle.rq").read_text() + + +class RepositoryQuery(unittest.TestCase): + def test_pointer_corpus_and_private_boundaries(self): + source = Graph() + doc, chunk = DOC["README.md"], URIRef(str(A) + "chunk/shuttle/README.md#C1") + expected = { + (doc, RDF.type, A.Document), + (doc, A.filePath, Literal("README.md")), + (chunk, RDF.type, A.Chunk), + (chunk, A.inDocument, doc), + (chunk, A.chunkOrder, Literal(0)), + (chunk, A.mentions, Literal("Store")), + (CODE["store.rs"], A.filePath, Literal("src/store.rs")), + } + for triple in expected: + source.add(triple) + blocked = { + (chunk, A.content, Literal("body must not be exported")), + (chunk, A.startLine, Literal(1)), + (chunk, A.endLine, Literal(30)), + (chunk, A.mentions, URIRef("https://example.org/resolved")), + (DOC["private"], RDFS.label, Literal("aegis-private123")), + (DOC["private"], RDF.type, A.Document), + (URIRef(str(A) + "doc/other/README.md"), A.filePath, Literal("foreign")), + (URIRef("urn:quipu:memory:private"), RDFS.label, Literal("memory")), + } + for triple in blocked: + source.add(triple) + result = set(source.query(QUERY).graph) + self.assertEqual(result, expected) + + def test_ideas_and_sssom_records_survive_without_identity_knots(self): + source = Graph() + S = Namespace("https://w3id.org/sssom/") + OWL = Namespace("http://www.w3.org/2002/07/owl#") + record = URIRef("https://example.org/fixture/mapping") + pointer = DOC["guide.md#idea"] + other = URIRef("https://example.org/other/idea") + expected = { + (pointer, RDF.type, A.Concept), + (pointer, RDFS.comment, Literal("Curated high-level idea")), + (pointer, A.reviewAfter, Literal("2026-11-01T00:00:00Z")), + (record, RDF.type, OWL.Axiom), + (record, S.subject_id, pointer), + (record, S.object_id, other), + (record, S.predicate_id, URIRef("http://www.w3.org/2004/02/skos/core#closeMatch")), + (record, S.mapping_justification, URIRef("https://w3id.org/semapv/vocab/LexicalMatching")), + } + for triple in expected: + source.add(triple) + source.add((pointer, OWL.sameAs, other)) + self.assertEqual(set(source.query(QUERY).graph), expected) + + def test_empty_is_empty_with_populated_positive(self): + self.assertEqual(len(Graph().query(QUERY).graph), 0) + source = Graph() + triple = (DOC["README.md"], A.filePath, Literal("README.md")) + source.add(triple) + self.assertEqual(set(source.query(QUERY).graph), {triple}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-repository-share.py b/scripts/verify-repository-share.py new file mode 100644 index 0000000..31e8d76 --- /dev/null +++ b/scripts/verify-repository-share.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Bounded hosted readback; build policy and credentials stay on trusted host.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import time + +spec = importlib.util.spec_from_file_location("publisher", Path(__file__).with_name("publish_repository_share.py")) +publisher = importlib.util.module_from_spec(spec) +spec.loader.exec_module(publisher) + + +def verify(folder, tag, revision): + if not re.fullmatch(r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag) or not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release identity") + archive = folder / ("shuttle-" + tag + "-repository.pendant.tar.gz") + sha = hashlib.sha256(archive.read_bytes()).hexdigest() + if archive.with_name(archive.name + ".sha256").read_text() != sha + " " + archive.name + "\n": + raise ValueError("archive checksum mismatch") + proof = json.loads(archive.with_name(archive.name + ".provenance.json").read_text()) + expected = { + "schema": "quipu.repository-share-producer/v1", + "source_repository": "scbrown/shuttle", + "source_revision": revision, + "release_tag": tag, + "archive_sha256": sha, + "destination": "outward", + } + if any(proof.get(k) != v for k, v in expected.items()): + raise ValueError("repository/source/tag/provenance mismatch") + for key in ("quipu_sha256", "quipu_server_sha256", "bobbin_sha256"): + if not re.fullmatch("[0-9a-f]{64}", proof.get(key, "")): + raise ValueError("missing engine/indexer digest") + if not re.fullmatch("[0-9a-f]{40}", proof.get("engine_source_revision", "")): + raise ValueError("missing engine source pin") + with tempfile.TemporaryDirectory(prefix="verify-pendant-") as temp: + stage = Path(temp) + with tarfile.open(archive) as tar: + members = tar.getmembers() + if len(members) != len({m.name for m in members}) or any( + not m.isfile() or m.name not in publisher.MEMBERS for m in members + ): + raise ValueError("unexpected, duplicate, or unsafe archive member") + # Every member is a flat allowlisted regular file; no path, link or device extraction. + for member in members: + (stage / member.name).write_bytes(tar.extractfile(member).read()) + publisher.check_share(stage) + return expected + + +def wait(tag, revision, attempts=80, pause=time.sleep, run=subprocess.run): + if not re.fullmatch(r"v\d+\.\d+\.\d+(?:[.-][A-Za-z0-9.-]+)?", tag) or not re.fullmatch("[0-9a-f]{40}", revision): + raise ValueError("invalid release identity before lookup") + stem = "shuttle-" + tag + "-repository.pendant.tar.gz" + last_read = "not attempted" + for number in range(attempts): + with tempfile.TemporaryDirectory(prefix="repository-pendant-readback-") as temp: + folder = Path(temp) + try: + result = run( + [ + "gh", + "release", + "download", + tag, + "--repo", + "scbrown/shuttle", + "--dir", + temp, + "--pattern", + stem, + "--pattern", + stem + ".sha256", + "--pattern", + stem + ".provenance.json", + ], + capture_output=True, + text=True, + timeout=45, + ) + last_read = "exit=" + str(result.returncode) + if result.returncode == 0: + return verify(folder, tag, revision) + except subprocess.TimeoutExpired: + last_read = "read timeout" + if number + 1 < attempts: + pause(30) + raise ValueError("repository pendant unverified after bounded trusted-producer wait; " + last_read) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit("usage: verify-repository-share.py ") + print(json.dumps(wait(sys.argv[1], sys.argv[2]))) From 797a058bb00ad170045e33fc2603ba4f700cc3b4 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Wed, 21 Oct 2026 19:45:02 -0400 Subject: [PATCH 2/3] fix(pendant): carry native alignments through bounded publication --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 20 ++++ docs/knowledge/README.md | 9 +- docs/knowledge/repository-knowledge.json | 12 +-- scripts/build-repository-share.py | 98 ++++++++++++++++---- scripts/publish_repository_share.py | 31 ++++++- scripts/test-repository-knowledge.py | 98 +++++++++++++++++++- scripts/test-repository-share-publication.py | 84 +++++++++++++++++ scripts/test_release_workflow.py | 28 ++++++ scripts/verify-repository-share.py | 7 +- 10 files changed, 353 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ee7a63..e0e573b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: python-version: "3.11" - - run: python -m pip install 'rdflib==7.1.4' + - run: python -m pip install 'rdflib==7.6.0' - run: | python scripts/test-repository-knowledge.py python scripts/test-repository-share-publication.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5322385..8075737 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,6 +116,7 @@ jobs: repository-pendant: name: Verify repository pendant + if: vars.SHUTTLE_PENDANT_PUBLICATION == 'enabled' needs: release runs-on: ubuntu-latest timeout-minutes: 45 @@ -131,3 +132,22 @@ jobs: fetch-depth: 0 - name: Verify the bounded public pointer corpus run: python3 scripts/verify-repository-share.py "$RELEASE_TAG" "$(git rev-parse HEAD)" + + repository-pendant-admission: + name: Record repository pendant admission + needs: release + runs-on: ubuntu-latest + permissions: + contents: read + env: + PENDANT_PUBLICATION: ${{ vars.SHUTTLE_PENDANT_PUBLICATION }} + steps: + - name: Record the publication decision + run: | + set -euo pipefail + if [ "$PENDANT_PUBLICATION" = enabled ]; then + state="ADMITTED: pendant verification remains required" + else + state="NOT ADMITTED: no trusted Shuttle pendant publisher is enabled" + fi + printf '%s\n' "$state" | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/docs/knowledge/README.md b/docs/knowledge/README.md index a7ac3df..290b620 100644 --- a/docs/knowledge/README.md +++ b/docs/knowledge/README.md @@ -9,6 +9,9 @@ shapes travel with the share. `alignments.ttl` is currently a valid empty SSSOM/RDF carrier. Its accompanying status file explicitly records zero approved mappings and seals those bytes. +An engine advertising `share --alignments` carries this reviewed file physically, +with its exact manifest filename and SHA-256 binding. Older engines retain the +inline empty carrier without adding an undeclared archive member. This is not evidence of global absence. Missing, changed or malformed input refuses; an explicit empty carrier does not. Dependencies and glossary references are not identity alignments. Code/commit deduplication and unrelated exactMatch @@ -21,8 +24,10 @@ record may produce derived identity through the engine's existing apply path. The current source profile deliberately does not import arbitrary nonempty mapping data until an actual curated artifact and its transport are reviewed. -The copied release workflow verifies the three public assets only after the -ordinary release finishes. The managed trusted host runs this repository's own +The copied release workflow verifies the three public assets after the ordinary +release finishes only when `SHUTTLE_PENDANT_PUBLICATION` is exactly `enabled`. +A separate job records the admission state; `NOT ADMITTED` does not claim +verified assets or enable a producer. The managed trusted host runs this repository's own publisher, keeps its publication policy private, pins both native engine executables and the indexer, and refuses retained uncertain upload outcomes. Default configuration still publishes only the existing Quipu corpus. Enabling diff --git a/docs/knowledge/repository-knowledge.json b/docs/knowledge/repository-knowledge.json index 2d67fe9..c98ac7b 100644 --- a/docs/knowledge/repository-knowledge.json +++ b/docs/knowledge/repository-knowledge.json @@ -1,6 +1,6 @@ { - "source_revision": "67ec0d108ce9d51207fdff1ce10b1d1241cad82a", - "reviewed_at": "2026-10-21T00:05:39.939993Z", + "source_revision": "33001f173986a0496b3fc9f0afe4ebc658e2618f", + "reviewed_at": "2026-10-21T23:38:05.033622Z", "max_age_seconds": 2592000, "records": [ { @@ -8,7 +8,7 @@ "source": "README.md", "subject": "http://aegis.gastown.local/ontology/doc/shuttle/README.md", "summary": "Shuttle records workflow transitions in a local append-only JSONL log and exports signed history into governed, time-windowed Quipu graphs.", - "source_sha256": "d5ae89b2dda09cd6a7d292b34f4335737088845372924630a1c510d8b455086d", + "source_sha256": "af53641f5f1d6b5a191f477cd40742cfb5ef5e8d2a37ffe4dc2af34563d3d82f", "type": "Concept" }, { @@ -66,7 +66,7 @@ "publicName": "Quipu client" }, "depends_on": [ - "https://github.com/scbrown/quipu" + "http://aegis.gastown.local/ontology/repo_quipu" ] }, { @@ -80,7 +80,7 @@ "publicName": "Seed-step reconciliation" }, "depends_on": [ - "https://github.com/scbrown/seeds" + "http://aegis.gastown.local/ontology/repo_seeds" ] }, { @@ -100,7 +100,7 @@ "source": "README.md", "subject": "http://aegis.gastown.local/ontology/doc/shuttle/README.md", "summary": "Begin with the workflow quick start and role in the stack; registration and production writes remain explicit operator actions.", - "source_sha256": "d5ae89b2dda09cd6a7d292b34f4335737088845372924630a1c510d8b455086d", + "source_sha256": "af53641f5f1d6b5a191f477cd40742cfb5ef5e8d2a37ffe4dc2af34563d3d82f", "properties": { "startHereOrder": 1, "glossaryTerm": "https://scbrown.github.io/quechua/ns#Decision" diff --git a/scripts/build-repository-share.py b/scripts/build-repository-share.py index 19da8ab..e9e8794 100644 --- a/scripts/build-repository-share.py +++ b/scripts/build-repository-share.py @@ -15,25 +15,47 @@ import io import tarfile import subprocess +import sys -def command(args, *, env=None, cwd=None): - return subprocess.run([str(v) for v in args], check=True, capture_output=True, text=True, env=env, cwd=cwd).stdout - - -def index_environment(private): - """Keep user-global endpoints and credentials outside private indexing.""" - env = {k: v for k, v in os.environ.items() - if k not in ("BOBBIN_SERVER", "BOBBIN_QUIPU_REMOTE") and not k.startswith("QUIPU_")} - for key, name in (("HOME", "index-home"), ("XDG_CONFIG_HOME", "index-config")): - folder = Path(private) / name - folder.mkdir(mode=0o700) - env[key] = str(folder) - # Models are public cached artifacts, not host configuration. Preserve an - # explicitly configured cache; otherwise resolve its original default. - env.setdefault("XDG_CACHE_HOME", str(Path.home() / ".cache")) +def isolated_environment(private): + """Native workers must not inherit a host endpoint or credential catalogue.""" + private = Path(private) + env = dict(os.environ) + for key in list(env): + if key in { + "BOBBIN_SERVER", + "QUIPU_SERVER", + "QUIPU_DB", + "QUIPU_AUTH_TOKEN", + "QUIPU_AUTH_TOKEN_FILE", + "GH_TOKEN", + "GITHUB_TOKEN", + } or key.startswith("BOBBIN_QUIPU_"): + env.pop(key) + for key, name in ( + ("HOME", "native-home"), + ("XDG_CONFIG_HOME", "native-config"), + ("XDG_DATA_HOME", "native-data"), + ("XDG_CACHE_HOME", "native-cache"), + ): + directory = private / name + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + env[key] = str(directory) return env +def command(args, *, env=None, cwd=None): + if env is None and cwd is not None: + env = isolated_environment(cwd) + return subprocess.run( + [str(v) for v in args], + check=True, + capture_output=True, + text=True, + env=env, + cwd=cwd, + ).stdout + def local_mcp(binary, database, calls): """Short-lived native process over a private DB; no HTTP or live graph.""" @@ -64,6 +86,7 @@ def local_mcp(binary, database, calls): stderr=subprocess.PIPE, text=True, cwd=Path(database).parent, + env=isolated_environment(Path(database).parent), ) as process: selector = selectors.DefaultSelector() selector.register(process.stdout, selectors.EVENT_READ) @@ -187,6 +210,34 @@ def alignment_carrier(source): return data.decode() +def native_alignments(binary, private): + """Engines that advertise --alignments carry the reviewed physical file. + + Older engines keep the inline empty carrier and never emit an undeclared + archive member that an older receiver would refuse. + """ + usage = subprocess.run( + [str(binary), "share", "--help"], + capture_output=True, + text=True, + env=isolated_environment(private), + cwd=private, + ) + return "--alignments " in usage.stdout + usage.stderr + +def verify_carried_alignments(stage, alignments): + data = Path(alignments).read_bytes() + carried = json.loads((Path(stage) / "manifest.json").read_text()) + copied = Path(stage) / "alignments.ttl" + if ( + carried.get("alignments_hash") != "sha256:" + hashlib.sha256(data).hexdigest() + or carried.get("files", {}).get("alignments") != "alignments.ttl" + or not copied.is_file() + or copied.read_bytes() != data + ): + raise ValueError("native share did not carry the reviewed alignments") + + def build(revision, source, tooling, binary, bobbin, output): repository = "scbrown/shuttle" source, tooling, output = map(Path, (source, tooling, output)) @@ -212,7 +263,7 @@ def run(args, *, env=None): return command(args, env=env, cwd=private) policy = private / "policy.ttl" - run(["python3", tooling / "scripts/private-share-policy.py", policy]) + run(["python3", tooling / "scripts/private-share-policy.py", policy], env=dict(os.environ)) corpus = materialize_release(source, revision, private / "source") (corpus / ".bobbin").mkdir(exist_ok=True) index_database = private / "indexed-graph.db" @@ -239,7 +290,7 @@ def run(args, *, env=None): model = "all-MiniLM-L6-v2" batch_size = 64 """) - env = index_environment(private) + env = isolated_environment(private) run([bobbin, "index", index, "--source", corpus, "--repo", slug, "--force", "--skip-calibrate", "--json"], env=env) database = private / "producer.db" if not index_database.is_file(): @@ -282,7 +333,12 @@ def run(args, *, env=None): "repository_knowledge", source / "scripts/build-repository-knowledge.py" ) generator = importlib.util.module_from_spec(spec) - spec.loader.exec_module(generator) + _previous_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + spec.loader.exec_module(generator) + finally: + sys.dont_write_bytecode = _previous_bytecode records = json.loads(registry_path.read_text())["records"] labels = indexed_labels(binary, database, sorted({record["subject"] for record in records})) knowledge.write_text( @@ -294,7 +350,11 @@ def run(args, *, env=None): register_queries(binary, database, json.loads(queries.read_text())) query = (source / "queries/repository-share-shuttle.rq").read_text() stage = private / "share" - run([binary, "share", "--construct", query, "--turtle", "--output", stage, "--db", database]) + native = native_alignments(binary, private) + run([binary, "share", "--construct", query, "--turtle", "--output", stage] + + (["--alignments", alignments] if native else []) + ["--db", database]) + if native: + verify_carried_alignments(stage, alignments) if not (stage / "shapes.ttl").is_file() or (stage / "shapes.ttl").stat().st_size == 0: raise ValueError("repository share omitted receiver shapes") receiver = private / "receiver.db" diff --git a/scripts/publish_repository_share.py b/scripts/publish_repository_share.py index 29eee46..e55ed26 100644 --- a/scripts/publish_repository_share.py +++ b/scripts/publish_repository_share.py @@ -15,7 +15,7 @@ import tempfile REPOSITORY = "scbrown/shuttle" -MEMBERS = {"export.nt", "export.ttl", "payload.nq", "manifest.json", "manifest.ttl", "shapes.ttl", "queries.ttl"} +MEMBERS = {"export.nt", "export.ttl", "payload.nq", "manifest.json", "manifest.ttl", "shapes.ttl", "queries.ttl", "alignments.ttl"} def command(args): @@ -30,6 +30,24 @@ def digest(path): return result.hexdigest() +def check_alignment_member(stage, manifest): + """Accept a physical optional carrier only with its exact manifest binding.""" + files = manifest.get("files", {}) + if not isinstance(files, dict): + raise ValueError("invalid alignment member declaration") + member = stage / "alignments.ttl" + claimed = manifest.get("alignments_hash") + if member.exists() or claimed is not None or "alignments" in files: + if ( + files.get("alignments") != "alignments.ttl" + or not isinstance(claimed, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", claimed) is None + or not member.is_file() + or claimed != "sha256:" + digest(member) + ): + raise ValueError("missing, undeclared or corrupt alignment member") + + def check_share(stage): files = list(stage.iterdir()) if any(not p.is_file() or p.is_symlink() or p.name not in MEMBERS for p in files): @@ -37,7 +55,9 @@ def check_share(stage): for name in ("export.nt", "manifest.json", "manifest.ttl", "shapes.ttl", "queries.ttl"): if not (stage / name).is_file() or not (stage / name).stat().st_size: raise ValueError("required nonempty share member missing: " + name) - recipe = json.loads((stage / "manifest.json").read_text()).get("embedding") + manifest = json.loads((stage / "manifest.json").read_text()) + check_alignment_member(stage, manifest) + recipe = manifest.get("embedding") if ( not isinstance(recipe, dict) or not isinstance(recipe.get("model"), str) @@ -121,7 +141,12 @@ def publish(tag, revision, output): "bobbin_share_build", source / "scripts/build-repository-share.py" ) builder = importlib.util.module_from_spec(spec) - spec.loader.exec_module(builder) + _previous_bytecode = sys.dont_write_bytecode + try: + sys.dont_write_bytecode = True + spec.loader.exec_module(builder) + finally: + sys.dont_write_bytecode = _previous_bytecode builder.build(revision, source, tooling, binary, bobbin, stage) archive_name = "shuttle-" + tag + "-repository.pendant.tar.gz" prepared = private / "assets" diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index 49e49c1..b1bca38 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -10,8 +10,10 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location("knowledge", ROOT / "scripts/build-repository-knowledge.py") @@ -63,12 +65,12 @@ def test_private_configuration_and_credentials_do_not_follow_parent(self): with tempfile.TemporaryDirectory() as folder: with patch.dict(os.environ, {"HOME": "/synthetic-parent", "XDG_CONFIG_HOME": "/synthetic-global", "BOBBIN_SERVER": "https://invalid.example", "QUIPU_AUTH_TOKEN": "synthetic-only"}): - env = builder.index_environment(Path(folder)) - self.assertEqual(env["HOME"], str(Path(folder) / "index-home")) - self.assertEqual(env["XDG_CONFIG_HOME"], str(Path(folder) / "index-config")) + env = builder.isolated_environment(Path(folder)) + self.assertEqual(env["HOME"], str(Path(folder) / "native-home")) + self.assertEqual(env["XDG_CONFIG_HOME"], str(Path(folder) / "native-config")) self.assertNotIn("BOBBIN_SERVER", env) self.assertNotIn("QUIPU_AUTH_TOKEN", env) - self.assertEqual(list((Path(folder) / "index-config").iterdir()), []) + self.assertEqual(list((Path(folder) / "native-config").iterdir()), []) class AlignmentCarrier(unittest.TestCase): @@ -102,6 +104,94 @@ def test_explicit_empty_positive_missing_corrupt_and_nonempty_negative(self): builder.alignment_carrier(root) + def test_native_alignment_transport_is_advertised_and_hash_bound(self): + import hashlib + + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + private = root / "private" + private.mkdir() + for name, usage in ( + ("current", "[--queries ]... [--alignments ] [--turtle]"), + ("older", "[--queries ]... [--turtle]"), + ): + engine = root / name + # Real engines print share usage to stderr and exit nonzero. + engine.write_text(f"#!/bin/sh\necho 'usage: quipu share {usage}' >&2\nexit 1\n") + engine.chmod(0o755) + self.assertTrue(builder.native_alignments(root / "current", private)) + self.assertFalse(builder.native_alignments(root / "older", private)) + reviewed = ROOT / "docs/knowledge/alignments.ttl" + stage = root / "share" + stage.mkdir() + shutil.copyfile(reviewed, stage / "alignments.ttl") + manifest = { + "alignments_hash": "sha256:" + + hashlib.sha256(reviewed.read_bytes()).hexdigest(), + "files": {"alignments": "alignments.ttl"}, + } + (stage / "manifest.json").write_text(json.dumps(manifest)) + builder.verify_carried_alignments(stage, reviewed) + for corrupt in ( + {**manifest, "alignments_hash": manifest["alignments_hash"][7:]}, + {**manifest, "files": {}}, + ): + (stage / "manifest.json").write_text(json.dumps(corrupt)) + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + (stage / "manifest.json").write_text(json.dumps(manifest)) + with (stage / "alignments.ttl").open("a") as copied: + copied.write(" .\n") + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + (stage / "alignments.ttl").unlink() + with self.assertRaisesRegex(ValueError, "reviewed alignments"): + builder.verify_carried_alignments(stage, reviewed) + + +class NativeIsolation(unittest.TestCase): + def test_real_subprocess_cannot_see_synthetic_global_config_or_endpoints(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + host, private = root / "host", root / "private" + private.mkdir() + config = host / ".config/bobbin" + config.mkdir(parents=True) + (config / "config.toml").write_text('[server]\nurl="http://127.0.0.1:9"\n') + self.assertTrue((config / "config.toml").exists()) + code = ( + "import os,json;from pathlib import Path;" + 'visible=(Path(os.environ["HOME"])/".config/bobbin/config.toml").exists();' + "remote=any(k in os.environ for k in " + '("BOBBIN_SERVER","BOBBIN_QUIPU_REMOTE","QUIPU_SERVER","QUIPU_AUTH_TOKEN","GH_TOKEN"));' + 'print(json.dumps({"global_config_visible":visible,"remote_or_upload_env_visible":remote}));' + "raise SystemExit(3 if visible or remote else 0)" + ) + environment = { + "HOME": str(host), + "XDG_CONFIG_HOME": str(host / ".config"), + "BOBBIN_SERVER": "http://127.0.0.1:9", + "BOBBIN_QUIPU_REMOTE": "http://127.0.0.1:9", + "QUIPU_SERVER": "http://127.0.0.1:9", + "QUIPU_AUTH_TOKEN": "synthetic-only", + "GH_TOKEN": "synthetic-only", + } + with patch.dict(os.environ, environment): + output = builder.command([sys.executable, "-c", code], cwd=private) + policy_code = code.replace( + "3 if visible or remote else 0", "0 if visible and remote else 3" + ) + policy = builder.command( + [sys.executable, "-c", policy_code], + env=dict(os.environ), + cwd=private, + ) + self.assertIn('"global_config_visible": false', output) + self.assertIn('"remote_or_upload_env_visible": false', output) + self.assertIn('"global_config_visible": true', policy) + self.assertIn('"remote_or_upload_env_visible": true', policy) + + class Knowledge(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() diff --git a/scripts/test-repository-share-publication.py b/scripts/test-repository-share-publication.py index 24fc534..c02acda 100644 --- a/scripts/test-repository-share-publication.py +++ b/scripts/test-repository-share-publication.py @@ -6,6 +6,10 @@ import importlib.util import json +import os +import shutil +import subprocess +import sys from pathlib import Path import tempfile import unittest @@ -119,6 +123,86 @@ def test_unreadable_hold_fails_closed(self): with self.assertRaisesRegex(ValueError, "cannot inspect publication hold"): publisher.held() + def test_optional_alignment_member_is_declared_and_hash_bound(self): + import hashlib + data = b"@prefix sssom: .\n" + member = self.stage / "alignments.ttl" + member.write_bytes(data) + manifest = json.loads((self.stage / "manifest.json").read_text()) + bound = {**manifest, "alignments_hash": "sha256:" + hashlib.sha256(data).hexdigest(), + "files": {"alignments": "alignments.ttl"}} + (self.stage / "manifest.json").write_text(json.dumps(bound)) + self.prepare() + tag = self.proof["release_tag"] + self.assertEqual(verifier.verify(self.root, tag, self.revision)["source_revision"], self.revision) + for invalid in ( + manifest, + {**bound, "files": {}}, + {**bound, "alignments_hash": bound["alignments_hash"][7:]}, + {**bound, "files": {"alignments": "other.ttl"}}, + ): + (self.stage / "manifest.json").write_text(json.dumps(invalid)) + with self.assertRaisesRegex(ValueError, "alignment member"): + publisher.check_share(self.stage) + (self.stage / "manifest.json").write_text(json.dumps(bound)) + member.write_bytes(data + b"# tampered\n") + with self.assertRaisesRegex(ValueError, "alignment member"): + publisher.check_share(self.stage) + member.unlink() + with self.assertRaisesRegex(ValueError, "alignment member"): + publisher.check_share(self.stage) + (self.stage / "manifest.json").write_text(json.dumps(manifest)) + publisher.check_share(self.stage) + + def test_readonly_verifier_keeps_nonpython_checkout_clean(self): + source = self.root / "clean-source" + (source / "scripts").mkdir(parents=True) + for name in ( + "verify-repository-share.py", + "publish_repository_share.py", + "build-repository-share.py", + "build-repository-knowledge.py", + ): + shutil.copyfile(ROOT / name, source / "scripts" / name) + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(source), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.org", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + environment = dict(os.environ) + environment.pop("PYTHONDONTWRITEBYTECODE", None) + result = subprocess.run( + [ + sys.executable, + str(source / "scripts/verify-repository-share.py"), + "not-a-release", + "a" * 40, + ], + cwd=source, + env=environment, + capture_output=True, + text=True, + timeout=10, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid release identity", result.stderr) + status = subprocess.check_output( + ["git", "-C", str(source), "status", "--porcelain"], text=True + ) + self.assertEqual(status, "") + def test_wait_is_bounded(self): calls, pauses = [], [] diff --git a/scripts/test_release_workflow.py b/scripts/test_release_workflow.py index ae2afe3..bd6525e 100644 --- a/scripts/test_release_workflow.py +++ b/scripts/test_release_workflow.py @@ -120,5 +120,33 @@ def test_draft_publication_and_failures(self): self.assertTrue(all("--clobber" not in call for call in calls)) +class PendantAdmissionTests(unittest.TestCase): + def test_verification_requires_explicit_publisher_admission(self): + workflow = yaml.load((ROOT / ".github/workflows/release.yml").read_text(), Loader=yaml.BaseLoader) + verify = workflow["jobs"]["repository-pendant"] + self.assertEqual(verify["if"], "vars.SHUTTLE_PENDANT_PUBLICATION == 'enabled'") + self.assertEqual(verify["needs"], "release") + self.assertEqual(verify["permissions"], {"contents": "read"}) + self.assertIn("scripts/verify-repository-share.py", verify["steps"][-1]["run"]) + self.assertEqual(verify["timeout-minutes"], "45") + + def test_actual_recorded_state_is_truthful_without_publishing(self): + workflow = yaml.load((ROOT / ".github/workflows/release.yml").read_text(), Loader=yaml.BaseLoader) + job = workflow["jobs"]["repository-pendant-admission"] + self.assertEqual(job["env"]["PENDANT_PUBLICATION"], "${{ vars.SHUTTLE_PENDANT_PUBLICATION }}") + self.assertEqual(job["needs"], "release") + for value in ("", "disabled", "enabled", "true", "ENABLED", " enabled"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as scratch: + summary = Path(scratch) / "summary" + result = subprocess.run(["/bin/bash", "-c", job["steps"][0]["run"]], + env={"PATH": "/usr/bin:/bin", "PENDANT_PUBLICATION": value, + "GITHUB_STEP_SUMMARY": str(summary)}, + capture_output=True, text=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + expected = "ADMITTED:" if value == "enabled" else "NOT ADMITTED:" + self.assertTrue(result.stdout.startswith(expected), result.stdout) + self.assertEqual(summary.read_text(), result.stdout) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/verify-repository-share.py b/scripts/verify-repository-share.py index 31e8d76..41bcc4b 100644 --- a/scripts/verify-repository-share.py +++ b/scripts/verify-repository-share.py @@ -14,7 +14,12 @@ spec = importlib.util.spec_from_file_location("publisher", Path(__file__).with_name("publish_repository_share.py")) publisher = importlib.util.module_from_spec(spec) -spec.loader.exec_module(publisher) +_previous_bytecode = sys.dont_write_bytecode +try: + sys.dont_write_bytecode = True + spec.loader.exec_module(publisher) +finally: + sys.dont_write_bytecode = _previous_bytecode def verify(folder, tag, revision): From 358a2bd9b1ac99ca51ad8c2f8c99c02b4d0425e0 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Wed, 21 Oct 2026 20:54:42 -0400 Subject: [PATCH 3/3] fix: require repository metadata by knowledge kind --- .../knowledge/repository-knowledge.shapes.ttl | 32 ++++++++++----- scripts/build-repository-knowledge.py | 12 +++++- scripts/test-repository-knowledge.py | 40 +++++++++++++++++++ 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/docs/knowledge/repository-knowledge.shapes.ttl b/docs/knowledge/repository-knowledge.shapes.ttl index 4c81ee9..e0fd2bd 100644 --- a/docs/knowledge/repository-knowledge.shapes.ttl +++ b/docs/knowledge/repository-knowledge.shapes.ttl @@ -21,20 +21,30 @@ a:ShuttleKnowledgeProfile a sh:NodeShape ; sh:property [ sh:path a:maxAge ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:integer ; sh:minExclusive 0 ; ] ; sh:property [ sh:path prov:wasDerivedFrom ; sh:minCount 1 ; sh:nodeKind sh:IRI ; ] . +# Select on the stable record discriminator, never on a field whose absence +# this profile must detect. The first branch admits the other known kinds; +# the second enforces this kind's metadata, even when its selector is deleted. a:ShuttleInterfaceProfile a sh:NodeShape ; - sh:targetSubjectsOf a:interfaceKind ; - sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; - sh:property [ sh:path a:interfaceKind ; sh:minCount 1 ; sh:maxCount 1 ; - sh:in ( "CLI" "MCP" "REST" "config" "file-format" "library" ) ; ] ; - sh:property [ sh:path a:publicName ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; - sh:property [ sh:path a:depends_on ; sh:nodeKind sh:IRI ; ] . + sh:targetSubjectsOf a:knowledgeKind ; + sh:or ( + [ sh:property [ sh:path a:knowledgeKind ; sh:minCount 1 ; + sh:in ( "idea" "decision" "start-here" ) ; ] ] + [ sh:property [ sh:path a:interfaceKind ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "CLI" "MCP" "REST" "config" "file-format" "library" ) ; ] ; + sh:property [ sh:path a:publicName ; sh:minCount 1 ; sh:datatype xsd:string ; ] ; + sh:property [ sh:path a:depends_on ; sh:nodeKind sh:IRI ; ] ] + ) . a:ShuttleDecisionProfile a sh:NodeShape ; - sh:targetSubjectsOf a:supersessionStatus ; - sh:property [ sh:path rdfs:label ; sh:minCount 1 ; ] ; - sh:property [ sh:path a:status ; sh:minCount 1 ; sh:maxCount 1 ; sh:in ( "active" "superseded" "proposed" ) ; ] ; - sh:property [ sh:path a:supersessionStatus ; sh:minCount 1 ; sh:maxCount 1 ; - sh:in ( "not-recorded-in-source" "recorded" ) ; ] . + sh:targetSubjectsOf a:knowledgeKind ; + sh:or ( + [ sh:property [ sh:path a:knowledgeKind ; sh:minCount 1 ; + sh:in ( "idea" "interface" "start-here" ) ; ] ] + [ sh:property [ sh:path a:status ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "active" "superseded" "proposed" ) ; ] ; + sh:property [ sh:path a:supersessionStatus ; sh:minCount 1 ; sh:maxCount 1 ; + sh:in ( "not-recorded-in-source" "recorded" ) ; ] ] + ) . # Existing SSSOM/RDF is a record, not a second identity-assertion format. The # native align decide/apply path remains the only way to derive identity knots. diff --git a/scripts/build-repository-knowledge.py b/scripts/build-repository-knowledge.py index 1ebacf6..4dd72cd 100644 --- a/scripts/build-repository-knowledge.py +++ b/scripts/build-repository-knowledge.py @@ -90,7 +90,17 @@ def fact(subject, predicate, obj, iri=False, datatype=None): "https://github.com/scbrown/shuttle/blob/" + revision + "/" + path, iri=True, ) - for key, value in record.get("properties", {}).items(): + properties = record.get("properties", {}) + required = { + "interface": {"interfaceKind", "publicName"}, + "decision": {"status", "supersessionStatus"}, + }.get(record["kind"], set()) + if not isinstance(properties, dict) or any( + not isinstance(properties.get(key), str) or not properties[key].strip() + for key in required + ): + raise ValueError("required metadata missing for " + record["kind"]) + for key, value in properties.items(): if key not in PROPERTIES: raise ValueError("unknown content property") if key == "startHereOrder": diff --git a/scripts/test-repository-knowledge.py b/scripts/test-repository-knowledge.py index b1bca38..1b27c7e 100644 --- a/scripts/test-repository-knowledge.py +++ b/scripts/test-repository-knowledge.py @@ -245,6 +245,46 @@ def test_missing_required_content_refuses(self): with self.assertRaisesRegex(ValueError, "inventory incomplete"): self.render() + def test_kind_metadata_deletions_refuse_in_renderer(self): + for kind, fields in (("interface", ("interfaceKind", "publicName")), + ("decision", ("status", "supersessionStatus"))): + for deleted in ((fields[0],), (fields[1],), fields): + with self.subTest(kind=kind, deleted=deleted): + self.registry = json.loads((ROOT / "docs/knowledge/repository-knowledge.json").read_text()) + for record in self.registry["records"]: + if record["kind"] == kind: + for key in deleted: + record["properties"].pop(key) + self.save() + with self.assertRaisesRegex(ValueError, "required metadata"): + self.render() + + @unittest.skipUnless(os.environ.get("QUIPU_TEST_BIN"), "native schema control requires explicit test artifact") + def test_native_schema_kind_metadata_deletions_refuse(self): + import rdflib + a = rdflib.Namespace(module.A) + valid = self.render() + data = self.root / "kind-metadata.ttl" + args = [os.environ["QUIPU_TEST_BIN"], "validate", "--shapes", + str(ROOT / "docs/knowledge/repository-knowledge.shapes.ttl"), + "--data", str(data), "--format", "json"] + data.write_text(valid) + positive = subprocess.run(args, capture_output=True, text=True, cwd=self.root) + self.assertEqual(positive.returncode, 0, positive.stderr + positive.stdout) + for kind, fields in (("interface", ("interfaceKind", "publicName")), + ("decision", ("status", "supersessionStatus"))): + for deleted in ((fields[0],), (fields[1],), fields): + with self.subTest(kind=kind, deleted=deleted): + graph = rdflib.Graph().parse(data=valid, format="turtle") + subjects = set(graph.subjects(a.knowledgeKind, rdflib.Literal(kind))) + self.assertTrue(subjects) + for subject in subjects: + for key in deleted: + graph.remove((subject, a[key], None)) + data.write_text(graph.serialize(format="turtle")) + result = subprocess.run(args, capture_output=True, text=True, cwd=self.root) + self.assertNotEqual(result.returncode, 0, result.stderr + result.stdout) + def test_bad_freshness_and_private_summary_refuse(self): self.registry["max_age_seconds"] = 0 self.save()