From fda3c0492ef65a6ccb1066ac768d78d811ddd7c7 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:42:07 -0400 Subject: [PATCH 1/4] feat: install checksummed wheel in an isolated environment --- .github/workflows/install.yml | 19 +++ README.md | 183 +++++++----------------- docs/installation-reference.md | 150 ++++++++++++++++++++ install.sh | 242 ++++++++++++++++++++++++++++++++ justfile | 5 + scripts/test-install-release.py | 146 +++++++++++++++++++ 6 files changed, 612 insertions(+), 133 deletions(-) create mode 100644 .github/workflows/install.yml create mode 100644 docs/installation-reference.md create mode 100755 install.sh create mode 100644 scripts/test-install-release.py diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml new file mode 100644 index 0000000..061eb67 --- /dev/null +++ b/.github/workflows/install.yml @@ -0,0 +1,19 @@ +name: Installer controls +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + installer: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Validate standalone entrypoint and refusal controls + run: | + sh -n install.sh + python3 scripts/test-install-release.py diff --git a/README.md b/README.md index 06cafe6..3f6c25d 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,64 @@ -

- Shuttle logo — a weaving shuttle carrying a red weft thread through vertical warp threads, with a small knot seal below -

- -

shuttle

- -

- 🧵 The workflow engine of the quipu stack — every transition signed, every window frozen -

- -

- License: MIT - Part of the quipu stack -

- -> *A shuttle carries the weft through the warp. Runs move through their -> workflow's declared steps; each pass leaves a signed knot in the record.* - -Workflow engines usually mean a daemon, a queue, a server, and a dashboard -that disagrees with reality. Shuttle is none of those. A **local JSONL log is -the hot path**; [quipu](https://github.com/scbrown/quipu) is the governed -record. Runs move through their workflow's declared steps, every transition is -**signed by the agent that performed it** (per-agent ed25519), and the -append-only history is exported into quipu's time-windowed operational graphs — -where completed windows are **deep-frozen** whole into read-only archives that -stay queryable forever. - -Consumers — [shantytown](https://github.com/scbrown/shantytown) crews, -[creel](https://github.com/scbrown/creel) tabs, -[NeuralAmplifier](https://github.com/scbrown/NeuralAmplifier) — read the -graph, never shuttle. - -## Role in the stack - -```text - agents / crews (advance runs; each step signed) - │ - ▼ - shuttle (state machine + JSONL outbox) - │ export: signed Turtle, idempotent batches - ▼ - quipu (windowed graphs · deep-frozen archives) - │ - ▼ - shantytown · creel · NeuralAmplifier (read the graph, never shuttle) -``` +# Shuttle -- **[camayoc](https://github.com/scbrown/camayoc)** owns the vocabulary: - the workflow slice (`WorkflowRun`, `TransitionEvent`, …), each term owed to a - competency question in `competency/workflow-and-archive.md`. -- **[caboodle](https://github.com/scbrown/caboodle)** installs and verifies - shuttle as part of the box. +Signed agent workflows with append-only execution records. Steps link back +to work items and export their evidence to Quipu. -## Quick start +## Install -```bash -pip install -e . # Python 3.11+, one dep (cryptography) +```sh +curl -fsSL https://raw.githubusercontent.com/scbrown/shuttle/main/install.sh | sh +``` -shuttle keys init agent-a --workflow triage -# -> prints the public key + registration Turtle; a HUMAN loads it into -# the dataKind=identity graph. Shuttle never registers its own keys. +Python 3.11+ with venv/pip, curl, and a Linux or macOS shell. Downloads the latest checksummed wheel into its own virtual environment; +command links go in `~/.local/bin`. Add that directory to `PATH`. -shuttle define examples/triage.json -shuttle start triage run-1 --agent agent-a -shuttle advance run-1 --step claim --agent agent-a # signed -shuttle advance run-1 --step work --agent agent-a -shuttle advance run-1 --step finish --agent agent-a # terminal -shuttle status +Set `STACK_INSTALL_ROOT` on the receiving shell for another prefix. +Pinned releases, alternatives, configuration and removal are in the +[installation and usage reference](docs/installation-reference.md). -shuttle export # drain the log into quipu windows -shuttle verify run-1 # re-verify signatures FROM the graph -shuttle freeze-window 2026-07 # archive a completed window -``` +## Try it -## The shape +Check the installed workflow engine: -| file | owns | -|---|---| -| `shuttle/model.py` | the pure state machine; events are the truth, state is a fold | -| `shuttle/state.py` | the append-only JSONL outbox + export watermark | -| `shuttle/signing.py` | per-agent ed25519 (0600 host files), `shuttle-transition-v1` canonical message | -| `shuttle/quipu_client.py` | windowed writes ride `/knot`'s STRICT `graph` lane; a SHACL refusal is a refusal, never a silent 200 | -| `shuttle/windows.py` | `{ns}/window/shuttle/runs/{YYYY-MM}` graphs + the `urn:shuttle:dataset:open` dataset | -| `shuttle/export.py` | log → signed Turtle → idempotent `/episode` batches | -| `shuttle/seeds.py` | seed steps: a [seeds](https://github.com/scbrown/seeds) work item performs the step, `reconcile` advances the run when it closes | -| `shuttle/cli.py` | the whole v1 surface | - -Design: [docs/design/shuttle.md](docs/design/shuttle.md). - -## Seed steps - -A transition can be performed by a [seeds](https://github.com/scbrown/seeds) -work item instead of by hand: - -```json -{"step": "review", "from": "open", "to": "reviewed", - "seed": {"title": "Review the change", "labels": ["review"]}, - "on_abandon": "reject"} +```sh +shuttle version ``` -`shuttle reconcile --agent ` (on a timer, or by hand) does two things for -every open run: - -- For each seed step legal from the run's state, it runs `sd create - --workflow-run --step --visit `. seeds derives the id from - that key, so repeating the create returns the same seed. A run that re-enters - the state gets a new visit and a new seed. -- It reads the seed with `sd show --json`. Outcome `done` advances the run - through the step, signed, with the seed id, outcome and close time recorded - as evidence. Any other outcome takes `on_abandon`. With no `on_abandon` the - run is FLAGGED (exit 1) and never advanced silently. - -seeds knows nothing about workflows beyond the link field. A missed tick costs -one tick, not a stuck run. `SHUTTLE_SD` names the `sd` binary; where seeds -stores its work is sd's own configuration. - -## What runs today - -v1 is landed and in production use: signed runs, windowed export, freezable -history, seed steps, and the cross-repo acceptance passing against a live -quipu. On the fleet host it carries the ansible apply lane, repo watch and -stack watch workflows, and an unattended actuator installs each release. - -```bash -just check # parse + file-size ratchet -just test # check + the unit suite -just e2e # the cross-repo acceptance against a live quipu -``` +[Create a workflow](docs/installation-reference.md#quick-start) when you are ready +to register an agent identity. Installation does not enroll a key or run a workflow. -Documentation: the [shuttle book](https://scbrown.github.io/shuttle/). -Changes: [CHANGELOG.md](CHANGELOG.md). +## Next steps -## License +- [Tambo](https://github.com/scbrown/tambo): guided stack installation and verification. +- [Camayoc](https://github.com/scbrown/camayoc): governed memory and vocabulary. +- [Detailed examples and agent setup](docs/installation-reference.md). -[MIT](LICENSE) +**Load this repository’s graph:** public repository-pendant assets have not yet been verified. When a release supplies the archive, checksum and provenance, follow the [verified import workflow](https://scbrown.github.io/quipu/sharing/index.html): extract the share directory and use `quipu import SHARE_DIR --db repository.db`. Import stages knowledge; ROOT stays unchanged. -For pre-registered job identities, put `--require-existing-keys` before the -subcommand (for example, `shuttle --require-existing-keys advance ...`). It -requires an existing Ed25519 key owned by the current Unix user, in a regular -0600 file; missing keys, final-component symlinks and unsafe files are refused. -The policy covers the whole command, including export verification, and never -creates a replacement key. Older installations reject this flag. Default -first-use key initialization remains available for the normal registration -ceremony. This option checks local custody, not remote registration or job -scope admission. SDK callers can use `signing.existing_keys_only()` around an -entire operation, or `sign_transition(..., require_existing_key=True)` for one -signature. Parent directory custody remains the operator's responsibility. +## The stack + + + +The Quipu stack is built around a governed, bitemporal knowledge store: shared +facts, decisions and work history your agents can query. Quipu can check writes +against your rules, records where each fact came from, and answers what was true +when. + +| tool | what it gives your agents | +|---|---| +| [tambo](https://github.com/scbrown/tambo) | 🧰 Agent Platform Toolkit - installs and configures the quipu stack | +| [quipu](https://github.com/scbrown/quipu) | stores shared facts with provenance and history; can check writes against your rules | +| [camayoc](https://github.com/scbrown/camayoc) | prepares knowledge and vocabulary for Quipu, with checks before it is admitted | +| [quechua](https://github.com/scbrown/quechua) | shared vocabulary and shapes for interoperable knowledge | +| [chaski](https://github.com/scbrown/chaski) | reacts to graph changes and delivers governed notifications | +| [bobbin](https://github.com/scbrown/bobbin) | searches your code and can include related knowledge from Quipu | +| [yupana](https://github.com/scbrown/yupana) | shows code impact and can read policies from Quipu to check changes | +| [desire-path](https://github.com/scbrown/desire-path) | turns agent mistakes into feature requests; works on its own without Quipu | +| [seeds](https://github.com/scbrown/seeds) | a work item tracker backed by the Quipu knowledge base | +| [shantytown](https://github.com/scbrown/shantytown) | runs crews of AI agents on work items, triggered by hand, on a schedule or by events; can share work and memory through Quipu | +| [rikuy](https://github.com/scbrown/rikuy) | live brief: status, graphs, watches and signed decisions over Quipu | +| [shuttle](https://github.com/scbrown/shuttle) | runs signed agent workflows and keeps their records in Quipu | + + + +[License](LICENSE). diff --git a/docs/installation-reference.md b/docs/installation-reference.md new file mode 100644 index 0000000..32d639f --- /dev/null +++ b/docs/installation-reference.md @@ -0,0 +1,150 @@ +

+ Shuttle logo — a weaving shuttle carrying a red weft thread through vertical warp threads, with a small knot seal below +

+ +

shuttle

+ +

+ 🧵 The workflow engine of the quipu stack — every transition signed, every window frozen +

+ +

+ License: MIT + Part of the quipu stack +

+ +> *A shuttle carries the weft through the warp. Runs move through their +> workflow's declared steps; each pass leaves a signed knot in the record.* + +Workflow engines usually mean a daemon, a queue, a server, and a dashboard +that disagrees with reality. Shuttle is none of those. A **local JSONL log is +the hot path**; [quipu](https://github.com/scbrown/quipu) is the governed +record. Runs move through their workflow's declared steps, every transition is +**signed by the agent that performed it** (per-agent ed25519), and the +append-only history is exported into quipu's time-windowed operational graphs — +where completed windows are **deep-frozen** whole into read-only archives that +stay queryable forever. + +Consumers — [shantytown](https://github.com/scbrown/shantytown) crews, +[creel](https://github.com/scbrown/creel) tabs, +[NeuralAmplifier](https://github.com/scbrown/NeuralAmplifier) — read the +graph, never shuttle. + +## Role in the stack + +```text + agents / crews (advance runs; each step signed) + │ + ▼ + shuttle (state machine + JSONL outbox) + │ export: signed Turtle, idempotent batches + ▼ + quipu (windowed graphs · deep-frozen archives) + │ + ▼ + shantytown · creel · NeuralAmplifier (read the graph, never shuttle) +``` + +- **[camayoc](https://github.com/scbrown/camayoc)** owns the vocabulary: + the workflow slice (`WorkflowRun`, `TransitionEvent`, …), each term owed to a + competency question in `competency/workflow-and-archive.md`. +- **[caboodle](https://github.com/scbrown/caboodle)** installs and verifies + shuttle as part of the box. + +## Quick start + +```bash +pip install -e . # Python 3.11+, one dep (cryptography) + +shuttle keys init agent-a --workflow triage +# -> prints the public key + registration Turtle; a HUMAN loads it into +# the dataKind=identity graph. Shuttle never registers its own keys. + +shuttle define examples/triage.json +shuttle start triage run-1 --agent agent-a +shuttle advance run-1 --step claim --agent agent-a # signed +shuttle advance run-1 --step work --agent agent-a +shuttle advance run-1 --step finish --agent agent-a # terminal +shuttle status + +shuttle export # drain the log into quipu windows +shuttle verify run-1 # re-verify signatures FROM the graph +shuttle freeze-window 2026-07 # archive a completed window +``` + +## The shape + +| file | owns | +|---|---| +| `shuttle/model.py` | the pure state machine; events are the truth, state is a fold | +| `shuttle/state.py` | the append-only JSONL outbox + export watermark | +| `shuttle/signing.py` | per-agent ed25519 (0600 host files), `shuttle-transition-v1` canonical message | +| `shuttle/quipu_client.py` | windowed writes ride `/knot`'s STRICT `graph` lane; a SHACL refusal is a refusal, never a silent 200 | +| `shuttle/windows.py` | `{ns}/window/shuttle/runs/{YYYY-MM}` graphs + the `urn:shuttle:dataset:open` dataset | +| `shuttle/export.py` | log → signed Turtle → idempotent `/episode` batches | +| `shuttle/seeds.py` | seed steps: a [seeds](https://github.com/scbrown/seeds) work item performs the step, `reconcile` advances the run when it closes | +| `shuttle/cli.py` | the whole v1 surface | + +Design: [docs/design/shuttle.md](https://github.com/scbrown/shuttle/blob/main/docs/design/shuttle.md). + +## Seed steps + +A transition can be performed by a [seeds](https://github.com/scbrown/seeds) +work item instead of by hand: + +```json +{"step": "review", "from": "open", "to": "reviewed", + "seed": {"title": "Review the change", "labels": ["review"]}, + "on_abandon": "reject"} +``` + +`shuttle reconcile --agent ` (on a timer, or by hand) does two things for +every open run: + +- For each seed step legal from the run's state, it runs `sd create + --workflow-run --step --visit `. seeds derives the id from + that key, so repeating the create returns the same seed. A run that re-enters + the state gets a new visit and a new seed. +- It reads the seed with `sd show --json`. Outcome `done` advances the run + through the step, signed, with the seed id, outcome and close time recorded + as evidence. Any other outcome takes `on_abandon`. With no `on_abandon` the + run is FLAGGED (exit 1) and never advanced silently. + +seeds knows nothing about workflows beyond the link field. A missed tick costs +one tick, not a stuck run. `SHUTTLE_SD` names the `sd` binary; where seeds +stores its work is sd's own configuration. + +## What runs today + +v1 is landed and in production use: signed runs, windowed export, freezable +history, seed steps, and the cross-repo acceptance passing against a live +quipu. On the fleet host it carries the ansible apply lane, repo watch and +stack watch workflows, and an unattended actuator installs each release. + +```bash +just check # parse + file-size ratchet +just test # check + the unit suite +just e2e # the cross-repo acceptance against a live quipu +``` + +Documentation: the [shuttle book](https://scbrown.github.io/shuttle/). +Changes: [CHANGELOG.md](https://github.com/scbrown/shuttle/blob/main/CHANGELOG.md). + +## License + +[MIT](https://github.com/scbrown/shuttle/blob/main/LICENSE) + +For pre-registered job identities, put `--require-existing-keys` before the +subcommand (for example, `shuttle --require-existing-keys advance ...`). It +requires an existing Ed25519 key owned by the current Unix user, in a regular +0600 file; missing keys, final-component symlinks and unsafe files are refused. +The policy covers the whole command, including export verification, and never +creates a replacement key. Older installations reject this flag. Default +first-use key initialization remains available for the normal registration +ceremony. This option checks local custody, not remote registration or job +scope admission. SDK callers can use `signing.existing_keys_only()` around an +entire operation, or `sign_transition(..., require_existing_key=True)` for one +signature. Parent directory custody remains the operator's responsibility. + +## The stack + diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..0d5e5f9 --- /dev/null +++ b/install.sh @@ -0,0 +1,242 @@ +#!/bin/sh +# Verify the published package before installing; no service or enrollment action. +set -eu +exec python3 - "$@" <<'PYINSTALL' +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + + + +def fetch(url, destination): + for attempt in range(3): + try: + with urlopen(Request(url, headers={'User-Agent': CONFIG['repo'] + '-installer'}), timeout=60) as response: + with destination.open('wb') as output: + shutil.copyfileobj(response, output) + return + except (HTTPError, URLError, TimeoutError) as error: + transient = not isinstance(error, HTTPError) or error.code in (403, 429, 500, 502, 503, 504) + if attempt == 2 or not transient: + raise + time.sleep(attempt + 1) + + +def extract(archive, destination): + with tarfile.open(archive, 'r:gz') as bundle: + seen, links = set(), [] + total = 0 + for member in bundle: + name = PurePosixPath(member.name) + if name.is_absolute() or '..' in name.parts or str(name) in seen: + raise ValueError('unsafe or duplicate archive path') + seen.add(str(name)) + target = destination.joinpath(*name.parts) + target.parent.mkdir(parents=True, exist_ok=True) + if member.isdir(): + target.mkdir(exist_ok=True) + elif member.isfile(): + total += member.size + if total > 1024 * 1024 * 1024: + raise ValueError('release exceeds 1 GiB unpacked limit') + source = bundle.extractfile(member) + if source is None: + raise ValueError('missing archive bytes') + with source, target.open('xb') as output: + shutil.copyfileobj(source, output) + target.chmod(0o755 if member.mode & 0o111 else 0o644) + elif member.issym(): + link = PurePosixPath(member.linkname) + if link.is_absolute() or '..' in link.parts: + raise ValueError('unsafe archive link') + links.append((target, member.linkname)) + else: + raise ValueError('unsupported archive member') + for target, link in links: + target.symlink_to(link) + target.resolve(strict=True).relative_to(destination.resolve()) + + +CONFIG = {'kind': 'wheel', 'package': 'shuttle', 'assets': ['shuttle-{version}-py3-none-any.whl'], 'checksum': 'checksums.txt', 'commands': ['shuttle'], 'repo': 'shuttle'} + + +def bundle_identity(root): + result = {} + for path in root.rglob('*'): + relative = str(path.relative_to(root)) + if relative == '.release.json': + continue + if path.is_symlink(): + result[relative] = ('link', os.readlink(path)) + elif path.is_file(): + result[relative] = ('file', path.stat().st_mode & 0o777, + hashlib.sha256(path.read_bytes()).hexdigest()) + elif path.is_dir(): + result[relative] = ('directory',) + else: + raise ValueError('unexpected installed bundle entry') + return result + + +def install(): + if len(sys.argv) > 1 or platform.system() not in ('Linux', 'Darwin'): + raise ValueError('use a Linux or macOS shell; configure STACK_INSTALL_ROOT through the environment') + if CONFIG['kind'] == 'wheel' and sys.version_info < (3, 11): + raise ValueError('Python 3.11 or newer is required') + with tempfile.TemporaryDirectory(prefix=CONFIG['repo'] + '-release-') as temporary: + work = Path(temporary) + tag = os.environ.get('STACK_VERSION') + if not tag: + metadata = work / 'latest.json' + fetch('https://api.github.com/repos/scbrown/' + CONFIG['repo'] + '/releases/latest', metadata) + release = json.loads(metadata.read_text()) + if release.get('draft') or release.get('prerelease'): + raise ValueError('latest release is not stable') + tag = release['tag_name'] + if not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag): + raise ValueError('unsupported release tag') + version = tag[1:] + base = os.environ.get('STACK_RELEASE_BASE_URL', 'https://github.com/scbrown/' + CONFIG['repo'] + '/releases/download/' + tag) + manifest = work / 'checksums' + fetch(base + '/' + CONFIG['checksum'], manifest) + names = [name.format(tag=tag, version=version) for name in CONFIG['assets']] + digests = {} + for name in names: + matches = [fields[0].lower() for line in manifest.read_text().splitlines() + if len(fields := line.split()) == 2 and fields[1].lstrip('*') == name] + if len(matches) != 1 or not re.fullmatch('[0-9a-f]{64}', matches[0]): + raise ValueError('missing or ambiguous checksum for ' + name) + fetch(base + '/' + name, work / name) + digest = hashlib.sha256((work / name).read_bytes()).hexdigest() + if digest != matches[0]: + raise ValueError('checksum mismatch: ' + name) + digests[name] = digest + prefix = Path(os.environ.get('STACK_INSTALL_ROOT', str(Path.home() / '.local'))).expanduser().absolute() + releases = prefix / 'share' / CONFIG['repo'] + releases.mkdir(parents=True, exist_ok=True) + lock = releases / '.install-lock' + lock.mkdir() + destination = releases / tag + identity = {'tag': tag, 'assets': digests} + created = False + try: + if destination.is_symlink(): + raise ValueError('release destination is a symlink') + if destination.exists(): + if json.loads((destination / '.release.json').read_text()) != identity: + raise ValueError('existing release identity differs; previous install preserved') + if CONFIG['kind'] == 'source': + check = work / 'source-check' + check.mkdir() + extract(work / names[0], check) + if bundle_identity(destination) != bundle_identity(check / ('camayoc-' + tag)): + raise ValueError('installed source bytes differ') + elif CONFIG['kind'] == 'data': + for name in names: + if hashlib.sha256((destination / name).read_bytes()).hexdigest() != digests[name]: + raise ValueError('installed data bytes differ') + else: + if CONFIG['kind'] == 'wheel': + # The venv is created at its final location so console-script + # shebangs remain valid. No command is published before smoke. + destination.mkdir() + created = True + subprocess.run([sys.executable, '-m', 'venv', str(destination)], check=True, timeout=120) + python = destination / 'bin/python' + subprocess.run([str(python), '-m', 'pip', '--isolated', '--disable-pip-version-check', + 'install', '--no-input', str(work / names[0])], check=True, timeout=300) + actual = subprocess.check_output([str(python), '-c', + 'from importlib.metadata import version; print(version(' + repr(CONFIG['package']) + '))'], text=True, timeout=20).strip() + if actual != version: + raise ValueError('installed package version mismatch') + for command in CONFIG['commands']: + if not os.access(destination / 'bin' / command, os.X_OK): + raise ValueError('missing console entrypoint: ' + command) + elif CONFIG['kind'] == 'source': + unpacked = work / 'source' + unpacked.mkdir() + extract(work / names[0], unpacked) + source = unpacked / ('camayoc-' + tag) + actual = json.loads((source / '.claude-plugin/plugin.json').read_text())['version'] + if actual != version: + raise ValueError('source package version mismatch') + shutil.copytree(source, destination, symlinks=True) + created = True + else: + destination.mkdir() + created = True + for name in names: + data = (work / name).read_bytes() + if not data.strip(): + raise ValueError('empty vocabulary asset') + shutil.copyfile(work / name, destination / name) + (destination / '.release.json').write_text(json.dumps(identity, sort_keys=True) + '\n') + if CONFIG['kind'] == 'wheel': + # Pip's RECORD binds installed package/entrypoint bytes. Validate + # it on both first install and reuse; a stale marker is not proof. + code = """from importlib.metadata import distribution +from pathlib import Path +import base64, hashlib, sys +root = Path(sys.prefix).resolve() +dist = distribution(sys.argv[1]) +assert dist.version == sys.argv[2], 'installed package version mismatch' +for entry in dist.files or []: + if entry.hash is None: + continue + path = Path(dist.locate_file(entry)).resolve() + path.relative_to(root) + assert path.is_file(), 'installed package entry is not a regular file' + actual = base64.urlsafe_b64encode(hashlib.new(entry.hash.mode, path.read_bytes()).digest()).decode().rstrip('=') + assert actual == entry.hash.value, 'installed package file hash mismatch' +""" + subprocess.run([str(destination / 'bin/python'), '-c', code, + CONFIG['package'], version], check=True, timeout=30) + if CONFIG['kind'] == 'wheel': + binaries = prefix / 'bin' + binaries.mkdir(parents=True, exist_ok=True) + for command in CONFIG['commands']: + if (binaries / command).is_dir(): + raise ValueError('command destination is a directory') + for command in CONFIG['commands']: + fd, staged = tempfile.mkstemp(prefix='.' + command + '-', dir=binaries) + os.close(fd) + os.unlink(staged) + try: + os.symlink(destination / 'bin' / command, staged) + os.replace(staged, binaries / command) + finally: + if os.path.lexists(staged): + os.unlink(staged) + print('Installed verified ' + CONFIG['repo'] + ' ' + version + ' at ' + str(destination)) + if CONFIG['kind'] == 'source': + print('Source package only. Bootstrap and agent enrollment are separate explicit actions.') + elif CONFIG['kind'] == 'data': + print('Vocabulary and shapes only. Nothing was loaded into a running graph.') + else: + print('Add ' + str(prefix / 'bin') + ' to PATH. Previous release directories are retained.') + except BaseException: + if created: + shutil.rmtree(destination) + raise + finally: + lock.rmdir() + + +try: + install() +except (OSError, ValueError, KeyError, subprocess.SubprocessError, tarfile.TarError) as error: + print('Install refused: ' + str(error), file=sys.stderr) + sys.exit(1) +PYINSTALL diff --git a/justfile b/justfile index 966fb5e..9c79554 100644 --- a/justfile +++ b/justfile @@ -12,3 +12,8 @@ test: check # The cross-repo acceptance: a live quipu, a full run, a freeze. e2e quipu_bin="quipu": @bash scripts/e2e_slice.sh {{quipu_bin}} + +# Verify the standalone installer using private release fixtures. +installer-check: + sh -n install.sh + python3 scripts/test-install-release.py diff --git a/scripts/test-install-release.py b/scripts/test-install-release.py new file mode 100644 index 0000000..9706fd4 --- /dev/null +++ b/scripts/test-install-release.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Offline package fixtures exercise the real standalone installer.""" +from contextlib import redirect_stderr, redirect_stdout +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from urllib.request import urlopen as real_urlopen +from unittest.mock import patch +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = {'kind': 'wheel', 'package': 'shuttle', 'assets': ['shuttle-{version}-py3-none-any.whl'], 'checksum': 'checksums.txt', 'commands': ['shuttle'], 'repo': 'shuttle'} +TAG = 'v1.2.3' +VERSION = '1.2.3' + + +class PackageInstall(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix='package-install-test-') + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.release = self.root / 'release' + self.release.mkdir() + self.prefix = self.root / 'prefix' + (self.prefix / 'bin').mkdir(parents=True) + self.previous = self.prefix / 'bin' / CONFIG.get('commands', ['prior'])[0] + self.previous.write_text('previous bytes') + self.env = dict(os.environ, HOME=str(self.root / 'home'), STACK_VERSION=TAG, + STACK_INSTALL_ROOT=str(self.prefix), STACK_RELEASE_BASE_URL=self.release.as_uri()) + self.names = [name.format(tag=TAG, version=VERSION) for name in CONFIG['assets']] + self.fixture() + self.latest = {"tag_name": TAG, "draft": False, "prerelease": False} + + def fixture(self): + for name in self.names: + path = self.release / name + if CONFIG['kind'] == 'wheel': + distribution = CONFIG['package'].replace('-', '_') + metadata = distribution + '-' + VERSION + '.dist-info/' + with zipfile.ZipFile(path, 'w') as wheel: + wheel.writestr('fixture_cli.py', 'def main():\n print("fixture ' + VERSION + '")\n') + wheel.writestr(metadata + 'METADATA', 'Metadata-Version: 2.1\nName: ' + CONFIG['package'] + '\nVersion: ' + VERSION + '\n') + wheel.writestr(metadata + 'WHEEL', 'Wheel-Version: 1.0\nGenerator: fixture\nRoot-Is-Purelib: true\nTag: py3-none-any\n') + wheel.writestr(metadata + 'entry_points.txt', '[console_scripts]\n' + '\n'.join(command + ' = fixture_cli:main' for command in CONFIG['commands'])) + wheel.writestr(metadata + 'RECORD', '') + elif CONFIG['kind'] == 'source': + data = json.dumps({'name': 'camayoc', 'version': VERSION}).encode() + with tarfile.open(path, 'w:gz') as bundle: + member = tarfile.TarInfo('camayoc-' + TAG + '/.claude-plugin/plugin.json') + member.size, member.mode = len(data), 0o644 + bundle.addfile(member, io.BytesIO(data)) + else: + path.write_text('@prefix example: .\nexample:Thing a example:Class .\n') + manifest = ''.join(hashlib.sha256((self.release / name).read_bytes()).hexdigest() + ' ' + name + '\n' for name in self.names) + (self.release / CONFIG['checksum']).write_text(manifest) + + def open_release(self, request, **kwargs): + if request.full_url.startswith('https://api.github.com/repos/scbrown/'): + return io.BytesIO(json.dumps(self.latest).encode()) + return real_urlopen(request, **kwargs) + + def run_installer(self, system='Linux'): + script = (ROOT / 'install.sh').read_text().split("<<'PYINSTALL'\n", 1)[1].rsplit('PYINSTALL', 1)[0] + stdout, stderr = io.StringIO(), io.StringIO() + status = 0 + with patch.dict(os.environ, self.env, clear=True), patch('sys.argv', ['install.sh']), patch('urllib.request.urlopen', side_effect=self.open_release), \ + patch('platform.system', return_value=system), redirect_stdout(stdout), redirect_stderr(stderr): + try: + exec(compile(script, str(ROOT / 'install.sh'), 'exec'), {'__name__': '__main__'}) + except SystemExit as error: + status = error.code + return status, stdout.getvalue(), stderr.getvalue() + + def assert_preserved(self, result): + self.assertNotEqual(result[0], 0, result) + self.assertEqual(self.previous.read_text(), 'previous bytes') + self.assertFalse(self.previous.is_symlink()) + self.assertNotIn('Installed verified', result[1]) + + def test_actual_installer_installs_verified_package(self): + result = self.run_installer() + self.assertEqual(result[0], 0, result) + destination = self.prefix / 'share' / CONFIG['repo'] / TAG + self.assertEqual(json.loads((destination / '.release.json').read_text())['tag'], TAG) + if CONFIG['kind'] == 'wheel': + for command in CONFIG['commands']: + self.assertIn(VERSION, subprocess.check_output([str(self.prefix / 'bin' / command)], text=True)) + else: + self.assertEqual(self.previous.read_text(), 'previous bytes') + + def test_latest_release_selection_installs_the_resolved_tag(self): + self.env.pop('STACK_VERSION') + result = self.run_installer() + self.assertEqual(result[0], 0, result) + self.assertIn(TAG, result[1]) + + def test_prerelease_cannot_become_latest_stable_install(self): + self.env.pop('STACK_VERSION') + self.latest['prerelease'] = True + self.assert_preserved(self.run_installer()) + + def test_reinstall_of_intact_release_succeeds(self): + self.assertEqual(self.run_installer()[0], 0) + result = self.run_installer() + self.assertEqual(result[0], 0, result) + + def test_tampered_existing_install_cannot_claim_verified_success(self): + self.assertEqual(self.run_installer()[0], 0) + destination = self.prefix / 'share' / CONFIG['repo'] / TAG + if CONFIG['kind'] == 'wheel': + target = self.previous.resolve() + elif CONFIG['kind'] == 'source': + target = destination / '.claude-plugin/plugin.json' + else: + target = destination / self.names[0] + target.write_text('tampered installed bytes') + result = self.run_installer() + self.assertNotEqual(result[0], 0, result) + self.assertNotIn('Installed verified', result[1]) + self.assertEqual(target.read_text(), 'tampered installed bytes') + + def test_corrupt_checksum_preserves_previous(self): + path = self.release / self.names[0] + path.write_bytes(path.read_bytes() + b'corrupt') + self.assert_preserved(self.run_installer()) + + def test_missing_asset_preserves_previous(self): + (self.release / self.names[0]).unlink() + self.assert_preserved(self.run_installer()) + + def test_unsupported_platform_preserves_previous(self): + self.assert_preserved(self.run_installer(system='UnsupportedOS')) + + def test_missing_checksum_entry_preserves_previous(self): + (self.release / CONFIG['checksum']).write_text('') + self.assert_preserved(self.run_installer()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 3802b343336e8c0d7244fe9e3979ea13272aea4c Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:42:23 -0400 Subject: [PATCH 2/4] style: normalize installation reference ending --- docs/installation-reference.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/installation-reference.md b/docs/installation-reference.md index 32d639f..01b35ab 100644 --- a/docs/installation-reference.md +++ b/docs/installation-reference.md @@ -147,4 +147,3 @@ entire operation, or `sign_transition(..., require_existing_key=True)` for one signature. Parent directory custody remains the operator's responsibility. ## The stack - From b793213bfbc043cb8f7732752202b515eb71f8b7 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Tue, 20 Oct 2026 19:57:24 -0400 Subject: [PATCH 3/4] docs: preserve source links in moved reference --- docs/installation-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation-reference.md b/docs/installation-reference.md index 01b35ab..7a69660 100644 --- a/docs/installation-reference.md +++ b/docs/installation-reference.md @@ -9,7 +9,7 @@

- License: MIT + License: MIT Part of the quipu stack

From e2df981f6921cab1b4d43d2be14c2e8982a07d00 Mon Sep 17 00:00:00 2001 From: Steve Brown Date: Wed, 21 Oct 2026 00:45:30 -0400 Subject: [PATCH 4/4] fix: verify wheel payload and entrypoints on reinstall --- install.sh | 48 +++++++++++++++++++++++---------- scripts/test-install-release.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index 0d5e5f9..224fae5 100755 --- a/install.sh +++ b/install.sh @@ -1,7 +1,7 @@ #!/bin/sh # Verify the published package before installing; no service or enrollment action. set -eu -exec python3 - "$@" <<'PYINSTALL' +exec python3 -I - "$@" <<'PYINSTALL' import hashlib import json import os @@ -153,17 +153,14 @@ def install(): # shebangs remain valid. No command is published before smoke. destination.mkdir() created = True - subprocess.run([sys.executable, '-m', 'venv', str(destination)], check=True, timeout=120) + subprocess.run([sys.executable, '-I', '-m', 'venv', str(destination)], check=True, timeout=120) python = destination / 'bin/python' - subprocess.run([str(python), '-m', 'pip', '--isolated', '--disable-pip-version-check', + subprocess.run([str(python), '-I', '-m', 'pip', '--isolated', '--disable-pip-version-check', 'install', '--no-input', str(work / names[0])], check=True, timeout=300) - actual = subprocess.check_output([str(python), '-c', + actual = subprocess.check_output([str(python), '-I', '-c', 'from importlib.metadata import version; print(version(' + repr(CONFIG['package']) + '))'], text=True, timeout=20).strip() if actual != version: raise ValueError('installed package version mismatch') - for command in CONFIG['commands']: - if not os.access(destination / 'bin' / command, os.X_OK): - raise ValueError('missing console entrypoint: ' + command) elif CONFIG['kind'] == 'source': unpacked = work / 'source' unpacked.mkdir() @@ -187,22 +184,45 @@ def install(): # Pip's RECORD binds installed package/entrypoint bytes. Validate # it on both first install and reuse; a stale marker is not proof. code = """from importlib.metadata import distribution -from pathlib import Path -import base64, hashlib, sys +from pathlib import Path, PurePosixPath +import base64, hashlib, json, os, sys, zipfile root = Path(sys.prefix).resolve() dist = distribution(sys.argv[1]) assert dist.version == sys.argv[2], 'installed package version mismatch' +# RECORD may legitimately omit hashes. Bind every wheel payload file directly +# to the freshly checksum-verified release instead of trusting mutable RECORD. +with zipfile.ZipFile(sys.argv[3]) as wheel: + seen = set() + payload_count = 0 + for member in wheel.infolist(): + name = PurePosixPath(member.filename) + assert not name.is_absolute() and '..' not in name.parts, 'unsafe wheel path' + assert member.filename not in seen, 'duplicate wheel path' + seen.add(member.filename) + assert not any(part.endswith('.data') for part in name.parts), 'unsupported wheel data layout' + if member.is_dir() or member.filename.endswith('.dist-info/RECORD'): + continue + path = Path(dist.locate_file(member.filename)) + assert not path.is_symlink() and path.is_file(), 'missing or linked wheel payload' + path.resolve().relative_to(root) + assert path.read_bytes() == wheel.read(member), 'installed wheel payload differs' + payload_count += 1 + assert payload_count > 0, 'wheel has no payload' for entry in dist.files or []: if entry.hash is None: continue - path = Path(dist.locate_file(entry)).resolve() - path.relative_to(root) - assert path.is_file(), 'installed package entry is not a regular file' + path = Path(dist.locate_file(entry)) + assert not path.is_symlink() and path.is_file(), 'installed package entry is not a regular file' + path.resolve().relative_to(root) actual = base64.urlsafe_b64encode(hashlib.new(entry.hash.mode, path.read_bytes()).digest()).decode().rstrip('=') assert actual == entry.hash.value, 'installed package file hash mismatch' +for command in json.loads(sys.argv[4]): + path = root / 'bin' / command + assert not path.is_symlink() and path.is_file() and os.access(path, os.X_OK), 'missing or nonexecutable console entrypoint' """ - subprocess.run([str(destination / 'bin/python'), '-c', code, - CONFIG['package'], version], check=True, timeout=30) + subprocess.run([str(destination / 'bin/python'), '-I', '-c', code, + CONFIG['package'], version, str(work / names[0]), + json.dumps(CONFIG['commands'])], check=True, timeout=30) if CONFIG['kind'] == 'wheel': binaries = prefix / 'bin' binaries.mkdir(parents=True, exist_ok=True) diff --git a/scripts/test-install-release.py b/scripts/test-install-release.py index 9706fd4..57a66e3 100644 --- a/scripts/test-install-release.py +++ b/scripts/test-install-release.py @@ -125,6 +125,48 @@ def test_tampered_existing_install_cannot_claim_verified_success(self): self.assertNotIn('Installed verified', result[1]) self.assertEqual(target.read_text(), 'tampered installed bytes') + def test_caller_metadata_cannot_shadow_installed_distribution(self): + shadow = self.root / "shadow" + metadata = shadow / (CONFIG["package"].replace("-", "_") + "-0.0.0.dist-info") + metadata.mkdir(parents=True) + (metadata / "METADATA").write_text("Metadata-Version: 2.1\nName: " + CONFIG["package"] + "\nVersion: 0.0.0\n") + previous = Path.cwd() + try: + os.chdir(shadow) + result = self.run_installer() + finally: + os.chdir(previous) + self.assertEqual(result[0], 0, result) + + def test_missing_console_entrypoint_refuses_reuse(self): + self.assertEqual(self.run_installer()[0], 0) + destination = self.prefix / "share" / CONFIG["repo"] / TAG + command = destination / "bin" / CONFIG["commands"][0] + command.unlink() + result = self.run_installer() + self.assertNotEqual(result[0], 0, result) + self.assertNotIn("Installed verified", result[1]) + self.assertFalse(command.exists()) + + def test_unhashed_wheel_payload_tamper_refuses_reuse(self): + self.assertEqual(self.run_installer()[0], 0) + destination = self.prefix / "share" / CONFIG["repo"] / TAG + target = next(destination.rglob("fixture_cli.py")) + target.write_text("def main(): print('tampered')\n") + result = self.run_installer() + self.assertNotEqual(result[0], 0, result) + self.assertNotIn("Installed verified", result[1]) + self.assertEqual(target.read_text(), "def main(): print('tampered')\n") + + def test_nonexecutable_console_entrypoint_refuses_reuse(self): + self.assertEqual(self.run_installer()[0], 0) + destination = self.prefix / "share" / CONFIG["repo"] / TAG + command = destination / "bin" / CONFIG["commands"][0] + command.chmod(0o644) + result = self.run_installer() + self.assertNotEqual(result[0], 0, result) + self.assertNotIn("Installed verified", result[1]) + def test_corrupt_checksum_preserves_previous(self): path = self.release / self.names[0] path.write_bytes(path.read_bytes() + b'corrupt')