diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 4d739be5..4a4eeb32 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-science", - "version": "0.3.0+codex.20260720000100", + "version": "0.4.0+codex.20260721000100", "description": "Audited scientific skills, public-source tools, reproducible artifacts, and evidence review for Codex.", "author": { "name": "eightmm" diff --git a/authored-skills/docking-validation/SKILL.md b/authored-skills/docking-validation/SKILL.md index 098a08e1..9ed37044 100644 --- a/authored-skills/docking-validation/SKILL.md +++ b/authored-skills/docking-validation/SKILL.md @@ -6,6 +6,12 @@ license: MIT # Docking Validation +## Reference usage + +Read [benchmark design](references/benchmark-design.md) before defining states, pockets, controls, or data splits. Read [metrics and acceptance](references/metric-and-acceptance.md) before computing metrics, fixing thresholds, calibrating scores, or declaring a pass. Read [inference boundaries](references/inference-boundaries.md) before writing pose, ranking, affinity, mechanism, or generalization claims. + +Do not infer benchmark information boundaries, metric aggregation, or claim strength from an engine name or score. Record the reference hashes used for a material benchmark or claim in a `reference-use-ledger`. + ## Decision contract Prespecify the use case, prediction unit, receptor and ligand states, pocket information allowed at inference, benchmark split, controls, primary metric, uncertainty method, applicability domain, and success threshold before inspecting outcomes. Treat benchmark construction, preparation, pose generation, scoring, and decision analysis as separate stages with separate artifacts. diff --git a/authored-skills/docking-validation/quality.json b/authored-skills/docking-validation/quality.json new file mode 100644 index 00000000..98cad50d --- /dev/null +++ b/authored-skills/docking-validation/quality.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "skill": "docking-validation", + "roles": ["analyzer", "reviewer"], + "declared_maturity": "L4", + "output_schemas": [ + "sbdd-benchmark-v1", + "numeric-metric-v1", + "pose-table-v1", + "model-receipt-v2", + "review-receipt-v2" + ], + "acceptance_fixtures": [ + "examples/sbdd-acceptance/benchmark.json", + "examples/sbdd-acceptance/seeded-leaks.json", + "examples/sbdd-executable/input.json" + ], + "seeded_failures": [ + "held-out-pocket-leak", + "analog-series-leak", + "scaffold-leak", + "cold-target-violation", + "known-training-overlap", + "missing-subgroup-analysis", + "affinity-overclaim" + ], + "test_files": [ + "tests/test_sbdd_acceptance.py", + "tests/test_sbdd_execution.py" + ], + "limitations": [ + "The deterministic executable fixture validates contract handling rather than prospective docking accuracy.", + "Docking scores and model confidence are not experimental affinity or mechanism." + ] +} diff --git a/authored-skills/docking-validation/references/benchmark-design.md b/authored-skills/docking-validation/references/benchmark-design.md new file mode 100644 index 00000000..a5cdbfbd --- /dev/null +++ b/authored-skills/docking-validation/references/benchmark-design.md @@ -0,0 +1,31 @@ +# Docking benchmark design + +Read before creating a split, defining a pocket, choosing receptor states, or selecting positive and negative controls. + +## Prediction unit + +State whether the prediction unit is a complex, ligand, pose, target, target family, receptor state, or screening library. Metrics and independence groups must use the same unit. + +## State manifest + +Record biological assembly, chains, missing residues, mutations, protonation, tautomers, stereochemistry, formal charges, metals, cofactors, waters, covalent bonds, receptor state, ligand microstate, preparation software, parameters, and hashes. + +## Information boundary + +For a held-out complex, do not derive the pocket, ligand frame, receptor conformation, template, restraints, or feature engineering from the held-out bound pose unless the task explicitly permits that information. + +## Splits + +Preserve: + +- cold scaffold; +- cold analog series; +- cold target; +- cold target family; +- cold receptor state; +- temporal holdout; +- known and possible model-training overlap. + +## Controls + +At minimum include valid redocking controls, cross-docking where robustness is claimed, simple scoring or geometric baselines, chemically relevant negatives or property-matched decoys for screening, and known failure cases. diff --git a/authored-skills/docking-validation/references/index.json b/authored-skills/docking-validation/references/index.json new file mode 100644 index 00000000..1ac7866f --- /dev/null +++ b/authored-skills/docking-validation/references/index.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "skill": "docking-validation", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "benchmark-design", + "path": "references/benchmark-design.md", + "purpose": "Define state manifests, information boundaries, leakage-aware splits, controls, and prediction units.", + "read_when": ["building a docking benchmark", "defining a pocket", "creating train/test splits"], + "required_before": ["benchmark-construction", "pocket-definition", "split-construction"], + "search_patterns": ["## Information boundary", "## Splits", "## Controls"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Defines benchmark validity; it does not show that a docking engine performs well." + }, + { + "id": "metric-and-acceptance", + "path": "references/metric-and-acceptance.md", + "purpose": "Prespecify pose, screening, calibration, uncertainty, threshold, and failure metrics.", + "read_when": ["computing docking metrics", "setting acceptance thresholds", "calibrating scores"], + "required_before": ["metric-computation", "acceptance-decision", "score-calibration"], + "search_patterns": ["## Pose metrics", "## Screening metrics", "## Thresholds"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Metric definitions do not validate data splits, state preparation, or biological claims." + }, + { + "id": "inference-boundaries", + "path": "references/inference-boundaries.md", + "purpose": "Keep pose, confidence, ranking, affinity, mechanism, and translational claims distinct.", + "read_when": ["writing docking conclusions", "claiming affinity or mechanism", "defining applicability or abstention"], + "required_before": ["docking-claim-synthesis", "affinity-claim", "mechanism-claim"], + "search_patterns": ["## Distinct outputs", "## Required external evidence", "## Claim downgrade"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Provides inference limits and does not supply assay or mechanistic evidence." + } + ] +} diff --git a/authored-skills/docking-validation/references/inference-boundaries.md b/authored-skills/docking-validation/references/inference-boundaries.md new file mode 100644 index 00000000..804cf648 --- /dev/null +++ b/authored-skills/docking-validation/references/inference-boundaries.md @@ -0,0 +1,30 @@ +# Docking inference boundaries + +Read before writing pose, ranking, affinity, selectivity, mechanism, or translational claims. + +## Distinct outputs + +Keep separate: + +- generated pose; +- pose confidence; +- docking or rescoring value; +- ranking performance; +- predicted affinity; +- calibrated binding probability; +- experimental affinity; +- cellular or in vivo effect. + +A favorable docking score is not experimental affinity. A plausible pose is not a mechanism. A benchmark average is not evidence for one target–ligand pair outside the benchmark domain. + +## Required external evidence + +Affinity claims require assay-aware evidence with units, relation qualifiers, endpoint, conditions, and uncertainty. Mechanism claims require orthogonal structural, biochemical, mutational, kinetic, or cellular evidence as appropriate. + +## Applicability + +Report target family, receptor states, ligand classes, microstates, covalent or metal chemistry, size and flexibility range, and training-overlap state. Keep unsupported cases exploratory or abstain. + +## Claim downgrade + +When essential support is absent, downgrade from mechanistic to pose hypothesis, from affinity to ranking hypothesis, from generalization to within-benchmark performance, or from supported to inconclusive. diff --git a/authored-skills/docking-validation/references/metric-and-acceptance.md b/authored-skills/docking-validation/references/metric-and-acceptance.md new file mode 100644 index 00000000..9583737c --- /dev/null +++ b/authored-skills/docking-validation/references/metric-and-acceptance.md @@ -0,0 +1,23 @@ +# Docking metrics and acceptance + +Read before computing metrics, fixing thresholds, aggregating poses, or declaring a benchmark pass. + +## Pose metrics + +Report top-1 and top-k symmetry-aware heavy-atom RMSD, complete distributions, missing poses, and physical or chemical plausibility. Interaction recovery complements RMSD but does not replace geometry. + +## Screening metrics + +Use PR-AUC and prespecified enrichment metrics as primary measures when actives are rare. Report active/decoy counts, grouping, bootstrap intervals, and per-target results. ROC-AUC is secondary when class imbalance is material. + +## Calibration and abstention + +A score becomes a probability only after calibration on an appropriate held-out domain. Report reliability, Brier or log loss when relevant, subgroup calibration, and an abstention rule for unsupported chemical or target space. + +## Thresholds + +Fix metric, direction, threshold, confidence interval method, aggregation, exclusions, and subgroup requirements before inspecting held-out outcomes. Process completion is not an acceptance pass. + +## Failure taxonomy + +Distinguish preparation failure, unsupported chemistry, engine failure, no pose, invalid pose, geometric failure, interaction failure, ranking failure, calibration failure, and leakage invalidation. diff --git a/authored-skills/literature-review/SKILL.md b/authored-skills/literature-review/SKILL.md index 749ef5c5..4151b679 100644 --- a/authored-skills/literature-review/SKILL.md +++ b/authored-skills/literature-review/SKILL.md @@ -8,6 +8,12 @@ license: MIT Use this conductor when the deliverable is an auditable body-of-evidence synthesis rather than a citation list or list of papers. Discovery tools retrieve records; this skill owns the protocol, eligibility, study identity, extraction, dependency graph, synthesis boundary, update diff, and review handoff. +## Reference usage + +Before protocol freeze, study deduplication, or replication assessment, read [protocol and study identity](references/protocol-and-study-identity.md). Before structured extraction, risk-of-bias assessment, or claim-level evidence assertions, read [extraction and bias](references/extraction-and-bias.md). Before advancing a cutoff or replacing a reviewed snapshot, read [living update](references/living-update.md). + +Do not infer identifier relationships, extraction fields, bias judgments, or update invalidation rules from a record title. Save material reference use and SHA-256 values in a `reference-use-ledger`. + ## Decision contract Before retrieval, record: diff --git a/authored-skills/literature-review/quality.json b/authored-skills/literature-review/quality.json new file mode 100644 index 00000000..cc2990d8 --- /dev/null +++ b/authored-skills/literature-review/quality.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "skill": "literature-review", + "roles": ["conductor", "analyzer"], + "declared_maturity": "L4", + "output_schemas": [ + "literature-protocol-v1", + "query-ledger-v1", + "study-table-v1", + "study-family-v1", + "risk-of-bias-v1", + "claim-register-v1", + "evidence-graph-v2", + "literature-snapshot-v1-v2" + ], + "acceptance_fixtures": [ + "examples/literature-review-reviewed-run/manifest.json", + "examples/literature-review-reviewed-run/snapshot.previous.json", + "examples/literature-review-reviewed-run/snapshot.current.json" + ], + "seeded_failures": [ + "duplicate-study-identity", + "citation-mismatch", + "unsupported-conclusion", + "dependent-evidence-presented-as-replication" + ], + "test_files": [ + "tests/test_evidence_sidecars.py", + "tests/test_collaboration_literature_v2.py" + ], + "limitations": [ + "The checked-in reviewed run is a deterministic fixture rather than a live systematic-review conclusion.", + "Full-text licensing and source availability remain publication-specific." + ] +} diff --git a/authored-skills/literature-review/references/extraction-and-bias.md b/authored-skills/literature-review/references/extraction-and-bias.md new file mode 100644 index 00000000..a402f28f --- /dev/null +++ b/authored-skills/literature-review/references/extraction-and-bias.md @@ -0,0 +1,45 @@ +# Structured extraction and risk of bias + +Read before constructing the study table, effect extraction, risk-of-bias record, or claim-level evidence assertion. + +## Minimum study row + +Record, where applicable: + +- namespaced identifiers and study family; +- publication state, funding, and conflicts; +- design, setting, population or system, sample unit, sample size, and eligibility; +- intervention or exposure, comparator, endpoint, follow-up, and attrition; +- assay, preprocessing, model, covariates, missing-data handling, multiplicity correction, and protocol deviations; +- effect measure, estimate, interval, unit, denominator, and exact source locator; +- prespecified versus post-hoc status; +- supported and contradicted claim IDs; +- independence group and source dependencies. + +Do not infer missing methods, denominators, estimates, or uncertainty from an abstract or portal summary. + +## Risk-of-bias record + +Each record contains: + +```json +{ + "schema_version": 1, + "study_id": "study-1", + "instrument": "RoB 2 / ROBINS-I / domain-specific instrument", + "domains": [ + { + "name": "selection", + "judgment": "low | some-concerns | high | unclear | not-applicable", + "rationale": "source-grounded reason" + } + ], + "overall_judgment": "..." +} +``` + +A numeric summary score cannot replace domain judgments. The overall judgment cannot be more favorable than an essential high-risk domain without an explicit rationale. + +## Evidence assertion + +A material assertion should identify polarity, effect direction, effect measure, estimate and interval, population or system, exact source locator, independence group, and risk-of-bias reference. Citation presence alone is not sufficient. diff --git a/authored-skills/literature-review/references/index.json b/authored-skills/literature-review/references/index.json new file mode 100644 index 00000000..54191bcb --- /dev/null +++ b/authored-skills/literature-review/references/index.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "skill": "literature-review", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "protocol-and-study-identity", + "path": "references/protocol-and-study-identity.md", + "purpose": "Freeze the review protocol, normalize identifiers, resolve publication versions, and determine evidence independence.", + "read_when": ["freezing eligibility", "deduplicating studies", "resolving preprints, corrections, or registries"], + "required_before": ["protocol-freeze", "study-deduplication", "replication-assessment"], + "search_patterns": ["## Protocol freeze", "## Study-family resolution", "## Independence"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Defines review procedure; actual study records remain the evidence." + }, + { + "id": "extraction-and-bias", + "path": "references/extraction-and-bias.md", + "purpose": "Construct study rows, effect assertions, source locators, and explicit risk-of-bias records.", + "read_when": ["extracting studies", "grading risk of bias", "writing claim-level evidence assertions"], + "required_before": ["structured-extraction", "risk-of-bias-assessment", "evidence-assertion"], + "search_patterns": ["## Minimum study row", "## Risk-of-bias record", "## Evidence assertion"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Does not supply missing study values or replace primary-source verification." + }, + { + "id": "living-update", + "path": "references/living-update.md", + "purpose": "Advance a literature snapshot without overwriting prior evidence or reusing stale review receipts.", + "read_when": ["running living-update mode", "changing an evidence cutoff", "replacing a reviewed synthesis"], + "required_before": ["living-review-update", "review-snapshot-replacement"], + "search_patterns": ["## Immutable snapshots", "## Update diff", "## Invalidation"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Controls update lineage; it does not establish that new evidence changes a claim." + } + ] +} diff --git a/authored-skills/literature-review/references/living-update.md b/authored-skills/literature-review/references/living-update.md new file mode 100644 index 00000000..88415fa1 --- /dev/null +++ b/authored-skills/literature-review/references/living-update.md @@ -0,0 +1,33 @@ +# Living review updates + +Read before advancing an evidence cutoff, replacing a reviewed synthesis, or reusing a prior review receipt. + +## Immutable snapshots + +Preserve the prior protocol, exact queries, source releases, included and excluded studies, study-family mapping, claim register, and reviewed report. Create a new snapshot rather than editing the old one. + +## Update diff + +Compare: + +- query additions, removals, and changes; +- source availability or release changes; +- added, removed, and modified studies; +- publication-version, correction, and retraction changes; +- eligibility decisions; +- risk-of-bias changes; +- claim status, confidence, and applicability changes. + +Use: + +```bash +python scripts/diff_literature_review.py previous.json current.json --output diff.json +``` + +## Invalidation + +Invalidate the previous review receipt when covered queries, source snapshots, study table, evidence graph, extraction, claim register, or report bytes change. A narrative that appears unchanged still requires re-review when its evidence dependencies changed. + +## Stop rule + +Stop the update when every source has reached the declared cutoff, pagination is complete or explicitly bounded, unresolved study identities are visible, and claim-level changes have been reviewed. Source failure is unavailable evidence, not a negative study. diff --git a/authored-skills/literature-review/references/protocol-and-study-identity.md b/authored-skills/literature-review/references/protocol-and-study-identity.md new file mode 100644 index 00000000..6c721a55 --- /dev/null +++ b/authored-skills/literature-review/references/protocol-and-study-identity.md @@ -0,0 +1,45 @@ +# Protocol and study identity + +Read before freezing eligibility criteria, deduplicating records, linking publication versions, or claiming independent replication. + +## Protocol freeze + +Record the question framework, population or system, intervention or exposure, comparator, outcomes, designs, publication state, language, time window, source set, evidence cutoff, permitted inference, and stopping rule before reviewing results. + +A change to eligibility, primary outcome, evidence cutoff, or permitted inference creates a protocol revision. Preserve the prior protocol and explain the change. + +## Namespaced identifiers + +Use namespaced persistent identifiers: + +```text +doi:10.xxxx/... +pmid:12345678 +pmcid:PMC123456 +arxiv:2607.12345 +nct:NCT01234567 +``` + +Do not compare bare identifier strings across namespaces. + +## Study-family resolution + +One underlying study may have: + +- registry and protocol records; +- conference abstract; +- preprint; +- accepted manuscript; +- peer-reviewed article; +- correction or retraction; +- secondary analysis. + +Union records through shared identifiers and explicit relationships. Prefer the corrected peer-reviewed version as canonical, but retain every version and its publication state. + +## Independence + +Record shared cohort, sample, site, database, portal, author group, and analysis pipeline. Multiple papers, endpoints, or portal views from one cohort are not independent replication. + +## Exclusion records + +Every exclusion includes stage, reviewer, reason, and source record ID. Do not silently discard near matches, superseded versions, null results, or inaccessible full text. diff --git a/authored-skills/remote-scientific-compute/SKILL.md b/authored-skills/remote-scientific-compute/SKILL.md index 779b9084..bc74273c 100644 --- a/authored-skills/remote-scientific-compute/SKILL.md +++ b/authored-skills/remote-scientific-compute/SKILL.md @@ -8,6 +8,14 @@ description: "Run reproducible scientific workloads through an existing SSH host Move an approved scientific workload to existing remote infrastructure while protecting credentials, login nodes, private data, budgets, and reproducibility. +## Reference usage + +Read [the backend checklists](references/backends.md) for the selected SSH, Slurm/HPC, cloud GPU, or object-storage route before staging, submitting, allocating, or writing remotely. Do not infer scheduler flags, cost controls, storage semantics, or cleanup procedures from a backend name. + +Read [the durable job runtime contract](references/job-runtime.md) before constructing, approving, submitting, polling, cancelling, or collecting a local or Slurm job. A process receipt records execution state; it is not scientific acceptance. + +Record the reference version and SHA-256 in the run when it controls a submission or transfer. The backend reference governs execution safety; it is not evidence for the scientific result. + ## Inspect without changing state 1. Use only an existing host alias, provider profile, project, and account. Never @@ -68,3 +76,15 @@ Stop for unknown ownership, host-key changes, missing authorization, secrets in planned command, unapproved private-data transfer, unavailable cancellation, uncapped paid resources, a full filesystem, or a resource request unsuitable for the target. Report the blocker without weakening the boundary. + +## Decision contract + +Before execution, record the target, transfer boundary, argv or workflow, input hashes, runtime or container, resource and wall-time caps, cost cap, checkpoints, cancellation, outputs, and scientific acceptance criteria. Remote writes or allocation require explicit approval covering the exact job spec. + +## Outputs + +Save the job spec, preflight and approval receipts, job ID and states, stdout and stderr hashes, failure class, checkpoint references, collected output hashes, and `$science-provenance` and `$science-review` handoffs. + +## Boundaries + +Do not probe credentials, submit to unapproved accounts or partitions, stage sensitive data without approval, busy-poll, hide failed attempts, or treat process exit zero as scientific success. diff --git a/authored-skills/remote-scientific-compute/quality.json b/authored-skills/remote-scientific-compute/quality.json new file mode 100644 index 00000000..40887a56 --- /dev/null +++ b/authored-skills/remote-scientific-compute/quality.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "skill": "remote-scientific-compute", + "roles": ["executor", "infrastructure"], + "declared_maturity": "L3", + "output_schemas": [ + "job-spec-v1", + "job-approval-receipt-v1", + "job-state-v1", + "collection-receipt-v1" + ], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [ + "Local execution is directly supported; Slurm submission requires an existing scheduler configuration and explicit approval.", + "TES, WES, SSH staging, and cloud-provider adapters remain separate future backends." + ] +} diff --git a/authored-skills/remote-scientific-compute/references/index.json b/authored-skills/remote-scientific-compute/references/index.json new file mode 100644 index 00000000..85ae5eb0 --- /dev/null +++ b/authored-skills/remote-scientific-compute/references/index.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "skill": "remote-scientific-compute", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "backend-checklists", + "path": "references/backends.md", + "purpose": "Backend-specific SSH, Slurm/HPC, cloud GPU, and object-storage preflight, execution, monitoring, and cleanup rules.", + "read_when": ["selecting a remote backend", "submitting a Slurm job", "allocating cloud GPU", "transferring object storage data"], + "required_before": ["remote-stage", "remote-submit", "paid-allocation", "object-storage-write"], + "search_patterns": ["## SSH", "## Slurm/HPC", "## Cloud GPU", "## Object storage"], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Controls remote execution safety; it does not validate the scientific method or result." + }, + { + "id": "job-runtime", + "path": "references/job-runtime.md", + "purpose": "Durable local and Slurm job specs, approvals, submission, status, cancellation, checkpoint, and output collection receipts.", + "read_when": ["before constructing a durable job spec", "before submitting, polling, cancelling, or collecting a local or Slurm job", "before approving a non-local resource allocation"], + "required_before": ["calling scripts/science_job.py", "submitting work through LocalBackend or SlurmBackend"], + "search_patterns": ["## Job spec", "## Local execution", "## Approval receipt", "## Slurm execution", "## Checkpoint and resume", "## Failure handling"], + "authority": "Codex Science first-party compute runtime contract", + "version": "1", + "evidence_boundary": "A job receipt proves execution state and collected hashes, not scientific success." + } + ] +} diff --git a/authored-skills/remote-scientific-compute/references/job-runtime.md b/authored-skills/remote-scientific-compute/references/job-runtime.md new file mode 100644 index 00000000..5249680b --- /dev/null +++ b/authored-skills/remote-scientific-compute/references/job-runtime.md @@ -0,0 +1,396 @@ +# Durable job runtime: local and Slurm + +Read this reference before constructing a job spec, approving a non-local job, submitting local or Slurm work, polling state, cancelling, or collecting outputs. + +## Backend boundary + +Implemented execution backends: + +- `local` — durable worker process, terminal receipt, wait, cancel, and hash-based output collection; +- `slurm` — script rendering, approved `sbatch` submission, `sacct`/`squeue` polling, `scancel`, and shared-filesystem output collection. + +Not implemented by this runtime: + +- implicit SSH credential discovery; +- remote object-store transfer; +- cloud-provider provisioning; +- GA4GH TES/WES submission; +- software installation; +- scheduler account or partition discovery by trial submission. + +Those routes require separate adapters and approval contracts. Do not pass an unsupported backend name and do not disguise a remote shell command as `local`. + +## Job spec + +A job spec is a JSON object: + +```json +{ + "schema_version": 1, + "backend": "local", + "name": "compute-rmsd", + "command": [ + "python", + "scripts/measure_pose.py", + "--input", + "poses.sdf", + "--output", + "metrics.json" + ], + "working_directory": "/absolute/path/project", + "environment": { + "OMP_NUM_THREADS": "4" + }, + "inherit_environment": true, + "inputs": [ + { + "id": "poses", + "path": "poses.sdf", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "outputs": [ + "metrics.json" + ], + "resources": { + "cpus": 4, + "memory_mb": 8192, + "gpus": 0, + "gpu_type": null, + "wall_time_seconds": 1800, + "partition": null, + "account": null, + "qos": null + }, + "timeout_seconds": 1800, + "cost_cap": null, + "approval_required": false, + "scientific_run_id": "run-014", + "checkpoint_paths": [] +} +``` + +### Required semantics + +- `command` is an argv list, never a shell command string; +- `working_directory` must already exist; +- outputs and checkpoints are safe paths relative to the working directory; +- input hashes are recorded when available; +- secret-like environment keys are rejected; +- timeout is positive; +- local resource requests are recorded but not scheduler reservations; +- non-local submission requires approval even when `approval_required` is false in an invalid user-supplied spec. + +The spec fingerprint covers all fields, including command, environment values, resource request, input hashes, outputs, timeout, and cost cap. + +## CLI overview + +```bash +python scripts/science_job.py --state-dir .science-jobs preflight ... +python scripts/science_job.py --state-dir .science-jobs approve ... +python scripts/science_job.py --state-dir .science-jobs render-slurm ... +python scripts/science_job.py --state-dir .science-jobs submit ... +python scripts/science_job.py --state-dir .science-jobs status ... +python scripts/science_job.py --state-dir .science-jobs wait ... +python scripts/science_job.py --state-dir .science-jobs cancel ... +python scripts/science_job.py --state-dir .science-jobs collect ... +``` + +Every command writes a JSON receipt to `--output`. + +## Local execution + +### Preflight + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + preflight \ + --spec artifacts/run-014/job-spec.json \ + --output artifacts/run-014/job-preflight.json +``` + +Preflight checks: + +- backend matches; +- executable is resolvable; +- working directory exists; +- spec validates; +- resource request and spec fingerprint are recorded. + +A successful preflight does not prove that imports, model weights, databases, input semantics, or scientific assumptions are correct. + +### Submit + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + submit \ + --spec artifacts/run-014/job-spec.json \ + --output artifacts/run-014/job-submitted.json +``` + +The command starts a detached Python worker and returns a state receipt: + +```json +{ + "schema_version": 1, + "job_id": "local-...", + "backend": "local", + "job_spec_sha256": "...", + "state": "submitted", + "worker_pid": 12345, + "stdout_path": "stdout.log", + "stderr_path": "stderr.log", + "fingerprint": "..." +} +``` + +The job directory contains: + +```text +jobs/jobs/local-.../ + spec.json + state.json + stdout.log + stderr.log +``` + +`state.json` is written atomically. It is mutable operational state, not scientific evidence. Save terminal receipts and output hashes in the run manifest when they support a claim. + +### Status and wait + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + status \ + --backend local \ + --job-id local-... \ + --output artifacts/run-014/job-status.json + +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + wait \ + --backend local \ + --job-id local-... \ + --timeout 120 \ + --poll 0.2 \ + --output artifacts/run-014/job-terminal.json +``` + +States: + +```text +submitted +running +completed +failed +timed-out +cancelled +lost +``` + +Failure classes distinguish at least: + +```text +nonzero-exit +wall-time +executable-not-found +worker-exception +worker-lost +user-cancelled +``` + +A process exit code of zero means only that the command completed according to the operating system. It is not a scientific acceptance result. + +### Cancel + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + cancel \ + --backend local \ + --job-id local-... \ + --output artifacts/run-014/job-cancelled.json +``` + +Cancellation is idempotent for terminal jobs. The local backend sends `SIGTERM` to the worker process group and records `user-cancelled`. + +### Collect outputs + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + collect \ + --backend local \ + --job-id local-... \ + --output artifacts/run-014/job-collected.json +``` + +Collection is allowed only after a terminal state. Declared outputs are resolved inside the working directory: + +- files receive streaming SHA-256 and size; +- directories receive a Merkle root, total bytes, and entry count; +- missing outputs remain explicit with `artifact_type: missing`. + +Do not remove a missing output from the receipt to make a run appear complete. + +## Approval receipt + +Create approval for a spec only after reviewing target, resource, cost, transfer, cancellation, and scientific scope. + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + approve \ + --spec artifacts/run-014/slurm-job-spec.json \ + --approved-by jaemin \ + --target hpc-cluster-a \ + --output artifacts/run-014/slurm-approval.json +``` + +The receipt binds: + +```json +{ + "job_spec_sha256": "...", + "backend": "slurm", + "target": "hpc-cluster-a", + "approved": true, + "resource_cap": {}, + "cost_cap": null, + "fingerprint": "..." +} +``` + +Changing the job spec invalidates the approval. Approval authorizes execution boundaries; it does not endorse scientific validity. + +## Slurm execution + +### Render before submitting + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + render-slurm \ + --spec artifacts/run-014/slurm-job-spec.json \ + --job-id preview \ + --output artifacts/run-014/slurm-preview.sh +``` + +Inspect: + +- command and quoting; +- working directory; +- CPU, memory, GPU, time, partition, account, and QoS; +- stdout/stderr destination; +- environment exports; +- timeout wrapper. + +Rendering is safe and does not submit. + +### Submit + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + submit \ + --spec artifacts/run-014/slurm-job-spec.json \ + --approval artifacts/run-014/slurm-approval.json \ + --output artifacts/run-014/slurm-submitted.json +``` + +Preconditions: + +- `sbatch` and `scancel` available; +- `sacct` or `squeue` available; +- approval fingerprint matches the exact spec; +- scheduler account/partition already known and approved; +- working directory is a shared path visible to compute nodes; +- all required data are already staged by an approved route. + +The backend does not copy data or credentials. + +### Status + +```bash +python scripts/science_job.py \ + --state-dir artifacts/run-014/jobs \ + status \ + --backend slurm \ + --job-id slurm-local-... \ + --output artifacts/run-014/slurm-status.json +``` + +The adapter normalizes scheduler states while retaining the raw state in the message. `OUT_OF_MEMORY`, `NODE_FAIL`, `PREEMPTED`, `TIMEOUT`, and `CANCELLED` are not merged into one generic failure. + +### Cancel and collect + +```bash +python scripts/science_job.py --state-dir artifacts/run-014/jobs cancel \ + --backend slurm --job-id slurm-local-... --output cancelled.json + +python scripts/science_job.py --state-dir artifacts/run-014/jobs collect \ + --backend slurm --job-id slurm-local-... --output collected.json +``` + +Collection assumes a shared filesystem. Remote object storage or SSH transfer requires a separate approved staging backend. + +## Checkpoint and resume + +`checkpoint_paths` identify files expected to make a later attempt resumable. The runtime records and collects them but does not infer a program-specific resume command. + +Required procedure: + +1. verify checkpoint bytes and compatibility with code/config/model revision; +2. create a new job spec for the resumed attempt; +3. include the prior job and checkpoint hashes in run provenance; +4. obtain new approval when target, resources, cost, or command changes; +5. preserve the failed or preempted attempt. + +Never rewrite a failed job receipt into a successful resumed job. + +## Search patterns + +Use these exact searches when reading this reference: + +- `## Job spec` +- `## Local execution` +- `## Approval receipt` +- `## Slurm execution` +- `## Checkpoint and resume` +- `## Failure handling` +- `## Common mistakes` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| executable missing | stop; use the approved environment or installation gate | +| local worker lost | preserve logs/state, inspect cause, create a new attempt | +| wall-time | preserve checkpoint, revise resource/timeout contract, obtain approval if required | +| nonzero exit | inspect stderr and scientific inputs; do not retry unchanged indefinitely | +| OOM | reduce workload or request reviewed resources; record OOM separately | +| node failure or preemption | resume only from verified compatible checkpoint | +| `sbatch` error | do not probe random accounts/partitions; resolve scheduler configuration | +| output missing | keep explicit missing record and leave dependent plan steps incomplete | +| output escapes working directory | reject collection; correct the job spec | +| approval mismatch | create a new approval for the exact spec | +| stale input hash | create a new run or attempt contract before execution | + +## Common mistakes + +- Passing a shell pipeline as one command string instead of an argv list. +- Recording tokens, passwords, or private keys in the job environment. +- Treating requested local resources as enforced limits. +- Submitting Slurm work before reviewing the rendered script. +- Using a shared login node for heavy computation instead of the scheduler. +- Busy-polling a waiting job. +- Treating exit code zero as scientific success. +- Omitting failed attempts, missing outputs, or cancellation receipts. +- Reusing approval after changing code, command, resources, target, or input hashes. +- Claiming remote durability when state files exist only on an ephemeral local disk. + +## Evidence boundary + +The job runtime provides durable execution and collection receipts. It does not establish reproducibility across environments, validate model or database licensing, authenticate scheduler identity, measure cloud cost unless an adapter reports it, or prove a scientific result. diff --git a/catalog/inventory.json b/catalog/inventory.json index 7d341cde..7340a64e 100644 --- a/catalog/inventory.json +++ b/catalog/inventory.json @@ -426,7 +426,7 @@ { "description": "Validate docking, pose-prediction, rescoring, and virtual-screening workflows with leakage-aware controls and prespecified metrics. Use before interpreting AutoDock Vina, GNINA, DiffDock, or related docking outputs.", "executable_count": 0, - "instruction_line_count": 30, + "instruction_line_count": 36, "license": "MIT", "name": "cx-docking-validation", "path": "authored-skills/docking-validation", @@ -762,7 +762,7 @@ { "description": "Run a protocol-driven, updateable literature review with exact multi-source queries, study-level deduplication, structured extraction, evidence dependency tracking, bounded synthesis, provenance, and independent review. Use for systematic, scoping, rapid, living, or evidence-landscape reviews where discovery alone is insufficient.", "executable_count": 0, - "instruction_line_count": 93, + "instruction_line_count": 99, "license": "MIT", "name": "cx-literature-review", "path": "authored-skills/literature-review", @@ -1314,7 +1314,7 @@ { "description": "Run reproducible scientific workloads through an existing SSH host, Slurm/HPC cluster, cloud GPU provider, or remote object storage. Use for remote execution, job submission and monitoring, checkpointed long runs, GPU allocation, data staging, result retrieval, queue diagnosis, or resuming work that can outlive the current Codex task.", "executable_count": 0, - "instruction_line_count": 70, + "instruction_line_count": 90, "license": "Apache-2.0", "name": "cx-remote-scientific-compute", "path": "authored-skills/remote-scientific-compute", diff --git a/catalog/native-skill-policy.json b/catalog/native-skill-policy.json new file mode 100644 index 00000000..4b8d420c --- /dev/null +++ b/catalog/native-skill-policy.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "requirements": [ + { + "skill": "codex-science", + "minimum_maturity": "L3", + "required_roles": ["conductor"], + "require_reference_contract": true + }, + { + "skill": "science-provenance", + "minimum_maturity": "L3", + "required_roles": ["infrastructure"], + "require_reference_contract": true + }, + { + "skill": "science-review", + "minimum_maturity": "L3", + "required_roles": ["reviewer"], + "require_reference_contract": true + }, + { + "skill": "literature-review", + "minimum_maturity": "L4", + "required_roles": ["conductor", "analyzer"], + "require_reference_contract": true + }, + { + "skill": "docking-validation", + "minimum_maturity": "L4", + "required_roles": ["analyzer", "reviewer"], + "require_reference_contract": true + }, + { + "skill": "remote-scientific-compute", + "minimum_maturity": "L3", + "required_roles": ["executor", "infrastructure"], + "require_reference_contract": true + } + ] +} diff --git a/docs/NATIVE_RUNTIME_QUICKSTART.md b/docs/NATIVE_RUNTIME_QUICKSTART.md new file mode 100644 index 00000000..0e60b9e1 --- /dev/null +++ b/docs/NATIVE_RUNTIME_QUICKSTART.md @@ -0,0 +1,127 @@ +# Native runtime quickstart + +Codex Science 0.4 uses progressive references: keep the decision flow in `SKILL.md`, then load the exact command/schema reference only when the route is selected. Do not guess a command or output field from a skill summary. + +## 1. Inspect and annotate an artifact + +Read first: + +```text +skills/science-provenance/references/artifact-runtime.md +``` + +Then: + +```bash +python scripts/render_artifact_runtime.py describe \ + artifacts/run/receptor.pdb \ + --artifact-path receptor.pdb \ + --sha256 \ + --kind receptor-structure \ + --artifact-type file \ + --media-type chemical/x-pdb \ + --max-bytes 1048576 \ + --max-records 200 \ + --output artifacts/run/receptor.runtime.json + +python scripts/render_artifact_runtime.py render \ + artifacts/run/receptor.runtime.json \ + --output artifacts/run/receptor.runtime.html + +python scripts/render_artifact_runtime.py select \ + artifacts/run/receptor.runtime.json \ + --selector-type residue \ + --selector '{"chain":"A","residue_number":145}' \ + --selected-by reviewer \ + --reason "Inspect receptor-state consistency." \ + --output artifacts/run/residue-A145.selection.json + +python scripts/render_artifact_runtime.py propose \ + artifacts/run/residue-A145.selection.json \ + --operation exclude-alternate-conformation \ + --parameters '{"altloc":"B"}' \ + --reason "Use one receptor microstate before docking." \ + --affected-step receptor-preparation \ + --affected-step docking \ + --expected-output prepared/receptor.pdbqt \ + --proposed-by analyst \ + --output artifacts/run/exclude-A145.proposal.json +``` + +The proposal does not edit or rerun anything. Feed it into impact analysis, approval, execution, diff, and re-review. + +## 2. Manage hypothesis branches + +Read first: + +```text +skills/codex-science/references/project-evidence-store.md +``` + +Then: + +```bash +python scripts/science_project.py --database project/evidence.sqlite init \ + --project-id project-1 --title "Project" --question "Question" \ + --output project/created.json + +python scripts/science_project.py --database project/evidence.sqlite import-run \ + --project-id project-1 --manifest artifacts/run-001/manifest.json \ + --branch main --output project/import-001.json + +python scripts/science_project.py --database project/evidence.sqlite fork \ + --project-id project-1 --source-run run-001 \ + --branch receptor-state-inactive --output project/fork.json +``` + +Import each later branch run as a new immutable run. Use `compare` and `merge-plan`; never overwrite or auto-merge scientific claims. + +## 3. Execute a durable job + +Read first: + +```text +authored-skills/remote-scientific-compute/references/job-runtime.md +``` + +Then: + +```bash +python scripts/science_job.py --state-dir artifacts/run/jobs preflight \ + --spec artifacts/run/job-spec.json --output artifacts/run/preflight.json + +python scripts/science_job.py --state-dir artifacts/run/jobs submit \ + --spec artifacts/run/job-spec.json --output artifacts/run/submitted.json + +python scripts/science_job.py --state-dir artifacts/run/jobs wait \ + --backend local --job-id --timeout 300 --poll 0.2 \ + --output artifacts/run/terminal.json + +python scripts/science_job.py --state-dir artifacts/run/jobs collect \ + --backend local --job-id --output artifacts/run/collected.json +``` + +For Slurm, render the script and create an approval receipt before `submit --approval`. + +## 4. Audit skill maturity + +```bash +python scripts/audit_native_skill_maturity.py \ + --output /tmp/native-skill-quality.json \ + --markdown /tmp/native-skill-quality.md \ + --require-clean +``` + +Maturity is computed from actual instructions, progressive references, declared output schemas, checked fixtures, seeded failures, and tests. `quality.json` is a declaration, not an automatic promotion. + +## Completion boundary + +These runtime commands improve inspection, lineage, execution, and auditability. Scientific completion still requires a prespecified decision contract, valid domain methods, material evidence, uncertainty, and independent review. + +## 6. Prepare an independent review packet + +Read `skills/science-review/references/independent-review-runtime.md`, then use `scripts/science_reviewer.py prepare` and `finalize`. A producer second pass is not independent, and reproduction requires a separate execution. + +## 7. Validate an authenticated connector action + +Read `skills/codex-science/references/authenticated-connector-packs.md`. Use `scripts/connector_action_contract.py` for spec, preview, and approval validation. Provider execution belongs to an optional adapter pack; credentials never enter the action JSON. diff --git a/docs/NATIVE_SKILL_STANDARD.md b/docs/NATIVE_SKILL_STANDARD.md index 6d239e6b..5b4a0450 100644 --- a/docs/NATIVE_SKILL_STANDARD.md +++ b/docs/NATIVE_SKILL_STANDARD.md @@ -286,3 +286,11 @@ When editing an authored skill: 4. run `./scripts/check.sh fast`; 5. run `./scripts/check.sh public` only for explicit live-source validation; 6. describe what was verified, what was not, and which scientific claims remain unvalidated. + +## Progressive reference contract + +Keep the decision-bearing workflow, outputs, stop rules, and inference boundary in `SKILL.md`. Move source-specific operations, command signatures, large schemas, benchmark details, backend procedures, and worked examples into `references/`. + +A skill with detailed references should provide `references/index.json` and a `## Reference usage` section. The index declares stable reference IDs, purposes, `read_when` routes, mandatory `required_before` operations, search patterns, authority, version, and evidence boundary. The agent must read the selected reference before executing a controlled operation and must not guess arguments or schemas that exist only in the reference. + +When a reference materially controls a query, command, transformation, threshold, or claim, record a hash-bound `reference-use-ledger`. Reference documents guide procedure; they are not scientific evidence by themselves. Run `python scripts/audit_skill_references.py --require-clean` before treating a migrated core skill as complete. diff --git a/docs/PLATFORM_CONTRACTS.md b/docs/PLATFORM_CONTRACTS.md index 01fcd124..c2befaa6 100644 --- a/docs/PLATFORM_CONTRACTS.md +++ b/docs/PLATFORM_CONTRACTS.md @@ -286,3 +286,15 @@ The workbench is a local offline HTML view of validated claims, findings, annota - A deterministic SBDD fixture is not an affinity experiment. - Artifact or model changes invalidate covered receipts. - Fresh install and managed update activation require the complete candidate contract. + +## Progressive skill references + +Core skills use `references/index.json` to load the minimum detailed instructions needed for the selected route. Mandatory references must be read before their `required_before` operation, and material use is recorded in a hash-bound `reference-use-ledger`. See `docs/REFERENCE_CONTRACT.md`. + +## Connector Contract v3 + +`science_query_source_v3` adds true source pagination, bounded retries, response-size limits, ETag and Last-Modified capture, rate-limit state, page-level request and response hashes, explicit completeness, and replayable raw page snapshots. Initial v3 operations cover PubMed, Europe PMC, ChEMBL molecule search, and RCSB PDB search. Sources not yet migrated remain available through v2 with their maturity visible. + +## Large artifact protocol + +Artifact validation uses streaming SHA-256 for files and deterministic Merkle roots for directories. Chunked descriptors support resumable verification, while a local content-addressed store reuses immutable verified files. Large scientific objects are never required to be loaded with `read_bytes()`. diff --git a/docs/REFERENCE_CONTRACT.md b/docs/REFERENCE_CONTRACT.md new file mode 100644 index 00000000..d5432fdd --- /dev/null +++ b/docs/REFERENCE_CONTRACT.md @@ -0,0 +1,172 @@ +# Scientific skill reference contract + +Codex Science uses progressive disclosure: `SKILL.md` contains the decision-bearing procedure, while `references/` contains detailed source schemas, command matrices, benchmark rules, and worked examples that are loaded only for the selected route. + +This contract follows the strongest pattern in Google DeepMind Science Skills: a skill lists its reference groups, states when each group is mandatory, and forbids guessing arguments or schemas that live only in a reference. Codex Science adds machine-readable indexing, use receipts, hash invalidation, and repository-wide validation. + +## Directory contract + +```text +skill-name/ +├── SKILL.md +└── references/ + ├── index.json + ├── source-query.md + └── output-schema.md +``` + +Keep reference files one level below `references/`. A reference may link to an authoritative external source, but the indexed local file must state what the source is used for, its evidence boundary, and any version or access requirement. + +## `references/index.json` + +```json +{ + "schema_version": 1, + "skill": "skill-name", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "source-query", + "path": "references/source-query.md", + "purpose": "Exact source operations, filters, pagination, and output fields.", + "read_when": [ + "retrieving records from the source", + "debugging a schema or pagination change" + ], + "required_before": [ + "source-search", + "source-fetch" + ], + "search_patterns": [ + "## search", + "## pagination", + "## output schema" + ], + "authority": "official-source-contract", + "version": "2026-07", + "evidence_boundary": "Describes retrieval behavior; it does not establish the scientific truth of returned records." + } + ] +} +``` + +### Required fields + +- `id`: stable identifier used in receipts and tests. +- `path`: safe path exactly one level below `references/`. +- `purpose`: what decision or operation this reference informs. +- `read_when`: natural-language routes that make the reference relevant. +- `required_before`: operation IDs that cannot be executed correctly without reading it. + +### Optional fields + +- `search_patterns`: headings or terms to locate in a long reference. Required by policy for large files. +- `authority`: repository contract, official source documentation, published standard, or another explicit authority class. +- `version`: reference or source-contract version. +- `evidence_boundary`: claims the reference itself cannot support. + +## `SKILL.md` reference usage section + +An indexed core skill contains `## Reference usage` or `## Reference map` and links every indexed file. The section must say: + +1. when to read the file; +2. which operation requires it before execution; +3. what not to infer without it; +4. whether a use receipt is required. + +Example: + +```markdown +## Reference usage + +Before calling any source operation, inspect `references/index.json` and load only the entries selected for the route. + +- Read [source query operations](references/source-query.md) before `source-search` or `source-fetch`. Do not infer argument names, filters, pagination, or output fields from an operation name. +- Read [output schema](references/output-schema.md) before writing a claim-bearing extraction table. + +For a material claim or executed command, save the loaded reference hash and reason in a `reference-use-ledger` artifact. +``` + +## Runtime loading procedure + +1. Load the selected skill contract. +2. If `references/index.json` exists, validate it before execution. +3. Select the minimum entries whose `read_when` or `required_before` match the route. +4. Read mandatory entries before the corresponding operation. Do not guess missing arguments, schemas, thresholds, source semantics, or licenses. +5. For a long file, search the indexed patterns first, then read the smallest contiguous section that resolves the route. +6. Record the reference file SHA-256 when it materially controls a query, command, threshold, transformation, or claim. +7. If the reference hash changes, invalidate receipts and reviews that depended on the previous bytes. + +Use: + +```bash +python scripts/reference_lookup.py authored-skills/literature-review \ + --route study-deduplication \ + --query "preprint correction registry" \ + --receipt-dir artifacts/run/reference-uses \ + --claim C1 +``` + +## Reference use ledger + +```json +{ + "schema_version": 1, + "skill": "literature-review", + "uses": [ + { + "schema_version": 1, + "skill": "literature-review", + "reference_id": "protocol-and-study-identity", + "path": "references/protocol-and-study-identity.md", + "sha256": "...", + "read_reason": "study-deduplication", + "used_for": ["claim-C1", "study-table"], + "sections": ["Study identity"], + "search_terms": ["correction", "preprint"], + "loaded_at": "2026-07-20T00:00:00Z" + } + ] +} +``` + +Attach the ledger to the run manifest with `kind: reference-use-ledger`. It records procedural grounding, not scientific evidence by itself. + +## What belongs where + +### Keep in `SKILL.md` + +- trigger and non-trigger boundary; +- decision contract; +- ordered workflow; +- mandatory outputs; +- stop and approval rules; +- core inference boundary; +- reference selection instructions. + +### Move to `references/` + +- source-specific query syntax and field maps; +- detailed command signatures; +- benchmark split and metric definitions; +- large schemas; +- long troubleshooting matrices; +- worked examples; +- standards and reporting checklists; +- backend-specific procedures. + +### Keep in `scripts/` + +- deterministic operations intended to execute; +- validators and converters; +- small CLIs with explicit dependencies and error handling. + +A reference is read for reasoning. A script is executed for a deterministic operation. `SKILL.md` must not leave this distinction ambiguous. + +## Quality gate + +```bash +python scripts/audit_skill_references.py --require-clean +``` + +The audit checks indexed paths, one-level layout, duplicate IDs, missing files, unindexed files, links from strict core skills, required read conditions, and search patterns for large references. Non-migrated first-party skills are reported for progressive cleanup; strict core skills fail the gate. diff --git a/pyproject.toml b/pyproject.toml index 30712fec..b501aa34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-science" -version = "0.3.0" +version = "0.4.0" description = "Audited scientific research workflows and public-source tools for Codex" readme = "README.md" authors = [ diff --git a/release/manifest.json b/release/manifest.json index f724a17d..ed8d737c 100644 --- a/release/manifest.json +++ b/release/manifest.json @@ -1,8 +1,8 @@ { "schema_version": 1, - "package_version": "0.3.0", - "plugin_version": "0.3.0+codex.20260720000100", - "mcp_version": "0.3.0", + "package_version": "0.4.0", + "plugin_version": "0.4.0+codex.20260721000100", + "mcp_version": "0.4.0", "runtime_affecting_prefixes": [ ".codex-plugin/", ".mcp.json", diff --git a/scripts/audit_native_skill_maturity.py b/scripts/audit_native_skill_maturity.py new file mode 100644 index 00000000..0b5600e3 --- /dev/null +++ b/scripts/audit_native_skill_maturity.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Audit first-party skills against machine-readable L0-L4 maturity policy.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.skill_maturity import audit_native_skills, render_maturity_markdown # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--markdown", type=Path) + parser.add_argument("--require-clean", action="store_true") + args = parser.parse_args() + try: + report = audit_native_skills(args.root.resolve(), policy_path=args.policy) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + rendered = json.dumps(report, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + print(f"Success! Data written to: {args.output.resolve()}") + else: + print(rendered, end="") + if args.markdown: + args.markdown.parent.mkdir(parents=True, exist_ok=True) + args.markdown.write_text(render_maturity_markdown(report), encoding="utf-8") + print(f"Success! Report written to: {args.markdown.resolve()}") + if args.require_clean and report["status"] != "passed": + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_skill_references.py b/scripts/audit_skill_references.py new file mode 100755 index 00000000..07cef43e --- /dev/null +++ b/scripts/audit_skill_references.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Audit progressive-disclosure reference contracts for first-party skills.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from codex_science.reference_contract import audit_reference_roots # noqa: E402 + +STRICT_SKILLS = { + "codex-science", + "science-provenance", + "science-review", + "literature-review", + "docking-validation", + "remote-scientific-compute", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--output", type=Path) + parser.add_argument("--require-clean", action="store_true") + args = parser.parse_args() + root = args.root.resolve() + report = audit_reference_roots( + [root / "skills", root / "authored-skills"], + strict_names=STRICT_SKILLS, + ) + text = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + summary = report["summary"] + print( + "reference audit: " + f"skills={summary['skills']} indexed={summary['indexed_skills']} " + f"references={summary['references']} findings={summary['findings']} " + f"major={summary['major_findings']}", + file=sys.stderr, + ) + return 1 if args.require_clean and summary["major_findings"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/candidate_contract_check.py b/scripts/candidate_contract_check.py index 7258473c..0de48433 100755 --- a/scripts/candidate_contract_check.py +++ b/scripts/candidate_contract_check.py @@ -31,6 +31,7 @@ def main() -> int: run([python, "scripts/validate_release.py", "--root", str(root)], cwd=root) run([python, "scripts/validate_connector_contracts.py", "--root", str(root)], cwd=root) + run([python, "scripts/audit_skill_references.py", "--root", str(root), "--require-clean"], cwd=root) run([python, "-m", "compileall", "-q", "src", "scripts"], cwd=root) with tempfile.TemporaryDirectory() as tempdir: diff --git a/scripts/check.sh b/scripts/check.sh index 9023295b..509b20be 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -37,6 +37,7 @@ run_science_contracts() ( uv run python scripts/validate_release.py uv run python scripts/validate_connector_contracts.py + uv run python scripts/audit_skill_references.py --require-clean uv run python scripts/validate_models.py uv run python scripts/validate_model_registry_v2.py uv run python scripts/run_reviewer_benchmark.py --output "$benchmark_tmp" --require-safe diff --git a/scripts/connector_action_contract.py b/scripts/connector_action_contract.py new file mode 100644 index 00000000..8056038d --- /dev/null +++ b/scripts/connector_action_contract.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Validate connector action specs/previews and issue hash-bound approvals.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.action_connectors import ActionSpec, build_action_approval, validate_preview # noqa: E402 + + +def read_object(path: Path) -> dict: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON object required: {path}") + return payload + + +def write(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Success! Data written to: {path.resolve()}") + + +def validate_spec(args: argparse.Namespace) -> None: + spec = ActionSpec.from_payload(read_object(args.spec)) + write(args.output, {"schema_version": 1, "status": "valid", "action_spec_sha256": spec.fingerprint, "spec": spec.to_dict()}) + + +def validate_preview_command(args: argparse.Namespace) -> None: + preview = read_object(args.preview) + spec = None if args.spec is None else ActionSpec.from_payload(read_object(args.spec)) + validate_preview(preview, spec) + write(args.output, {"schema_version": 1, "status": "valid", "preview_id": preview["preview_id"], "preview_fingerprint": preview["fingerprint"]}) + + +def approve(args: argparse.Namespace) -> None: + preview = read_object(args.preview) + write(args.output, build_action_approval(preview, approved_by=args.approved_by, approved_scopes=args.scope)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("validate-spec") + command.add_argument("spec", type=Path) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=validate_spec) + command = commands.add_parser("validate-preview") + command.add_argument("preview", type=Path) + command.add_argument("--spec", type=Path) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=validate_preview_command) + command = commands.add_parser("approve") + command.add_argument("preview", type=Path) + command.add_argument("--approved-by", required=True) + command.add_argument("--scope", action="append", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=approve) + args = parser.parse_args() + try: + args.handler(args) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/describe_artifact.py b/scripts/describe_artifact.py new file mode 100755 index 00000000..7240f78d --- /dev/null +++ b/scripts/describe_artifact.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Create or verify streaming/Merkle descriptors for large scientific artifacts.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from codex_science.artifact_store import ( # noqa: E402 + ContentAddressedStore, + describe_directory, + describe_file, + validate_descriptor, + write_descriptor, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--verify", type=Path, help="Existing descriptor to validate against path") + parser.add_argument("--chunk-size", type=int) + parser.add_argument("--media-type") + parser.add_argument("--store", type=Path, help="Add a file to a content-addressed store") + args = parser.parse_args() + target = args.path.resolve() + if args.verify: + payload = json.loads(args.verify.read_text(encoding="utf-8")) + descriptor = validate_descriptor(payload, target) + elif target.is_dir(): + descriptor = describe_directory(target, media_type=args.media_type) + else: + descriptor = describe_file(target, media_type=args.media_type, chunk_size=args.chunk_size) + if args.output: + write_descriptor(args.output.resolve(), descriptor) + else: + print(json.dumps(descriptor.to_dict(), indent=2, sort_keys=True)) + if args.store: + if not target.is_file(): + raise SystemExit("--store currently accepts files only") + digest, stored = ContentAddressedStore(args.store.resolve()).add_file(target) + print(json.dumps({"sha256": digest, "stored_path": str(stored)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_scientific_run.py b/scripts/export_scientific_run.py new file mode 100644 index 00000000..318faf39 --- /dev/null +++ b/scripts/export_scientific_run.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Export a validated run as RO-Crate-oriented, PROV, and scientific-BOM metadata.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.fair_export import export_run # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--receipt", type=Path, required=True) + args = parser.parse_args() + try: + receipt = export_run(args.manifest, args.output) + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(f"Success! Export written to: {args.output.resolve()}") + print(f"Success! Receipt written to: {args.receipt.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/plan_next_experiment.py b/scripts/plan_next_experiment.py new file mode 100644 index 00000000..454c0659 --- /dev/null +++ b/scripts/plan_next_experiment.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create a deterministic, non-executing multi-objective experiment proposal.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.experiment_planner import plan_next_experiment, validate_experiment_proposal # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + payload = json.loads(args.input.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + proposal = plan_next_experiment(payload) + validate_experiment_proposal(proposal) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(proposal, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(f"Success! Proposal written to: {args.output.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/promote_run_to_skill.py b/scripts/promote_run_to_skill.py new file mode 100644 index 00000000..a2e9bd76 --- /dev/null +++ b/scripts/promote_run_to_skill.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Generate a non-active skill draft from a currently passed scientific run.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.pipeline_compiler import compile_pipeline_draft # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("manifest", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--name", required=True) + parser.add_argument("--description", required=True) + parser.add_argument("--commands", type=Path, help="JSON array of argv arrays") + parser.add_argument("--limitation", action="append", default=[]) + parser.add_argument("--receipt", type=Path, required=True) + args = parser.parse_args() + try: + commands = None + if args.commands: + payload = json.loads(args.commands.read_text(encoding="utf-8")) + if not isinstance(payload, list): + raise ValueError("--commands must contain a JSON array") + commands = payload + receipt = compile_pipeline_draft( + manifest_path=args.manifest, + output_dir=args.output, + name=args.name, + description=args.description, + command_contract=commands, + limitations=args.limitation or None, + ) + args.receipt.parent.mkdir(parents=True, exist_ok=True) + args.receipt.write_text(json.dumps(receipt, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(f"Success! Draft written to: {args.output.resolve()}") + print(f"Success! Receipt written to: {args.receipt.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/query_source_v3.py b/scripts/query_source_v3.py new file mode 100755 index 00000000..83534d07 --- /dev/null +++ b/scripts/query_source_v3.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Execute a true-paginated Connector Contract v3 source query.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from codex_science.connector_contract import QueryRequest # noqa: E402 +from codex_science.source_operations_v3 import SOURCE_OPERATIONS_V3 # noqa: E402 +from codex_science.transport_v3 import execute_paginated # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", choices=sorted(SOURCE_OPERATIONS_V3)) + parser.add_argument("query") + parser.add_argument("--page-size", type=int, default=25) + parser.add_argument("--max-pages", type=int, default=1) + parser.add_argument("--evidence-cutoff") + parser.add_argument("--snapshot-dir", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + request = QueryRequest( + source=args.source, + operation="search", + parameters={"query": args.query}, + page_size=args.page_size, + max_pages=args.max_pages, + evidence_cutoff=args.evidence_cutoff, + source_contract_version="3", + ) + result = execute_paginated( + SOURCE_OPERATIONS_V3[args.source], + request, + snapshot_dir=None if args.snapshot_dir is None else args.snapshot_dir.resolve(), + ) + text = json.dumps(result.to_dict(), indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/reference_lookup.py b/scripts/reference_lookup.py new file mode 100755 index 00000000..d21b8786 --- /dev/null +++ b/scripts/reference_lookup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Select the minimum required skill references and optionally emit use receipts.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from codex_science.reference_contract import ( # noqa: E402 + build_reference_use_receipt, + load_reference_index, + select_references, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("skill_dir", type=Path) + parser.add_argument("--route", default="") + parser.add_argument("--query", default="") + parser.add_argument("--limit", type=int) + parser.add_argument("--receipt-dir", type=Path) + parser.add_argument("--claim", action="append", default=[]) + args = parser.parse_args() + skill_dir = args.skill_dir.resolve() + index = load_reference_index(skill_dir) + if index is None: + raise SystemExit(f"skill has no references/index.json: {skill_dir}") + selected = select_references(index, route=args.route, query=args.query, limit=args.limit) + payload: dict[str, object] = { + "schema_version": 1, + "skill": index.skill, + "selected": [entry.to_dict() for entry in selected], + } + if args.receipt_dir: + args.receipt_dir.mkdir(parents=True, exist_ok=True) + receipts = [] + for entry in selected: + receipt = build_reference_use_receipt( + skill_dir, + entry, + read_reason=args.route or args.query or "selected by reference lookup", + used_for=args.claim or ["workflow-route"], + search_terms=[args.query] if args.query else [], + ) + path = args.receipt_dir / f"{index.skill}--{entry.id}.json" + path.write_text(json.dumps(receipt.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + receipts.append(path.as_posix()) + payload["receipt_paths"] = receipts + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_artifact_runtime.py b/scripts/render_artifact_runtime.py new file mode 100644 index 00000000..9426b5af --- /dev/null +++ b/scripts/render_artifact_runtime.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Describe, render, select, or propose changes to a scientific artifact. + +This CLI always writes structured output to a file. It never mutates the source +artifact and never executes a transform proposal. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.artifact_runtime import ( # noqa: E402 + build_selection, + build_transform_proposal, + describe_runtime, + render_runtime_html, + validate_runtime_descriptor, + validate_selection, +) + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON object required: {path}") + return payload + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Success! Data written to: {path.resolve()}") + + +def _json_argument(value: str, label: str) -> dict[str, Any]: + try: + payload = json.loads(value) + except json.JSONDecodeError as error: + raise ValueError(f"{label} must be valid JSON: {error}") from error + if not isinstance(payload, dict): + raise ValueError(f"{label} must be a JSON object") + return payload + + +def command_describe(args: argparse.Namespace) -> None: + descriptor = describe_runtime( + args.artifact, + artifact_path=args.artifact_path, + artifact_sha256=args.sha256, + kind=args.kind, + artifact_type=args.artifact_type, + media_type=args.media_type, + max_bytes=args.max_bytes, + max_records=args.max_records, + ) + _write_json(args.output, descriptor.to_dict()) + + +def command_render(args: argparse.Namespace) -> None: + descriptor = _read_json(args.descriptor) + validate_runtime_descriptor(descriptor) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render_runtime_html(descriptor, title=args.title), encoding="utf-8") + print(f"Success! Runtime view written to: {args.output.resolve()}") + + +def command_select(args: argparse.Namespace) -> None: + descriptor = _read_json(args.descriptor) + selection = build_selection( + descriptor, + selector_type=args.selector_type, + selector=_json_argument(args.selector, "--selector"), + selected_by=args.selected_by, + reason=args.reason, + label=args.label, + ) + _write_json(args.output, selection) + + +def command_propose(args: argparse.Namespace) -> None: + selection = _read_json(args.selection) + validate_selection(selection) + proposal = build_transform_proposal( + selection, + operation=args.operation, + parameters=_json_argument(args.parameters, "--parameters"), + reason=args.reason, + affected_steps=args.affected_step, + expected_outputs=args.expected_output, + proposed_by=args.proposed_by, + requires_approval=not args.no_approval_required, + approval_boundary=args.approval_boundary, + ) + _write_json(args.output, proposal) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + + describe = subcommands.add_parser("describe", help="Build a bounded runtime descriptor") + describe.add_argument("artifact", type=Path) + describe.add_argument("--artifact-path", required=True, help="Run-relative artifact path") + describe.add_argument("--sha256", help="Expected manifest SHA-256") + describe.add_argument("--kind", required=True) + describe.add_argument("--artifact-type", default="file") + describe.add_argument("--media-type") + describe.add_argument("--max-bytes", type=int, required=True) + describe.add_argument("--max-records", type=int, required=True) + describe.add_argument("--output", type=Path, required=True) + describe.set_defaults(handler=command_describe) + + render = subcommands.add_parser("render", help="Render a validated offline HTML view") + render.add_argument("descriptor", type=Path) + render.add_argument("--title") + render.add_argument("--output", type=Path, required=True) + render.set_defaults(handler=command_render) + + select = subcommands.add_parser("select", help="Create a typed hash-bound selection") + select.add_argument("descriptor", type=Path) + select.add_argument("--selector-type", required=True) + select.add_argument("--selector", required=True, help="JSON object") + select.add_argument("--selected-by", required=True) + select.add_argument("--reason", required=True) + select.add_argument("--label", default="") + select.add_argument("--output", type=Path, required=True) + select.set_defaults(handler=command_select) + + propose = subcommands.add_parser("propose", help="Create a non-executing transform proposal") + propose.add_argument("selection", type=Path) + propose.add_argument("--operation", required=True) + propose.add_argument("--parameters", required=True, help="JSON object") + propose.add_argument("--reason", required=True) + propose.add_argument("--affected-step", action="append", required=True) + propose.add_argument("--expected-output", action="append", default=[]) + propose.add_argument("--proposed-by", required=True) + propose.add_argument("--approval-boundary", default="artifact mutation and rerun") + propose.add_argument("--no-approval-required", action="store_true") + propose.add_argument("--output", type=Path, required=True) + propose.set_defaults(handler=command_propose) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + args.handler(args) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/replay_source_v3.py b/scripts/replay_source_v3.py new file mode 100755 index 00000000..191310bb --- /dev/null +++ b/scripts/replay_source_v3.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Verify and replay raw pages saved by Connector Contract v3.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from codex_science.transport_v3 import replay_snapshot_directory # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("snapshot_dir", type=Path) + parser.add_argument("query_id") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + pages = replay_snapshot_directory(args.snapshot_dir.resolve(), args.query_id) + report = { + "schema_version": 1, + "query_id": args.query_id, + "page_count": len(pages), + "pages": pages, + } + text = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/science_job.py b/scripts/science_job.py new file mode 100644 index 00000000..e378fab5 --- /dev/null +++ b/scripts/science_job.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Submit, inspect, cancel, and collect durable local or Slurm jobs. + +Non-local submission requires an explicit approval receipt produced by the +`approve` subcommand. Scientific completion must still be established by the +run's metrics, claims, provenance, and review. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.compute_backends import ( # noqa: E402 + LocalBackend, + SlurmBackend, + backend_for, + build_approval_receipt, + load_spec, +) + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON object required: {path}") + return payload + + +def _write(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Success! Data written to: {path.resolve()}") + + +def command_preflight(args: argparse.Namespace) -> None: + spec = load_spec(args.spec) + _write(args.output, backend_for(spec.backend, args.state_dir).preflight(spec)) + + +def command_approve(args: argparse.Namespace) -> None: + spec = load_spec(args.spec) + receipt = build_approval_receipt(spec, approved_by=args.approved_by, target=args.target) + _write(args.output, receipt) + + +def command_render_slurm(args: argparse.Namespace) -> None: + spec = load_spec(args.spec) + backend = SlurmBackend(args.state_dir) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(backend.render_script(spec, job_id=args.job_id), encoding="utf-8") + print(f"Success! Slurm script written to: {args.output.resolve()}") + + +def command_submit(args: argparse.Namespace) -> None: + spec = load_spec(args.spec) + approval = None if args.approval is None else _read_json(args.approval) + _write(args.output, backend_for(spec.backend, args.state_dir).submit(spec, approval=approval)) + + +def command_status(args: argparse.Namespace) -> None: + _write(args.output, backend_for(args.backend, args.state_dir).status(args.job_id)) + + +def command_wait(args: argparse.Namespace) -> None: + if args.backend != "local": + raise ValueError("wait is currently supported by the durable local backend; poll Slurm with status") + _write(args.output, LocalBackend(args.state_dir).wait(args.job_id, timeout_seconds=args.timeout, poll_seconds=args.poll)) + + +def command_cancel(args: argparse.Namespace) -> None: + _write(args.output, backend_for(args.backend, args.state_dir).cancel(args.job_id)) + + +def command_collect(args: argparse.Namespace) -> None: + _write(args.output, backend_for(args.backend, args.state_dir).collect(args.job_id)) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state-dir", type=Path, required=True) + subcommands = parser.add_subparsers(dest="command", required=True) + + command = subcommands.add_parser("preflight") + command.add_argument("--spec", type=Path, required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=command_preflight) + + command = subcommands.add_parser("approve") + command.add_argument("--spec", type=Path, required=True) + command.add_argument("--approved-by", required=True) + command.add_argument("--target", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=command_approve) + + command = subcommands.add_parser("render-slurm") + command.add_argument("--spec", type=Path, required=True) + command.add_argument("--job-id", default="preview") + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=command_render_slurm) + + command = subcommands.add_parser("submit") + command.add_argument("--spec", type=Path, required=True) + command.add_argument("--approval", type=Path) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=command_submit) + + for name, handler in (("status", command_status), ("cancel", command_cancel), ("collect", command_collect)): + command = subcommands.add_parser(name) + command.add_argument("--backend", choices=["local", "slurm"], required=True) + command.add_argument("--job-id", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=handler) + + command = subcommands.add_parser("wait") + command.add_argument("--backend", choices=["local", "slurm"], required=True) + command.add_argument("--job-id", required=True) + command.add_argument("--timeout", type=float, required=True) + command.add_argument("--poll", type=float, default=0.2) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=command_wait) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + args.handler(args) + except (OSError, ValueError, TimeoutError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/science_project.py b/scripts/science_project.py new file mode 100644 index 00000000..6b009113 --- /dev/null +++ b/scripts/science_project.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Manage a local project evidence store over immutable Codex Science runs. + +Every subcommand writes its result to --output. The merge-plan command is +non-executing and never mutates imported run bundles. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.project_store import ProjectStore # noqa: E402 + + +def _write(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Success! Data written to: {path.resolve()}") + + +def _json_object(value: str, label: str) -> dict[str, Any]: + try: + payload = json.loads(value) + except json.JSONDecodeError as error: + raise ValueError(f"{label} must be valid JSON: {error}") from error + if not isinstance(payload, dict): + raise ValueError(f"{label} must be a JSON object") + return payload + + +def create(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + _write(args.output, store.create_project(project_id=args.project_id, title=args.title, question=args.question)) + + +def import_run(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + record = store.import_run(project_id=args.project_id, manifest_path=args.manifest, branch_name=args.branch, parent_run_id=args.parent_run) + _write(args.output, record.to_dict()) + + +def fork(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + _write(args.output, store.fork_branch(project_id=args.project_id, source_run_id=args.source_run, branch_name=args.branch)) + + +def assertion(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + payload = store.add_assertion( + project_id=args.project_id, + run_id=args.run_id, + claim_id=args.claim_id, + source_id=args.source_id, + polarity=args.polarity, + locator=_json_object(args.locator, "--locator"), + independence_group=args.independence_group, + effect_measure=args.effect_measure, + estimate=args.estimate, + interval_low=args.interval_low, + interval_high=args.interval_high, + unit=args.unit, + sample_size=args.sample_size, + population=args.population, + risk_of_bias_ref=args.risk_of_bias_ref, + ) + _write(args.output, payload) + + +def compare(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + _write(args.output, store.compare_runs(project_id=args.project_id, previous_run_id=args.previous_run, current_run_id=args.current_run)) + + +def merge_plan(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + _write(args.output, store.prepare_merge_plan(project_id=args.project_id, source_branch=args.source_branch, target_branch=args.target_branch)) + + +def summary(args: argparse.Namespace) -> None: + store = ProjectStore(args.database) + _write(args.output, store.summary(project_id=args.project_id)) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database", type=Path, required=True) + subcommands = parser.add_subparsers(dest="command", required=True) + + command = subcommands.add_parser("init", help="Initialize a project") + command.add_argument("--project-id", required=True) + command.add_argument("--title", required=True) + command.add_argument("--question", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=create) + + command = subcommands.add_parser("import-run", help="Validate and import an immutable run") + command.add_argument("--project-id", required=True) + command.add_argument("--manifest", type=Path, required=True) + command.add_argument("--branch", required=True) + command.add_argument("--parent-run") + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=import_run) + + command = subcommands.add_parser("fork", help="Fork a project branch from an imported run") + command.add_argument("--project-id", required=True) + command.add_argument("--source-run", required=True) + command.add_argument("--branch", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=fork) + + command = subcommands.add_parser("assert", help="Add a quantitative, located evidence assertion") + command.add_argument("--project-id", required=True) + command.add_argument("--run-id", required=True) + command.add_argument("--claim-id", required=True) + command.add_argument("--source-id", required=True) + command.add_argument("--polarity", choices=["supports", "contradicts", "qualifies", "neutral", "unavailable"], required=True) + command.add_argument("--locator", required=True, help="JSON object with artifact path/hash and an exact locator") + command.add_argument("--independence-group", required=True) + command.add_argument("--effect-measure") + command.add_argument("--estimate", type=float) + command.add_argument("--interval-low", type=float) + command.add_argument("--interval-high", type=float) + command.add_argument("--unit") + command.add_argument("--sample-size", type=int) + command.add_argument("--population") + command.add_argument("--risk-of-bias-ref") + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=assertion) + + command = subcommands.add_parser("compare", help="Compare two imported run manifests") + command.add_argument("--project-id", required=True) + command.add_argument("--previous-run", required=True) + command.add_argument("--current-run", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=compare) + + command = subcommands.add_parser("merge-plan", help="Prepare a non-executing scientific merge plan") + command.add_argument("--project-id", required=True) + command.add_argument("--source-branch", required=True) + command.add_argument("--target-branch", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=merge_plan) + + command = subcommands.add_parser("summary", help="Summarize project lineage and event-chain state") + command.add_argument("--project-id", required=True) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=summary) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + args.handler(args) + except (OSError, ValueError, sqlite3.Error if False else ValueError) as error: # type: ignore[misc] + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/science_reviewer.py b/scripts/science_reviewer.py new file mode 100644 index 00000000..a36528a6 --- /dev/null +++ b/scripts/science_reviewer.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Prepare a blinded review packet or finalize a hash-covered review response.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from codex_science.reviewer_runtime import build_review_packet, finalize_review_response # noqa: E402 + + +def read_object(path: Path) -> dict: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON object required: {path}") + return payload + + +def write(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Success! Data written to: {path.resolve()}") + + +def prepare(args: argparse.Namespace) -> None: + packet = build_review_packet( + args.manifest, + review_modes=args.mode, + independent_required=not args.independence_not_required, + review_questions=args.question or None, + ) + write(args.output, packet) + + +def finalize(args: argparse.Namespace) -> None: + packet = read_object(args.packet) + response = read_object(args.response) + write(args.output, finalize_review_response(packet, response)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("prepare") + command.add_argument("manifest", type=Path) + command.add_argument("--mode", action="append", choices=["record", "source", "method", "reproduction"], required=True) + command.add_argument("--question", action="append", default=[]) + command.add_argument("--independence-not-required", action="store_true") + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=prepare) + command = commands.add_parser("finalize") + command.add_argument("packet", type=Path) + command.add_argument("response", type=Path) + command.add_argument("--output", type=Path, required=True) + command.set_defaults(handler=finalize) + args = parser.parse_args() + try: + args.handler(args) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_connector_contracts.py b/scripts/validate_connector_contracts.py index aab7b124..f70a1228 100755 --- a/scripts/validate_connector_contracts.py +++ b/scripts/validate_connector_contracts.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -"""Validate Connector Contract v2, source registry, and offline replay semantics.""" +"""Validate Connector Contracts v2/v3, source registries, pagination, and replay.""" from __future__ import annotations import argparse +import json import sys +import tempfile from pathlib import Path +from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) @@ -13,9 +16,37 @@ from codex_science.connector_sources import SOURCE_BY_KEY, SOURCE_BY_TOOL, SOURCE_SPECS # noqa: E402 from codex_science.connectors import PubMedConnector # noqa: E402 from codex_science.mcp_server import CONNECTOR_SPECS # noqa: E402 +from codex_science.source_operations_v3 import PubMedSearchV3, SOURCE_OPERATIONS_V3 # noqa: E402 +from codex_science.transport_v3 import PageRequest, TransportResponse, execute_paginated, replay_snapshot_directory # noqa: E402 from codex_science.typed_connectors import ClinVarConnector, GnomADConnector, VersionedSnapshotConnector # noqa: E402 +class FixtureTransport: + def __init__(self, payloads: list[dict[str, Any]]) -> None: + self.payloads = list(payloads) + self.requests: list[PageRequest] = [] + + def send(self, request: PageRequest) -> TransportResponse: + self.requests.append(request) + payload = self.payloads.pop(0) + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return TransportResponse( + status_code=200, + url=request.url, + headers={ + "content-type": "application/json", + "etag": f'"fixture-{len(self.requests)}"', + "last-modified": "Sun, 19 Jul 2026 00:00:00 GMT", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + }, + body=body, + media_type="application/json", + retrieved_at="2026-07-19T00:00:00Z", + attempts=1, + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=ROOT) @@ -30,6 +61,8 @@ def main() -> int: raise SystemExit("legacy connector registry must preserve 34 entries") if set(keys) != set(SOURCE_BY_KEY) or set(tools) != set(SOURCE_BY_TOOL): raise SystemExit("connector source registry indexes are stale") + if set(SOURCE_OPERATIONS_V3) != {"pubmed", "europepmc", "chembl", "pdb"}: + raise SystemExit("connector v3 operation registry is stale") def pubmed_payload(_url: str) -> dict: return { @@ -47,9 +80,36 @@ def pubmed_payload(_url: str) -> dict: snapshot = result.snapshot() replayed = replay_snapshot(snapshot) if replayed.records != result.records: - raise SystemExit("connector snapshot replay changed normalized records") + raise SystemExit("connector v2 snapshot replay changed normalized records") if classify_drift(snapshot, snapshot)["drift_types"] != ["none"]: - raise SystemExit("identical connector snapshots must not drift") + raise SystemExit("identical connector v2 snapshots must not drift") + + v3_request = QueryRequest( + "pubmed", + "search", + {"query": "protein folding"}, + page_size=2, + max_pages=3, + source_contract_version="3", + ) + fixture = FixtureTransport( + [ + {"esearchresult": {"count": "3", "retstart": "0", "idlist": ["1", "2"]}}, + {"esearchresult": {"count": "3", "retstart": "2", "idlist": ["3"]}}, + ] + ) + with tempfile.TemporaryDirectory() as tempdir: + snapshot_dir = Path(tempdir) + v3_result = execute_paginated( + PubMedSearchV3(), v3_request, transport=fixture, snapshot_dir=snapshot_dir + ) + if v3_result.receipt.status != "complete" or len(v3_result.records) != 3: + raise SystemExit("connector v3 true pagination failed") + if v3_result.receipt.pages[0].etag != '"fixture-1"': + raise SystemExit("connector v3 did not preserve ETag") + replayed_pages = replay_snapshot_directory(snapshot_dir, v3_request.query_id) + if len(replayed_pages) != 2: + raise SystemExit("connector v3 raw snapshot replay failed") clinvar = ClinVarConnector(fetch_json=lambda _url: {"esearchresult": {"idlist": ["42"]}}) if clinvar.search("BRCA1", limit=1)[0]["id"] != "42": @@ -73,7 +133,10 @@ def pubmed_payload(_url: str) -> dict: if not release: raise SystemExit("versioned eQTL release snapshot is unavailable") - print(f"connector contracts: valid ({len(SOURCE_SPECS)} sources, 34 legacy tools)") + print( + f"connector contracts: valid ({len(SOURCE_SPECS)} sources, " + "34 legacy tools, 4 v3 true-pagination operations)" + ) return 0 diff --git a/skills/codex-science/SKILL.md b/skills/codex-science/SKILL.md index 85a66e2d..8db41013 100644 --- a/skills/codex-science/SKILL.md +++ b/skills/codex-science/SKILL.md @@ -36,6 +36,14 @@ While this mode is active, prefer Codex Science MCP tools and `status: active` c Fall back to general-purpose tools or skills only when no suitable Science capability exists, the Science capability is unavailable or fails, or a higher-priority instruction requires another route. This preference does not override the activation gate, the approved plan, user instructions, project rules, or safety policy; never select an inactive skill silently. +## Reference usage + +After selecting a skill, inspect its `references/index.json` when present and load only the entries required by the active route. Read [the reference-loading protocol](references/reference-loading.md) before any reference-controlled operation; do not infer arguments, output schemas, source semantics, thresholds, or licenses from names alone. Read [the catalog policy](references/catalog-policy.md) before selecting or inspecting an inactive catalog skill. Read [the Goal-loop contract](references/goal-loop-contract.md) before starting, resuming, or completing a native Goal run. + +Read [the project evidence store contract](references/project-evidence-store.md) before managing multi-run lineage, hypotheses, comparisons, or merge plans. Read [the pipeline promotion contract](references/pipeline-promotion.md) before generating a reusable draft from a reviewed run, and [the next-experiment planning contract](references/next-experiment-planning.md) before ranking a proposed batch. Read [the authenticated connector pack contract](references/authenticated-connector-packs.md) before designing, approving, or using a credentialed or write-capable provider action. + +When a reference materially controls a query, command, transformation, threshold, or claim, record its path, SHA-256, read reason, and affected claim or artifact in a `reference-use-ledger`. A reference explains procedure; the actual source record or computation remains the scientific evidence. + ## Research contract and evidence graph For every non-trivial run, create a compact contract before retrieval or execution: diff --git a/skills/codex-science/quality.json b/skills/codex-science/quality.json new file mode 100644 index 00000000..5f81d86b --- /dev/null +++ b/skills/codex-science/quality.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "skill": "codex-science", + "roles": ["conductor"], + "declared_maturity": "L3", + "output_schemas": [ + "research-contract-v1", + "checkpoint-v4", + "artifact-manifest-v1", + "lane-receipt-v1", + "reference-use-ledger-v1", + "pipeline-promotion-v1", + "experiment-proposal-v1", + "connector-action-spec-v1", + "connector-action-preview-v1", + "connector-action-approval-v1", + "connector-action-receipt-v1" + ], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [ + "The coordinator delegates domain validity to specialist skills and independent review.", + "Task persistence depends on the installed Codex host and trusted hooks." + ] +} diff --git a/skills/codex-science/references/authenticated-connector-packs.md b/skills/codex-science/references/authenticated-connector-packs.md new file mode 100644 index 00000000..cd7fe664 --- /dev/null +++ b/skills/codex-science/references/authenticated-connector-packs.md @@ -0,0 +1,279 @@ +# Optional authenticated and write-capable connector packs + +Read this reference before designing, installing, previewing, approving, or executing a connector that accesses authenticated laboratory, clinical, cloud, registry, LIMS/ELN, workflow, or storage services. + +## Core boundary + +Codex Science core remains local-first and read-only for public discovery. Provider credentials and provider-specific write implementations belong in separately installed optional packs. + +An optional pack must implement the `ConnectorActionAdapter` contract: + +```python +class ConnectorActionAdapter(Protocol): + connector_name: str + + def snapshot(self, spec: ActionSpec) -> Mapping[str, Any]: ... + def preview(self, spec: ActionSpec, before: Mapping[str, Any]) -> Mapping[str, Any]: ... + def execute(self, spec: ActionSpec, preview: Mapping[str, Any]) -> Mapping[str, Any]: ... +``` + +Credentials are supplied out of band by the host or connector runtime. They must never appear in action specs, previews, approvals, logs, receipts, artifact bundles, or prompts. + +## Action spec + +```json +{ + "schema_version": 1, + "connector": "benchling-pack", + "operation": "create-assay-result", + "mode": "write", + "target": "tenant/project/experiment", + "parameters": { + "schema_id": "assay-result-v2", + "record": { + "sample_id": "sample-42", + "value": 0.72, + "unit": "fraction" + } + }, + "requested_scopes": [ + "assay:read", + "assay:write" + ], + "idempotency_key": "run-014-assay-result-sample-42-v1", + "expected_before_sha256": null, + "approval_required": true, + "destructive": false, + "paid": false, + "sensitive_data": true, + "compensation": "Archive the created result through a separately approved action." +} +``` + +### Required rules + +- `mode`: `read` or `write`; +- every write requires approval; +- destructive actions must use write mode; +- target and operation are explicit; +- parameters contain no credentials; +- requested OAuth/provider scopes are explicit; +- idempotency key is stable and unique to one semantic action; +- expected before-state hash is used when the caller already knows the remote state; +- cost, sensitivity, and compensation are explicit. + +The core validator rejects secret-like parameter keys such as token, password, secret, credential, API key, access key, private key, or client secret. + +## Validate a spec + +```bash +python scripts/connector_action_contract.py validate-spec \ + artifacts/run-014/action-spec.json \ + --output artifacts/run-014/action-spec.validated.json +``` + +Validation returns the canonical spec and `action_spec_sha256`. It does not contact the provider. + +## Provider preview + +The provider adapter calls: + +```python +preview = build_preview(adapter, spec) +``` + +The adapter must obtain a minimal safe current-state snapshot and proposed changes. Preview output includes: + +```json +{ + "preview_id": "preview-...", + "action_spec_sha256": "...", + "idempotency_key": "...", + "before_state_sha256": "...", + "before_state": {}, + "proposed_changes": {}, + "requested_scopes": [], + "approval_required": true, + "destructive": false, + "paid": false, + "sensitive_data": true, + "status": "preview", + "executed": false, + "fingerprint": "..." +} +``` + +The adapter should minimize `before_state`; do not include full private records when a version, schema, or selected fields are sufficient. + +Validate a provider-created preview: + +```bash +python scripts/connector_action_contract.py validate-preview \ + artifacts/run-014/action-preview.json \ + --spec artifacts/run-014/action-spec.json \ + --output artifacts/run-014/action-preview.validated.json +``` + +## Approval + +```bash +python scripts/connector_action_contract.py approve \ + artifacts/run-014/action-preview.json \ + --approved-by jaemin \ + --scope assay:read \ + --scope assay:write \ + --output artifacts/run-014/action-approval.json +``` + +Approval binds: + +- exact preview ID and fingerprint; +- exact action spec SHA-256; +- target and operation; +- before-state SHA-256; +- idempotency key; +- approved scopes; +- approver attestation and time. + +Approved scopes must cover requested scopes. A changed preview or before-state requires new approval. + +Approval authorizes the provider action boundary. It does not establish scientific validity, consent, privacy compliance, regulatory permission, affordability, or downstream interpretation. + +## Execute through an optional pack + +A provider pack calls: + +```python +receipt = execute_action( + adapter, + spec, + preview, + approval=approval, + ledger=ActionLedger(Path(".connector-actions.sqlite")), +) +``` + +Execution procedure: + +1. validate spec and preview; +2. resolve the exact approval for writes; +3. check the idempotency ledger; +4. obtain a fresh current-state snapshot; +5. compare its hash with preview before-state; +6. reject state drift; +7. call the provider adapter once; +8. record after-state hash, remote object IDs, provider receipt, compensation, and fingerprint; +9. persist the idempotency receipt; +10. add the action receipt to run provenance when material. + +A repeated request with the same idempotency key and same spec returns the original receipt without another provider call. Reusing the key for a different spec fails. + +## Provider pack requirements + +Each optional pack should contain: + +```text +plugin manifest +provider-specific skill +references/ + authentication.md + read-operations.md + write-operations.md + schemas.md + failure-recovery.md +adapter implementation +fixture adapter +contract tests +scope policy +license and data-use notes +``` + +Provider references must specify exact operation names, parameter schemas, pagination, remote object IDs, rate limits, idempotency behavior, write preview, compensation, and data retention. + +### Authentication + +- OAuth/device flow or host-managed secret store preferred; +- minimum scopes; +- separate read and write scopes; +- no credentials in command arguments when a secure host channel exists; +- token refresh and revocation behavior documented; +- test/sandbox tenant supported where provider offers one. + +### Write safety + +- preview before execution; +- exact target; +- optimistic concurrency or version check; +- idempotency; +- explicit destructive and paid flags; +- before/after state hashes; +- remote IDs; +- compensation or explanation when rollback is impossible; +- provider request ID; +- immutable audit receipt. + +### Sensitive data + +- minimize transferred fields; +- record data classification; +- obtain explicit approval; +- enforce provider region/tenant policy; +- avoid copying raw participant data into artifacts; +- store approved immutable references and hashes instead; +- document retention and deletion expectations. + +## Recommended pack separation + +Examples: + +```text +codex-science-benchling +codex-science-protocols +codex-science-terra +codex-science-modal +codex-science-lamindb +codex-science-redcap +``` + +These names describe optional architecture only. Core must not claim a provider is supported until a pack exists, its API version is pinned, and its read/write fixtures pass. + +## Search patterns + +- `## Action spec` +- `## Provider preview` +- `## Approval` +- `## Execute through an optional pack` +- `## Provider pack requirements` +- `## Failure handling` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| credential appears in spec or preview | reject and rotate the credential if exposed | +| requested scope not approved | obtain a new approval or reduce the action | +| remote state changed after preview | create a new snapshot, preview, and approval | +| idempotency key reused for different spec | create a new semantic key; do not overwrite ledger history | +| provider 409/version conflict | refresh state and return a new preview | +| provider 429 | bounded retry only when provider policy allows; preserve attempt receipt | +| provider 5xx | preserve failure and retry through a new attempt policy | +| destructive action has no compensation | make irreversibility explicit before approval | +| remote object ID missing | treat execution receipt as incomplete | +| sensitive data exceeds scope | stop transfer and revise the approved data boundary | +| provider schema/version drift | disable the operation until fixtures and references are updated | + +## Common mistakes + +- Putting tokens in action parameters. +- Letting a plugin write without preview. +- Treating a human approval message as approval for a changed spec. +- Omitting requested scopes. +- Reusing idempotency keys across semantically different actions. +- Logging full sensitive before/after records. +- Assuming provider success means the scientific value is correct. +- Calling a compensating action an automatic rollback when it requires separate approval. +- Shipping authenticated write operations in the public read-only core. + +## Evidence boundary + +The connector action framework standardizes preview, approval, idempotency, concurrency, and receipts. It does not provide provider credentials, implement a provider API by itself, establish scientific validity, or certify privacy, legal, or regulatory compliance. diff --git a/skills/codex-science/references/index.json b/skills/codex-science/references/index.json new file mode 100644 index 00000000..93bb4ff2 --- /dev/null +++ b/skills/codex-science/references/index.json @@ -0,0 +1,116 @@ +{ + "default_policy": "load-minimum-required", + "references": [ + { + "authority": "repository-contract", + "evidence_boundary": "Controls procedural grounding; it is not scientific evidence.", + "id": "reference-loading", + "path": "references/reference-loading.md", + "purpose": "Select, load, and receipt detailed skill references without flooding context or guessing undocumented arguments.", + "read_when": [ + "a selected skill contains references/index.json", + "a route depends on detailed source, command, benchmark, or backend semantics" + ], + "required_before": [ + "selected-skill-reference-loading", + "material-reference-controlled-operation" + ], + "search_patterns": [ + "## 2. Load the minimum required material", + "## 3. Read before acting", + "## 5. Record material use" + ], + "version": "1" + }, + { + "authority": "repository-contract", + "evidence_boundary": "Controls task lifecycle only; it does not attest scientific completion.", + "id": "goal-loop-contract", + "path": "references/goal-loop-contract.md", + "purpose": "Operate native Goal mode and checkpoint lifecycle without creating conflicting continuation loops.", + "read_when": [ + "starting or resuming a native Goal run", + "explaining completion_pending or Goal confirmation" + ], + "required_before": [ + "native-goal-start", + "native-goal-resume", + "native-goal-completion" + ], + "search_patterns": [ + "completion_pending", + "confirm-goal-complete", + "Stop" + ], + "version": "4" + }, + { + "authority": "repository-contract", + "evidence_boundary": "Controls catalog authorization; it does not authorize credentials, installs, writes, paid compute, or imported code execution.", + "id": "catalog-policy", + "path": "references/catalog-policy.md", + "purpose": "Interpret active/inactive catalog status, audit reasons, and the boundary between inspection acknowledgement and execution approval.", + "read_when": [ + "selecting a catalog skill", + "inspecting an inactive skill", + "explaining an activation reason" + ], + "required_before": [ + "inactive-skill-inspection", + "catalog-activation-decision" + ], + "search_patterns": [ + "## Status meaning", + "## Inactive reasons", + "Explicit acknowledgement" + ], + "version": "1" + }, + { + "id": "authenticated-connector-packs", + "path": "references/authenticated-connector-packs.md", + "purpose": "Approval-gated optional connector packs with out-of-band credentials, preview, scopes, idempotency, concurrency, and before/after receipts.", + "read_when": ["before installing or designing an authenticated connector pack", "before previewing or executing a provider write", "before transferring sensitive data to an external service"], + "required_before": ["issuing a connector action approval", "executing through ConnectorActionAdapter", "claiming a provider write integration is safe"], + "search_patterns": ["## Action spec", "## Provider preview", "## Approval", "## Execute through an optional pack", "## Failure handling"], + "authority": "Codex Science first-party optional connector contract", + "version": "1", + "evidence_boundary": "The framework standardizes action safety; it does not supply credentials, provider implementations, or compliance certification." + }, + { + "id": "next-experiment-planning", + "path": "references/next-experiment-planning.md", + "purpose": "Deterministic Pareto, uncertainty, control, diversity, budget, and batch planning for a non-executing experiment proposal.", + "read_when": ["before ranking candidates for a new experiment", "before using model uncertainty or multi-objective utility for selection"], + "required_before": ["calling scripts/plan_next_experiment.py", "treating a selected batch as an approved experiment"], + "search_patterns": ["## Input contract", "## Selection method", "## Output contract", "## Failure handling"], + "authority": "Codex Science first-party experiment proposal contract", + "version": "1", + "evidence_boundary": "The planner ranks declared properties; it does not calibrate uncertainty, infer similarity, or execute experiments." + }, + { + "id": "pipeline-promotion", + "path": "references/pipeline-promotion.md", + "purpose": "Generate a non-active reusable skill draft from a currently passed run without transferring conclusions or review receipts.", + "read_when": ["before converting a reviewed run into a reusable workflow", "before generating a skill draft from artifacts"], + "required_before": ["calling scripts/promote_run_to_skill.py", "adding a generated draft to authored-skills or a catalog"], + "search_patterns": ["## Preconditions", "## Generated draft", "## Promotion path", "## Failure handling"], + "authority": "Codex Science first-party promotion contract", + "version": "1", + "evidence_boundary": "Promotion preserves workflow shape; it does not activate a skill or prove generalization." + }, + { + "id": "project-evidence-store", + "path": "references/project-evidence-store.md", + "purpose": "Project-level immutable run imports, evidence assertions, hypothesis forks, comparisons, and non-executing merge plans.", + "read_when": ["before importing more than one run into a project", "before creating a hypothesis or experiment branch", "before recording a quantitative evidence assertion", "before comparing or merging scientific branches"], + "required_before": ["calling scripts/science_project.py", "treating project-level state as scientific lineage"], + "search_patterns": ["## Import a reviewed run", "## Fork a hypothesis branch", "## Add a quantitative evidence assertion", "## Prepare a merge plan", "## Failure handling"], + "authority": "Codex Science first-party project lineage contract", + "version": "1", + "evidence_boundary": "The project store indexes immutable runs and decisions; it does not replace manifests, evidence, or independent review." + } + ], + "schema_version": 1, + "skill": "codex-science" +} diff --git a/skills/codex-science/references/next-experiment-planning.md b/skills/codex-science/references/next-experiment-planning.md new file mode 100644 index 00000000..44beb28c --- /dev/null +++ b/skills/codex-science/references/next-experiment-planning.md @@ -0,0 +1,239 @@ +# Multi-objective next-experiment planning + +Read this reference before ranking candidates for a new assay, synthesis batch, simulation, data collection round, or other experiment. The planner produces a deterministic proposal from declared values and constraints. It does not execute an experiment or infer valid uncertainty, similarity, or information gain from a model label. + +## Input contract + +```json +{ + "schema_version": 1, + "decision": "Select the next 24 compounds.", + "objectives": [ + { + "name": "potency", + "direction": "maximize", + "weight": 0.5, + "required": true + }, + { + "name": "solubility", + "direction": "maximize", + "weight": 0.3, + "required": true + }, + { + "name": "off_target_risk", + "direction": "minimize", + "weight": 0.2, + "required": true + } + ], + "candidates": [ + { + "id": "compound-001", + "properties": { + "potency": 8.2, + "solubility": 0.6, + "off_target_risk": 0.2 + }, + "cost": 450, + "uncertainty": 0.7, + "diversity_group": "scaffold-A", + "control": false, + "eligible": true, + "metadata": { + "source_artifact": "candidate-table.parquet", + "source_record": "compound-001" + } + } + ], + "constraints": { + "batch_size": 24, + "budget": 12000, + "diversity_group_cap": 4, + "minimum_controls": 2 + }, + "uncertainty_weight": 0.25, + "diversity_bonus": 0.1, + "claim_ids": ["C-potency", "C-selectivity"] +} +``` + +### Objectives + +- `direction`: `maximize` or `minimize`; +- `weight`: non-negative; at least one weight must be positive; +- `required`: every eligible candidate must contain that property. + +The planner performs within-contract min-max normalization. Therefore scores from different input batches are not directly comparable unless their candidate population and scaling contract are identical. + +### Candidates + +Required: + +- stable candidate ID; +- numeric properties; +- non-negative cost; +- non-negative uncertainty; +- diversity group. + +Optional: + +- `control`; +- eligibility and exclusion reason; +- source metadata. + +`diversity_group` is an explicit category supplied by the workflow. It may represent scaffold, target family, donor, site, assay plate, or another prespecified grouping. The planner does not derive chemical or biological similarity. + +### Constraints + +- `batch_size`: maximum selected records; +- `budget`: total declared cost cap; +- `diversity_group_cap`: maximum selected records per group; +- `minimum_controls`: required eligible controls. + +Required controls are selected before optimization. If they cannot fit the batch, budget, or diversity cap, the proposal fails rather than silently removing them. + +## CLI + +```bash +python scripts/plan_next_experiment.py \ + artifacts/run-020/next-experiment-input.json \ + --output artifacts/run-020/next-experiment.proposal.json +``` + +The proposal is deterministic when the full input, including time field supplied by the API, is unchanged. + +## Selection method + +The current deterministic policy performs: + +1. eligibility filtering; +2. objective min-max normalization with minimize directions inverted; +3. Pareto-front construction; +4. mandatory control selection; +5. weighted objective utility; +6. declared uncertainty bonus; +7. explicit diversity-group bonus; +8. greedy selection subject to budget, batch, and group cap; +9. explicit rejection reasons. + +The utility score is a ranking instrument inside the proposal, not a calibrated expected outcome. + +## Output contract + +```json +{ + "schema_version": 1, + "proposal_id": "experiment-...", + "decision": "...", + "status": "proposed", + "executed": false, + "selected": [], + "rejected": [], + "pareto_fronts": [], + "total_cost": 11500, + "remaining_budget": 500, + "diversity_groups": {}, + "expected_information_gain_proxy": 0.42, + "required_controls_satisfied": true, + "approval_required": true, + "claim_ids": [], + "evidence_boundary": "...", + "fingerprint": "..." +} +``` + +Every selected record retains: + +- original declared properties; +- normalized objective values; +- uncertainty score; +- Pareto front; +- base utility; +- cost, control state, and diversity group. + +Every rejected record retains one or more reasons such as: + +```text +ineligible +reactive-substructure +budget +diversity-group-cap +not-selected-within-batch +``` + +Do not remove rejected candidates or reasons from the saved proposal. + +## Required workflow + +1. Define the decision and claims before candidate scoring. +2. Validate candidate identities and source artifacts. +3. Confirm objective direction, units, and applicability. +4. Validate uncertainty calibration separately; use zero uncertainty weight if unvalidated. +5. Define controls and diversity groups before outcome inspection. +6. Set budget and batch constraints. +7. Generate the proposal. +8. Inspect Pareto fronts, selected controls, repeated groups, costs, and rejection reasons. +9. Run domain-specific safety and feasibility filters that are not encoded in this generic planner. +10. Obtain experimental, remote, paid, synthesis, or write approval. +11. Save the approved experiment package and execute through the appropriate system. +12. Ingest results without dropping failed synthesis, assay failure, or missing outcomes. +13. Update claims and project evidence memory. +14. Review selection bias and whether the next round remains comparable. + +## Applicability examples + +### Compound selection + +Use declared scaffold or series as `diversity_group`. Candidate properties may include potency, selectivity, solubility, permeability, synthesis confidence, novelty, or model uncertainty. Do not treat docking score as experimental potency. + +### Single-cell follow-up + +Use donor, condition, or cluster as diversity group. Include donor balance and controls explicitly. Do not select cells as independent biological replicates when donors are the experimental unit. + +### Literature evidence acquisition + +Candidates may be full texts, datasets, or experiments. Cost may be retrieval or curation effort. Uncertainty must describe the decision-relevant evidence gap, not search-engine score. + +### Simulation allocation + +Candidates may be receptor states, ligand microstates, initial conditions, or parameter regimes. Use cost and resource caps; do not call multiple seeds independent experimental evidence. + +## Search patterns + +- `## Input contract` +- `## Selection method` +- `## Output contract` +- `## Required workflow` +- `## Failure handling` +- `## Common mistakes` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| required property missing | stop; repair the candidate table or mark the objective non-required with rationale | +| controls unavailable | redesign the batch; do not proceed without the declared controls | +| controls exceed budget/batch | revise the approved experiment contract | +| uncertainty not calibrated | set uncertainty weight to zero or treat it as exploratory only | +| diversity grouping unavailable | define a reviewed grouping or set a conservative cap of one per candidate | +| costs incomparable | convert to one declared unit or separate budgets | +| no candidate fits | return an empty/infeasible decision record; do not relax constraints silently | +| candidate excluded after proposal | preserve the original proposal and create a revision/diff | +| source properties changed | invalidate the proposal fingerprint and regenerate | + +## Common mistakes + +- Calling the utility score predicted efficacy. +- Treating the information-gain proxy as a calibrated expectation. +- Allowing model uncertainty to dominate without calibration. +- Deriving diversity from names instead of an explicit method. +- Omitting controls to fit more candidates. +- Counting samples, cells, conformers, or seeds as independent units incorrectly. +- Changing objective weights after inspecting the selected list without preserving a revision. +- Executing the proposal without feasibility, safety, cost, and write approval. + +## Evidence boundary + +The planner makes multi-objective constraints and selection bias inspectable. It does not validate candidate measurements, infer similarity, calibrate uncertainty, estimate real information gain, execute experiments, or prove that the selected batch is scientifically optimal. diff --git a/skills/codex-science/references/pipeline-promotion.md b/skills/codex-science/references/pipeline-promotion.md new file mode 100644 index 00000000..408dc2e1 --- /dev/null +++ b/skills/codex-science/references/pipeline-promotion.md @@ -0,0 +1,175 @@ +# Promote a reviewed run to a reusable skill draft + +Read this reference before converting a completed run into a reusable workflow. Promotion generates a draft only. It never activates a skill, changes the audited catalog, copies private inputs, or turns one successful run into evidence of generalization. + +## Preconditions + +The source run must satisfy all of the following: + +- complete, hash-valid artifact bundle; +- all recorded plan steps completed; +- manifest review status `passed`; +- current deterministic record review also `passed`; +- source manifest bytes available and immutable; +- no intent to copy credentials, private prompts, sensitive raw data, or proprietary inputs; +- commands represented as argv arrays rather than reconstructed shell prose. + +A run with findings, blocked work, missing outputs, stale receipts, or incomplete plan steps cannot be promoted. Preserve it as failed or inconclusive. + +## CLI + +```bash +python scripts/promote_run_to_skill.py \ + artifacts/run-014/manifest.json \ + drafts/receptor-state-docking \ + --name receptor-state-docking \ + --description "Run the reviewed receptor-state docking workflow on new approved inputs." \ + --commands artifacts/run-014/promoted-commands.json \ + --limitation "Validated only on the checked acceptance fixture." \ + --limitation "External docking engine and databases must be repinned." \ + --receipt drafts/receptor-state-docking.promotion.json +``` + +`--commands` contains a JSON array of argv arrays: + +```json +[ + ["python", "prepare_receptor.py", "--input", "receptor.cif", "--output", "prepared.pdbqt"], + ["vina", "--config", "docking.conf", "--out", "poses.pdbqt"], + ["python", "validate_poses.py", "--poses", "poses.pdbqt", "--output", "metrics.json"] +] +``` + +Do not place tokens, passwords, shell redirections, pipes, or unreviewed remote commands in this file. Create separate reviewed steps when piping or staging is scientifically material. + +## Generated draft + +```text +drafts/receptor-state-docking/ + SKILL.md + input.schema.json + output.schema.json + promotion.json + references/ + index.json + pipeline.md +``` + +### `SKILL.md` + +The draft includes: + +- new decision contract; +- mandatory progressive reference; +- source workflow shape; +- provenance and review handoffs; +- explicit non-activation and non-generalization boundaries. + +It must not contain the source conclusion as the new answer. + +### `input.schema.json` + +The new invocation must provide: + +```json +{ + "question": "...", + "parameters": {}, + "inputs": [ + { + "id": "receptor", + "path": "receptor.cif", + "sha256": "..." + } + ], + "acceptance": { + "criteria": [], + "review_required": true + } +} +``` + +An input needs an ID and either a local path or canonical external identifier. Add checksum, source, and release when practical. The source run's local input paths are retained only as historical metadata in the generated schema and must not be reused blindly. + +### `output.schema.json` + +A successful new invocation must return: + +- new run ID; +- new manifest path and SHA-256; +- current claims and artifacts; +- current passed review; +- current limitations. + +A source run, process exit receipt, or copied report does not satisfy the output schema. + +### `promotion.json` + +```json +{ + "status": "draft", + "activated": false, + "cataloged": false, + "source_run_id": "run-014", + "source_manifest_sha256": "...", + "required_next_steps": [], + "fingerprint": "..." +} +``` + +The receipt binds the source manifest hash, generated skill name, command count, source artifacts, limitations, and required promotion steps. + +## Promotion path + +A generated draft starts below active maturity. Before catalog activation: + +1. replace source-local assumptions with explicit input and environment contracts; +2. add a redistributable acceptance fixture; +3. seed material failure classes; +4. add deterministic tests; +5. add drift handling for sources, models, weights, and databases; +6. run the fixture in a clean environment; +7. run independent record, method, source, and reproduction review as applicable; +8. add an honest `quality.json`; +9. pass reference, maturity, inventory, wrapper, plugin, and scientific contract checks; +10. activate only through the repository catalog policy. + +One source run is not enough for L4 unless the generated draft also has explicit acceptance and seeded-defect coverage. + +## Search patterns + +Use these exact searches when reading the generated `references/pipeline.md`: + +- `## Source record` +- `## Environment` +- `## Command contract` +- `## Output contract` +- `## Validation and promotion` +- `## Failure handling` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| source review no longer passes | stop; do not generate or use the draft | +| source manifest hash changed | restore the reviewed bytes or use a new source run | +| command references unavailable local paths | parameterize the path and validate in the target environment | +| model/database revision unavailable | repin a reviewed replacement and create a new acceptance result | +| private input dependency | replace with an approved immutable reference or remove it | +| output schema mismatch | preserve the failed run and repair code/schema | +| source result fails to reproduce | report non-reproduction; do not weaken thresholds post hoc | +| output directory non-empty | choose a new empty directory; never overwrite an existing skill draft | + +## Common mistakes + +- Adding the generated directory directly to `authored-skills/` without review. +- Copying private source data into a fixture. +- Treating source commands as portable without environment and revision checks. +- Reusing source review receipts. +- Omitting failed or abandoned source steps. +- Generating a broad trigger description from a narrow fixture. +- Claiming the compiler has validated scientific generalization. + +## Evidence boundary + +The compiler preserves reviewed workflow structure and source lineage. It does not validate new inputs, activate the draft, ensure portability, establish model or database licensing, or prove that the method generalizes. diff --git a/skills/codex-science/references/project-evidence-store.md b/skills/codex-science/references/project-evidence-store.md new file mode 100644 index 00000000..9c7b5731 --- /dev/null +++ b/skills/codex-science/references/project-evidence-store.md @@ -0,0 +1,306 @@ +# Project evidence store and experiment forks + +Read this reference before importing multiple scientific runs into one project, creating a hypothesis branch, recording a quantitative evidence assertion, comparing runs, or preparing a merge plan. + +## Authority model + +The SQLite project store is an index and append-only decision ledger. It is **not** the source of truth for scientific bytes. + +Authority order: + +1. run-local artifact bytes and their SHA-256 values; +2. run-local `manifest.json`; +3. hash-covered sidecars and review receipts; +4. the project evidence store's imported hashes and lineage index; +5. derived project summaries, comparisons, and merge plans. + +If an imported manifest changes on disk, project comparison stops. Re-import it under a new run ID or restore the original bytes. Never update an existing run row to point at changed manifest bytes. + +## Data model + +The store records: + +- projects; +- immutable imported runs; +- branch base and head runs; +- run artifact hashes and claim records; +- quantitative evidence assertions with exact artifact locators; +- non-executing merge plans; +- a SHA-256 chained event log. + +The event chain detects accidental or unauthorized mutation of project-index history. It does not cryptographically identify the human or agent who caused an event. + +## CLI overview + +```bash +python scripts/science_project.py --database project/evidence.sqlite init ... +python scripts/science_project.py --database project/evidence.sqlite import-run ... +python scripts/science_project.py --database project/evidence.sqlite fork ... +python scripts/science_project.py --database project/evidence.sqlite assert ... +python scripts/science_project.py --database project/evidence.sqlite compare ... +python scripts/science_project.py --database project/evidence.sqlite merge-plan ... +python scripts/science_project.py --database project/evidence.sqlite summary ... +``` + +Every subcommand requires `--output` and writes a machine-readable JSON receipt. + +## Initialize a project + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + init \ + --project-id egfr-resistance \ + --title "EGFR resistance mechanism evaluation" \ + --question "Which receptor state and compound series best discriminate the resistance hypotheses?" \ + --output projects/egfr/project.created.json +``` + +The command is idempotent only when project ID, title, and question are identical. + +## Import a reviewed run + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + import-run \ + --project-id egfr-resistance \ + --manifest artifacts/run-001/manifest.json \ + --branch main \ + --output projects/egfr/import-run-001.json +``` + +For a continuation: + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + import-run \ + --project-id egfr-resistance \ + --manifest artifacts/run-002/manifest.json \ + --branch main \ + --parent-run run-001 \ + --output projects/egfr/import-run-002.json +``` + +Import performs full bundle validation before indexing. It records: + +```json +{ + "project_id": "egfr-resistance", + "run_id": "run-002", + "branch_name": "main", + "parent_run_id": "run-001", + "manifest_path": "/absolute/local/path/artifacts/run-002/manifest.json", + "manifest_sha256": "...", + "review_status": "passed", + "imported_at": "..." +} +``` + +The project database is local-machine state. For portable collaboration, export project receipts and retain the original run bundles; do not assume absolute manifest paths are valid on another machine. + +## Fork a hypothesis branch + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + fork \ + --project-id egfr-resistance \ + --source-run run-002 \ + --branch receptor-state-inactive \ + --output projects/egfr/fork-inactive.json +``` + +A fork points its base and head at an already imported run. It does not copy artifacts. The next imported run on that branch should name the branch and normally use the prior head as parent. + +Recommended branch names describe the scientific fork, not the intended winner: + +```text +receptor-state-inactive +receptor-state-active +microstate-neutral +microstate-cationic +assay-readout-biochemical +assay-readout-cellular +``` + +Avoid names such as `successful-model` or `best-compounds`; they encode an answer before the evidence is compared. + +## Add a quantitative evidence assertion + +An evidence assertion links one imported run claim to a source and an exact location in a hash-validated artifact. + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + assert \ + --project-id egfr-resistance \ + --run-id run-002 \ + --claim-id C-17 \ + --source-id PMID:12345678 \ + --polarity supports \ + --locator '{ + "artifact_path":"studies.json", + "artifact_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "record_id":"study-42", + "json_pointer":"/studies/3/effect" + }' \ + --independence-group cohort-A \ + --effect-measure hazard_ratio \ + --estimate 0.72 \ + --interval-low 0.61 \ + --interval-high 0.85 \ + --sample-size 842 \ + --population "advanced NSCLC" \ + --risk-of-bias-ref ROB-31 \ + --output projects/egfr/assertion-C17-study42.json +``` + +Required fields: + +- imported `run_id` and `claim_id`; +- source identifier; +- polarity: `supports`, `contradicts`, `qualifies`, `neutral`, or `unavailable`; +- locator with imported artifact path and exact SHA-256; +- at least one exact locator field: page, table, figure, cell, JSON pointer, line range, or record ID; +- independence group. + +Optional quantitative fields: + +- effect measure; +- estimate; +- lower and upper interval; +- unit; +- sample size; +- population; +- risk-of-bias record. + +The store verifies that the locator path and digest occur in the imported run. It does not verify that the cited table cell was extracted correctly; source and method review must do that. + +## Compare two runs + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + compare \ + --project-id egfr-resistance \ + --previous-run run-002 \ + --current-run run-007 \ + --output projects/egfr/run-002-to-007.diff.json +``` + +The comparison re-hashes each manifest before loading it and reports: + +- added, removed, and changed artifacts; +- added, removed, and changed claims; +- code changes; +- environment changes; +- whether prior review is invalidated. + +A run comparison is not a statistical comparison. It reports record changes, not which scientific conclusion is better. + +## Prepare a merge plan + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + merge-plan \ + --project-id egfr-resistance \ + --source-branch receptor-state-inactive \ + --target-branch main \ + --output projects/egfr/merge-inactive-to-main.json +``` + +Output includes: + +```json +{ + "base_run_id": "run-002", + "source_run_id": "run-007", + "target_run_id": "run-006", + "source_diff": {}, + "target_diff": {}, + "claim_conflicts": [], + "artifact_conflicts": [], + "base_ambiguity": false, + "review_receipts_invalidated": true, + "requires_scientific_review": true, + "status": "candidate", + "executed": false +} +``` + +A conflict is reported when source and target both changed the same claim or artifact differently from the base. The merge planner never resolves the conflict by preferring recency, review status, or a desired conclusion. + +If branch ancestry has no unambiguous base, `base_ambiguity` is true. Resolve lineage before treating the plan as a three-way comparison. + +## Summarize project state + +```bash +python scripts/science_project.py \ + --database projects/egfr/evidence.sqlite \ + summary \ + --project-id egfr-resistance \ + --output projects/egfr/summary.json +``` + +The summary contains project metadata, branches, runs, assertion count, merge-plan count, and the event-chain head. + +## Recommended project layout + +```text +projects/egfr/ + evidence.sqlite + receipts/ + comparisons/ + merge-plans/ +artifacts/ + run-001/ + run-002/ + run-006/ + run-007/ +``` + +Keep the database outside individual run bundles. Do not add mutable SQLite files to a run's artifact manifest as scientific evidence. + +## Search patterns + +Use these exact searches when reading this reference: + +- `## Import a reviewed run` +- `## Fork a hypothesis branch` +- `## Add a quantitative evidence assertion` +- `## Compare two runs` +- `## Prepare a merge plan` +- `## Failure handling` +- `## Common mistakes` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| imported manifest digest changed | stop; restore original bytes or import changed work under a new run ID | +| bundle validation fails | fix or preserve the run as failed/inconclusive; do not index it as reviewed evidence | +| duplicate run ID with different digest | create a new run ID; never overwrite project history | +| branch already exists | reuse it only if it represents the same fork; otherwise choose a distinct scientific name | +| assertion artifact digest mismatch | resolve the exact source artifact and locator before recording the assertion | +| interval only partly supplied | provide both bounds or omit the interval | +| merge conflict | create a decision/review task; do not auto-merge | +| ambiguous base | repair lineage or explicitly create a new comparison contract | +| database copied to another machine | verify every manifest path and digest before using summaries | + +## Common mistakes + +- Treating SQLite rows as a replacement for manifests and artifacts. +- Re-importing changed bytes under the same run ID. +- Forking after observing an outcome without preserving the original decision event. +- Recording an assertion without an exact artifact locator. +- Using publication count as independence count. +- Resolving claim conflicts by selecting the branch with a passed process receipt. +- Calling a merge plan an executed merge. +- Marking a branch merged without a new reviewed synthesis run. + +## Evidence boundary + +The project store improves lineage, comparison, quantitative evidence location, and experimental branching. It does not establish causality, choose the correct hypothesis, authenticate human identity, or replace independent scientific review. diff --git a/skills/codex-science/references/reference-loading.md b/skills/codex-science/references/reference-loading.md new file mode 100644 index 00000000..df83ec95 --- /dev/null +++ b/skills/codex-science/references/reference-loading.md @@ -0,0 +1,54 @@ +# Reference loading protocol + +Use this protocol after the coordinator selects a skill and before the selected skill executes a route controlled by detailed reference material. + +## 1. Inspect the reference index + +Look for `/references/index.json`. + +- When absent, follow the complete contract in `SKILL.md`; do not search the repository for undocumented instructions and do not invent a reference. +- When present, validate the index and select the minimum entries matching the route. +- Treat `required_before` as a hard procedural dependency. + +## 2. Load the minimum required material + +Rank entries by: + +1. exact `required_before` operation match; +2. `read_when` route match; +3. query terms matching purpose or indexed search patterns. + +Do not load every reference in a broad skill. This preserves context for primary evidence and actual execution records. + +## 3. Read before acting + +A mandatory reference is read before: + +- constructing an API or CLI call whose arguments are documented there; +- applying a source-specific filter or pagination rule; +- interpreting a field, score, confidence value, unit, or missingness state; +- choosing a benchmark split, threshold, or acceptance metric; +- making a write, compute, or approval decision controlled by a backend guide. + +Do not infer argument order, output schema, source semantics, or acceptance thresholds from names alone. + +## 4. Search long references deliberately + +Use the index `search_patterns` to find the relevant heading or field. Read enough contiguous context to capture prerequisites, exceptions, and output semantics. A matching sentence without its surrounding boundary is not sufficient. + +## 5. Record material use + +Create a reference-use receipt when the reference controls a material query, command, transformation, threshold, or claim. Record: + +- skill and reference ID; +- relative path and SHA-256; +- reason for reading; +- claim, artifact, or operation informed; +- sections and search terms used; +- load time. + +Aggregate receipts into a `reference-use-ledger` artifact. A changed reference hash invalidates the procedural assumption and any review receipt that explicitly covered it. + +## 6. Preserve evidence boundaries + +References describe how to use a source, model, method, or schema. They are not evidence that a returned biological, chemical, or clinical claim is true. Link the actual source record or computation separately in the evidence graph. diff --git a/skills/science-provenance/SKILL.md b/skills/science-provenance/SKILL.md index f42a69ee..9e46735e 100644 --- a/skills/science-provenance/SKILL.md +++ b/skills/science-provenance/SKILL.md @@ -9,6 +9,14 @@ Create `artifacts//manifest.json` in the user's current research project For a non-trivial active run, keep `checkpoint.json` beside the manifest as mutable control state. Schema v4 binds the run to an activation-generation key; records structured done criteria, hashed evidence and review receipts, continuation budget, native Goal task binding and phase, external waits, failed attempts, gates, and blockers; and prevents a second nonterminal run for the same owner across nested artifact roots. Update it with `/scripts/science_checkpoint.py`. Save a run-local progress record before `heartbeat --progress-ref`; do not cite the mutable checkpoint as scientific evidence or add it to the manifest. +## Reference usage + +Read [the artifact contract](references/artifact-contract.md) before authoring a manifest, sidecar, or bundle-version change. Read [the large-artifact protocol](references/large-artifacts.md) before registering a directory, a file too large for whole-memory loading, a chunked dataset, model weights, or an immutable external object. Do not guess sidecar fields, Merkle rules, or invalidation semantics. + +Read [the native artifact runtime contract](references/artifact-runtime.md) before rendering an artifact, recording a typed selection, or proposing a transform. Read [the FAIR-oriented export contract](references/fair-export.md) before archival, interoperable export, or scientific dependency and license handoff. + +When a reference controls artifact registration or review, save a `reference-use-ledger` with the reference hash. A changed reference hash invalidates a receipt that explicitly covered its prior procedural assumptions. + ## Run contract 1. Use a stable run ID. Create a new run when the question, estimand, baseline, metric, data split, success threshold, or intended decision changes. diff --git a/skills/science-provenance/quality.json b/skills/science-provenance/quality.json new file mode 100644 index 00000000..2662fbb8 --- /dev/null +++ b/skills/science-provenance/quality.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "skill": "science-provenance", + "roles": ["infrastructure"], + "declared_maturity": "L3", + "output_schemas": [ + "artifact-manifest-v1", + "claim-register-v1", + "evidence-graph-v1-v2", + "reference-use-ledger-v1", + "artifact-descriptor-v1", + "artifact-selection-v1", + "transform-proposal-v1", + "fair-export-receipt-v1", + "scientific-bom-v1", + "ro-crate-oriented-v1", + "prov-oriented-v1" + ], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [ + "A provenance bundle can be internally valid while the scientific design remains invalid.", + "Large or proprietary external inputs may be represented only by immutable references and checksums." + ] +} diff --git a/skills/science-provenance/references/artifact-contract.md b/skills/science-provenance/references/artifact-contract.md index 99b6dc8f..536d7616 100644 --- a/skills/science-provenance/references/artifact-contract.md +++ b/skills/science-provenance/references/artifact-contract.md @@ -73,3 +73,9 @@ Claim status and computation status are separate axes. A claim may be planned, s Create a new run when the research question, estimand, baseline, primary metric, data split, success threshold, intended decision, or material input identity changes. Keep revisions inside one run only when they answer the same contract and preserve original files, snapshot diffs, findings, and prior receipts. A review or acceptance receipt becomes stale when a covered claim, artifact hash, source snapshot, query, code revision, model contract, weight, database, configuration, input, or acceptance bundle changes. Retain failed and inconclusive manifests. Never rewrite a failed run into a successful one, detach counterevidence, silently transfer an annotation, or reuse a receipt for changed bytes. + +## Progressive references and large artifacts + +Use `reference-use-ledger` for hash-bound records of detailed skill references that controlled a material query, command, threshold, transformation, or claim. Use `artifact-descriptor` for streaming, chunked, directory-Merkle, or immutable external artifact metadata. + +Manifest schema version `1` remains stable. A local artifact may optionally declare `artifact_type` as `file`, `chunked-file`, or `directory-tree`, together with `size_bytes`, `entry_count`, `media_type`, and `descriptor_path`. Directory artifacts use the deterministic `sha256-merkle-v1` root described in [large-artifacts.md](large-artifacts.md). Changing referenced instructions, descriptor bytes, directory entries, or chunk hashes invalidates dependent review receipts. diff --git a/skills/science-provenance/references/artifact-runtime.md b/skills/science-provenance/references/artifact-runtime.md new file mode 100644 index 00000000..f3130dc4 --- /dev/null +++ b/skills/science-provenance/references/artifact-runtime.md @@ -0,0 +1,230 @@ +# Native scientific artifact runtime + +Read this reference before describing, rendering, selecting, or proposing a change to a scientific artifact. The runtime is bounded, local, hash-aware, and non-mutating. + +## What the runtime does + +The runtime produces three distinct records: + +1. `artifact-runtime-descriptor` — a bounded derived view of validated bytes; +2. `artifact-selection` — a typed pointer bound to the artifact path and SHA-256; +3. `transform-proposal` — a non-executing request that identifies affected steps and expected outputs. + +Never collapse these records into one action. A view does not validate an interpretation, a selection does not endorse a claim, and a proposal does not edit or rerun anything. + +## Supported viewers + +| Viewer | Typical formats | Typed selections | Current boundary | +| --- | --- | --- | --- | +| `structure-3d` | PDB, mmCIF | atom, residue, chain, ligand, spatial region | PDB coordinates are normalized from a bounded prefix; mmCIF is category/row preview only | +| `molecule-2d` | SDF, MOL, MOL2, SMILES | atom, bond, substructure, record | no aromaticity, stereochemistry, or valence inference beyond the source record | +| `genome-track` | VCF, BED, GFF, GTF, WIG, bigWig | variant, locus, interval, track record | bigWig is binary metadata only without an indexed reader | +| `single-cell` | H5AD, Loom | cell, cell set, feature, cluster, donor, layer | HDF5 container detection only; dimensions and layers are never guessed | +| `table` | CSV, TSV, Parquet, Arrow | row, column, cell, range | delimited text rows are bounded; binary columnar schema needs an Arrow backend | +| `figure` | PNG, JPEG, SVG, TIFF, WebP | region, axis, legend, panel | SVG markup is escaped and never executed | +| `evidence-graph` | evidence graph JSON | node, edge, claim, component | requires complete JSON inside the byte bound for exact counts | +| `trajectory` | XYZ, DCD, XTC, TRR, NetCDF | frame, frame range, atom, atom set | XYZ frames are previewed; binary trajectories expose metadata only | +| `directory` | Zarr, dataset directory, output tree | entry, entry set | directory Merkle descriptor is authoritative, view is entry bounded | +| `text` / `binary` | other files | line, JSON pointer, byte range, record | fallback view; no domain interpretation | + +## CLI overview + +All commands require explicit output files. No command edits the artifact. + +```bash +python scripts/render_artifact_runtime.py describe ... +python scripts/render_artifact_runtime.py render ... +python scripts/render_artifact_runtime.py select ... +python scripts/render_artifact_runtime.py propose ... +``` + +### Describe an artifact + +```bash +python scripts/render_artifact_runtime.py describe \ + artifacts/run-001/receptor.pdb \ + --artifact-path receptor.pdb \ + --sha256 8e7f...64-hex... \ + --kind receptor-structure \ + --artifact-type file \ + --media-type chemical/x-pdb \ + --max-bytes 1048576 \ + --max-records 200 \ + --output artifacts/run-001/receptor.runtime.json +``` + +Required arguments: + +- positional artifact path: local file or directory; +- `--artifact-path`: safe path relative to the run bundle; +- `--kind`: manifest artifact kind; +- `--max-bytes`: positive byte ceiling; +- `--max-records`: positive record ceiling; +- `--output`: descriptor destination. + +Recommended arguments: + +- `--sha256`: manifest digest; the command fails if bytes do not match; +- `--artifact-type`: `file`, `chunked-file`, `directory-tree`, or another declared manifest type; +- `--media-type`: explicit media type when filename inference is insufficient. + +Output fields: + +```json +{ + "schema_version": 1, + "artifact_path": "receptor.pdb", + "artifact_sha256": "...", + "artifact_kind": "receptor-structure", + "artifact_type": "file", + "media_type": "chemical/x-pdb", + "size_bytes": 481239, + "viewer": "structure-3d", + "generated_at": "2026-07-21T00:00:00Z", + "bounds": {"max_bytes": 1048576, "max_records": 200}, + "preview": {}, + "warnings": [], + "evidence_boundary": "...", + "fingerprint": "..." +} +``` + +The fingerprint covers the descriptor, including the artifact hash and view bounds. Changing the byte or record ceiling creates a different descriptor. + +### Render an offline view + +```bash +python scripts/render_artifact_runtime.py render \ + artifacts/run-001/receptor.runtime.json \ + --title "Receptor state inspection" \ + --output artifacts/run-001/receptor.runtime.html +``` + +The HTML uses no hosted assets and escapes source-controlled text. It is a derived navigation artifact and must not be cited as primary evidence. + +### Create a typed selection + +```bash +python scripts/render_artifact_runtime.py select \ + artifacts/run-001/receptor.runtime.json \ + --selector-type residue \ + --selector '{"chain":"A","residue_number":145,"insertion_code":""}' \ + --selected-by reviewer-agent-2 \ + --reason "Alternate conformation conflicts with the declared inactive receptor state." \ + --label "A:145" \ + --output artifacts/run-001/selections/residue-A-145.json +``` + +Selection output: + +```json +{ + "schema_version": 1, + "selection_id": "selection-...", + "artifact_path": "receptor.pdb", + "artifact_sha256": "...", + "viewer": "structure-3d", + "selector_type": "residue", + "selector": {"chain":"A","residue_number":145}, + "selected_by": "reviewer-agent-2", + "reason": "...", + "label": "A:145", + "created_at": "...", + "status": "active", + "fingerprint": "..." +} +``` + +A selection becomes stale when the current manifest digest for `artifact_path` differs. Never silently transfer a selector to changed bytes, even when the filename is unchanged. + +### Propose a transform + +```bash +python scripts/render_artifact_runtime.py propose \ + artifacts/run-001/selections/residue-A-145.json \ + --operation exclude-alternate-conformation \ + --parameters '{"altloc":"B","retain":"highest-occupancy"}' \ + --reason "Use one receptor microstate before docking." \ + --affected-step receptor-preparation \ + --affected-step docking \ + --affected-step interaction-analysis \ + --expected-output prepared/receptor.pdbqt \ + --expected-output docking/poses.sdf \ + --proposed-by analyst-agent-1 \ + --approval-boundary "structure mutation and downstream rerun" \ + --output artifacts/run-001/proposals/exclude-A-145-altloc-B.json +``` + +The proposal must retain: + +```json +{ + "status": "proposed", + "executed": false, + "requires_approval": true +} +``` + +A proposal is an input to impact analysis and selective rerun planning. It is not a shell command, patch, or approval receipt. + +## Manifest sidecar kinds + +Record generated JSON files as ordinary hashed artifacts: + +- runtime descriptor: `artifact-runtime-descriptor`; +- selection: `artifact-selection`; +- proposal: `transform-proposal`. + +The offline HTML is a derived view and normally should not be primary claim evidence. + +## Required workflow + +1. Validate the complete run bundle. +2. Resolve the artifact path, kind, media type, and manifest SHA-256. +3. Set explicit byte and record bounds appropriate to the format. +4. Generate and save the descriptor. +5. Inspect warnings before selecting anything. +6. Create the smallest typed selection that identifies the issue. +7. Create a non-executing transform proposal. +8. Run impact propagation and selective rerun planning. +9. Request approval if bytes, remote state, cost, or scientific scope will change. +10. Execute through the normal provenance and compute route. +11. Diff old and new artifacts and invalidate covered reviews. +12. Re-review the changed claims. + +## Search patterns + +Use these exact searches in this reference when needed: + +- `### Describe an artifact` +- `### Create a typed selection` +- `### Propose a transform` +- `## Manifest sidecar kinds` +- `## Failure handling` +- `## Common mistakes` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| digest mismatch | stop; resolve the correct manifest or artifact bytes | +| unsupported format | use bounded `binary` or `text` view and state the missing reader | +| truncated JSON | increase the explicit bound or use a format-specific indexed reader; do not infer missing records | +| stale selection | create a new descriptor and selection against the new hash | +| active SVG content | keep escaped; never execute script, external image, or event handlers | +| H5AD/Parquet/bigWig schema unavailable | load an approved format-specific backend; never infer dimensions or columns from the filename | +| proposal affects unlisted downstream work | revise the dependency graph before execution | + +## Common mistakes + +- Rendering before bundle/hash validation. +- Treating the bounded prefix as a complete atom, variant, cell, or row count. +- Calling a model-imagined depiction a view of the artifact. +- Selecting by filename without the artifact SHA-256. +- Applying a natural-language edit directly instead of producing a typed proposal. +- Marking the proposal executed before a real command and receipt exist. +- Reusing a review receipt after any covered artifact changes. + +## Evidence boundary + +The runtime improves inspection, collaboration, and rerun precision. It does not establish chemical validity, structural correctness, genome-build compatibility, cell-type identity, statistical validity, or scientific truth without the relevant domain checks and evidence. diff --git a/skills/science-provenance/references/fair-export.md b/skills/science-provenance/references/fair-export.md new file mode 100644 index 00000000..c6383827 --- /dev/null +++ b/skills/science-provenance/references/fair-export.md @@ -0,0 +1,229 @@ +# FAIR-oriented export and scientific dependency BOM + +Read this reference before exporting a validated run to RO-Crate-oriented metadata, W3C PROV-oriented lineage, or a scientific dependency bill of materials. + +## Scope and boundary + +The exporter packages **recorded metadata from a hash-validated run**. It does not: + +- certify formal RO-Crate conformance; +- certify W3C PROV completeness; +- establish 21 CFR Part 11, GxP, HIPAA, GDPR, or other regulatory compliance; +- provide a legal opinion about licenses; +- include bytes absent from the source bundle; +- discover unrecorded transitive dependencies; +- authenticate the identity of agents or reviewers. + +Use the exports for interoperability, archival review, dependency inspection, and handoff—not as a compliance certificate. + +## CLI + +```bash +python scripts/export_scientific_run.py \ + artifacts/run-014/manifest.json \ + artifacts/run-014/exports \ + --receipt artifacts/run-014/fair-export.receipt.json +``` + +Preconditions: + +1. `manifest.json` exists; +2. every manifest artifact exists and matches SHA-256 or Merkle root; +3. recognized sidecars validate; +4. secrets and credential-bearing URLs are absent; +5. external inputs use canonical identifier, source, release, and checksum when available. + +The exporter refuses changed artifact bytes because it calls complete bundle validation first. + +## Outputs + +```text +exports/ + ro-crate-metadata.json + prov.json + scientific-bom.json + export-receipt.json +fair-export.receipt.json +``` + +The receipt passed to `--receipt` duplicates the generated export receipt at a caller-selected path for workflow integration. + +## RO-Crate-oriented metadata + +`ro-crate-metadata.json` uses: + +```json +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [] +} +``` + +The graph includes: + +- root run dataset; +- manifest file; +- artifact file or dataset entities; +- artifact SHA-256, kind, media type, size, and directory Merkle metadata when recorded; +- claim entities and evidence links; +- review status; +- explicit Codex Science evidence boundary. + +The exporter does not copy artifact bytes into the export directory. `hasPart` points to run-relative artifacts. Preserve the source bundle or copy it through a separate verified packaging step. + +## W3C PROV-oriented lineage + +`prov.json` records: + +- input entities; +- artifact entities; +- execution activities; +- run activity; +- `used` links from recorded inputs to executions; +- `wasGeneratedBy` links from artifacts to the run; +- exit codes and artifact digests where recorded. + +The export cannot reconstruct events omitted from the source manifest. A sparse provenance record remains sparse and should not be described as complete. + +## Scientific dependency BOM + +`scientific-bom.json` gathers recorded components: + +```text +code +package +container digest +lockfile digest +code revision +model registry digest +model receipt +input +dataset +database snapshot +model weight +license BOM +``` + +Component examples: + +```json +{ + "component_type": "dataset", + "id": "doi:10.1000/example", + "record": { + "source": "repository", + "release": "2026-01", + "sha256": "..." + } +} +``` + +A missing component or license field means “not recorded,” not “not required” or “permitted.” Resolve license and data-use terms separately. + +## Export receipt + +```json +{ + "schema_version": 1, + "run_id": "run-014", + "source_manifest_path": "/absolute/path/manifest.json", + "source_manifest_sha256": "...", + "generated_at": "...", + "exports": [ + { + "name": "ro_crate", + "path": "ro-crate-metadata.json", + "sha256": "...", + "size_bytes": 1234 + } + ], + "status": "generated", + "certified": false, + "regulatory_compliance_claimed": false, + "evidence_boundary": "...", + "fingerprint": "..." +} +``` + +The receipt fingerprint covers the source manifest digest and every generated export digest. Regenerating after a source change creates a new receipt. + +## Recommended archival package + +A portable archival package should contain: + +```text +run/ + manifest.json + all included artifact bytes or immutable external-reference records + review receipts + export/ + ro-crate-metadata.json + prov.json + scientific-bom.json + export-receipt.json +``` + +After copying: + +1. revalidate every file hash and directory Merkle root; +2. resolve absolute local paths in metadata where portability matters; +3. preserve external access restrictions and licenses; +4. record the packaging command and package checksum; +5. run an offline open/read test in a clean environment. + +Do not place mutable SQLite project stores into a run bundle as primary evidence. Export their receipts and retain the original immutable runs instead. + +## Regulatory-supporting mode + +The export can support—but does not itself provide—controlled-record workflows. A regulated implementation additionally needs reviewed policies for: + +- system validation for intended use; +- authenticated users and role separation; +- electronic signatures where applicable; +- audit-trail retention; +- trusted time source; +- reason-for-change; +- record retention and retrieval; +- backup and disaster recovery; +- controlled SOP and software versions; +- incident and deviation handling; +- vendor and infrastructure qualification. + +Describe these as compliance-supporting controls unless the complete system has been formally assessed. + +## Search patterns + +- `## RO-Crate-oriented metadata` +- `## W3C PROV-oriented lineage` +- `## Scientific dependency BOM` +- `## Export receipt` +- `## Recommended archival package` +- `## Failure handling` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| artifact digest mismatch | stop; restore or regenerate the source run bundle | +| missing external source release | add the release or mark it unavailable explicitly | +| output directory contains older exports | preserve or clear it through an explicit versioned archival action | +| absolute path not portable | retain it as local provenance and add a portable identifier or package-relative mapping | +| license unknown | record unknown/restricted and stop redistribution until resolved | +| external bytes unavailable | preserve immutable reference and missingness; do not invent inclusion | +| BOM omits a material model/database | correct the source provenance before exporting | +| export schema rejected downstream | preserve the failed export, correct mapping, and regenerate a new receipt | + +## Common mistakes + +- Calling the generated metadata formally certified. +- Exporting before bundle validation. +- Assuming `hasPart` means bytes were copied. +- Treating missing license metadata as permissive licensing. +- Omitting model weights, databases, or container identity. +- Reusing an export receipt after source bytes change. +- Claiming a provenance graph is complete when executions or inputs were not recorded. +- Calling a compliance-supporting package regulatory compliant. + +## Evidence boundary + +FAIR-oriented export improves findability, portability, lineage review, and dependency inspection. It does not certify completeness, licensing, regulatory compliance, or scientific validity. diff --git a/skills/science-provenance/references/index.json b/skills/science-provenance/references/index.json new file mode 100644 index 00000000..f2c21b4c --- /dev/null +++ b/skills/science-provenance/references/index.json @@ -0,0 +1,75 @@ +{ + "schema_version": 1, + "skill": "science-provenance", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "artifact-contract", + "path": "references/artifact-contract.md", + "purpose": "Manifest fields, validated sidecars, evidence links, versioning, and review invalidation.", + "read_when": [ + "creating or changing a manifest", + "adding a sidecar", + "reviewing bundle versioning or invalidation" + ], + "required_before": [ + "manifest-authoring", + "sidecar-authoring", + "bundle-versioning" + ], + "search_patterns": [ + "## Validated sidecars", + "## Bundle validation", + "## Versioning and invalidation" + ], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Defines packaging and validation; it does not validate domain-specific scientific claims." + }, + { + "id": "large-artifacts", + "path": "references/large-artifacts.md", + "purpose": "Streaming, chunked, Merkle-tree, external-reference, and content-addressed handling for large scientific outputs.", + "read_when": [ + "an artifact is a directory", + "a file is too large for whole-file memory loading", + "a run uses trajectories, AnnData, BAM/CRAM, Zarr, Parquet, weights, or remote immutable objects" + ], + "required_before": [ + "large-artifact-registration", + "directory-artifact-registration", + "content-addressed-cache-insertion" + ], + "search_patterns": [ + "## Artifact types", + "## Content-addressed cache", + "## Validation and failure semantics" + ], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Validates artifact identity and transfer semantics, not the scientific meaning of the artifact." + }, + { + "id": "artifact-runtime", + "path": "references/artifact-runtime.md", + "purpose": "Bounded native views, hash-bound selections, and non-executing transform proposals for scientific artifacts.", + "read_when": ["before rendering a scientific artifact", "before creating an atom, residue, locus, cell, table, graph, figure, or trajectory selection", "before converting an annotation into a proposed edit or rerun"], + "required_before": ["calling scripts/render_artifact_runtime.py", "recording artifact-runtime-descriptor, artifact-selection, or transform-proposal sidecars"], + "search_patterns": ["### Describe an artifact", "### Create a typed selection", "### Propose a transform", "## Failure handling", "## Common mistakes"], + "authority": "Codex Science first-party runtime contract", + "version": "1", + "evidence_boundary": "A bounded runtime view improves inspection but does not validate a scientific interpretation or execute a transform." + }, + { + "id": "fair-export", + "path": "references/fair-export.md", + "purpose": "Export validated run metadata as RO-Crate-oriented, W3C PROV-oriented, and scientific dependency BOM records.", + "read_when": ["before archival or interoperable export", "before generating a scientific dependency or license handoff"], + "required_before": ["calling scripts/export_scientific_run.py", "describing an export as portable, FAIR, or compliance-supporting"], + "search_patterns": ["## RO-Crate-oriented metadata", "## W3C PROV-oriented lineage", "## Scientific dependency BOM", "## Failure handling"], + "authority": "Codex Science first-party export contract", + "version": "1", + "evidence_boundary": "The exporter packages recorded metadata; it does not certify completeness, licensing, regulatory compliance, or scientific validity." + } + ] +} diff --git a/skills/science-provenance/references/large-artifacts.md b/skills/science-provenance/references/large-artifacts.md new file mode 100644 index 00000000..d3b25304 --- /dev/null +++ b/skills/science-provenance/references/large-artifacts.md @@ -0,0 +1,62 @@ +# Large artifact protocol + +Use this protocol for files or directories that should not be loaded wholly into memory, including trajectories, HDF5 or AnnData files, BAM/CRAM, bigWig, Zarr stores, Parquet datasets, model weights, and large image collections. + +## Artifact types + +### `file` + +A regular local file. Compute SHA-256 with streaming reads. + +### `chunked-file` + +A regular file with a descriptor containing ordered chunk offsets, sizes, and SHA-256 values. The root SHA-256 remains the full-file digest. Chunk records support resumable verification and transfer. + +### `directory-tree` + +A directory represented by a deterministic list of relative file paths, sizes, and SHA-256 values. The root digest is: + +```text +sha256(canonical_json({"algorithm":"sha256-merkle-v1","entries":[...]})) +``` + +Symbolic links are rejected by default. A directory entry cannot escape the artifact root. + +### `external-reference` + +An immutable remote or database object described by URI or accession, version, SHA-256, size when known, media type, and license. It is not considered locally verified until bytes are materialized and checked. + +## Commands + +```bash +python scripts/describe_artifact.py trajectory/ \ + --media-type application/x-zarr \ + --output trajectory.descriptor.json + +python scripts/describe_artifact.py model.safetensors \ + --chunk-size 8388608 \ + --output model.descriptor.json + +python scripts/describe_artifact.py model.safetensors \ + --verify model.descriptor.json +``` + +Add a descriptor with `kind: artifact-descriptor`. For a directory artifact, add the directory itself to the manifest with `artifact_type: directory-tree`, the descriptor root SHA-256, total bytes, and entry count. + +## Content-addressed cache + +A local cache stores immutable files at: + +```text +/objects/sha256// +``` + +Insertion computes the digest before and after copy, writes through a temporary file, marks the object read-only, and reuses an existing verified object. Cache presence is not proof that a source license permits redistribution. + +## Validation and failure semantics + +- Reject missing files, digest changes, path traversal, escaping links, non-contiguous chunks, and descriptor size mismatches. +- Preserve incomplete downloads outside the immutable object path. +- Distinguish `downloaded`, `partially-downloaded`, `locally-verified`, and `externally-referenced` states. +- Do not hash a multi-gigabyte file with `read_bytes()`. +- Do not copy private or licensed data into a run bundle merely to make it self-contained. diff --git a/skills/science-review/SKILL.md b/skills/science-review/SKILL.md index 423b8a9c..77733221 100644 --- a/skills/science-review/SKILL.md +++ b/skills/science-review/SKILL.md @@ -7,6 +7,14 @@ description: Independently review a scientific run by checking claims against th Review the record adversarially; do not assume the producing agent's conclusion, selected method, or confidence is correct. +## Reference usage + +Read [the review checklist](references/review-checklist.md) before assigning severity, deciding whether a review passes, or closing a material finding. Use the indexed headings rather than loading unrelated reference material. Do not infer a pass rule or downgrade a finding from prose convenience. + +Read [the independent reviewer packet contract](references/independent-review-runtime.md) before preparing a separate review packet, accepting a reviewer response, or claiming independence or reproduction. + +If the checklist materially controls a review receipt, include its SHA-256 in the run's `reference-use-ledger`. The checklist defines review procedure; it does not prove the reviewed conclusion true. + ## Inputs and review mode Require the decision contract, approved plan, artifact manifest, claim register, query and execution logs, outputs, environment, lane receipts, cited sources, and prior findings. Report missing evidence rather than filling gaps from memory. diff --git a/skills/science-review/quality.json b/skills/science-review/quality.json new file mode 100644 index 00000000..359bfea2 --- /dev/null +++ b/skills/science-review/quality.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "skill": "science-review", + "roles": ["reviewer"], + "declared_maturity": "L3", + "output_schemas": [ + "review-receipt-v2", + "scientific-finding-v1", + "review-resolution-v1", + "review-task-v1", + "review-response-v1" + ], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [ + "Deterministic review checks do not replace domain-expert judgment.", + "Independent reproduction is claimed only after a separate execution recreates the material result." + ] +} diff --git a/skills/science-review/references/independent-review-runtime.md b/skills/science-review/references/independent-review-runtime.md new file mode 100644 index 00000000..7e8eb4bd --- /dev/null +++ b/skills/science-review/references/independent-review-runtime.md @@ -0,0 +1,224 @@ +# Independent reviewer packet runtime + +Read this reference before preparing a separate human or agent review, handing artifacts to a reviewer, accepting a review response, or calling a review independent. + +## Independence boundary + +The runtime creates a deterministic packet and validates the response. It does not authenticate the reviewer or prove true organizational independence. + +`independent: true` is an attestation by the calling workflow. Governance must ensure the reviewer: + +- did not produce the material result; +- did not share hidden producer rationale or intended conclusion; +- receives the approved decision contract and recorded evidence rather than a desired answer; +- uses a separate workspace or context; +- has no undisclosed conflict that changes trust in the review. + +A second pass by the producer must use `independent: false`. + +## Prepare a review packet + +```bash +python scripts/science_reviewer.py prepare \ + artifacts/run-014/manifest.json \ + --mode record \ + --mode source \ + --mode method \ + --question "Does every numerical claim agree with the saved table?" \ + --question "Are shared cohorts counted as one evidence dependency?" \ + --output artifacts/run-014/review-task.json +``` + +Use `--mode reproduction` only when the reviewer will actually execute specified computations in a separate workspace. + +The packet includes: + +- source run and manifest SHA-256; +- approved question and plan; +- input, code, execution, and environment records; +- material claims; +- artifact path/hash/kind list; +- evidence graph, study table, query records, lane receipts, and model receipts when present; +- review questions; +- response contract; +- packet fingerprint and task ID. + +The packet excludes fields named like: + +```text +intended_conclusion +producer_rationale +chain_of_thought +private_scratchpad +hidden_answer +suspected_bug +``` + +Secret-like keys are redacted. The runtime does not extract or transmit private chain-of-thought. + +## Packet identity + +```json +{ + "review_task_id": "review-task-...", + "source_run_id": "run-014", + "source_manifest_sha256": "...", + "review_modes": ["method", "record", "source"], + "independent_required": true, + "material_claim_ids": ["C1", "C2"], + "artifacts": [ + {"path": "metrics.json", "sha256": "...", "kind": "metrics"} + ], + "fingerprint": "..." +} +``` + +Changing the source manifest, review modes, questions, claims, artifact hashes, or packet content creates a different fingerprint and task ID. + +## Reviewer response contract + +The reviewer returns JSON: + +```json +{ + "schema_version": 1, + "review_task_id": "review-task-...", + "packet_fingerprint": "...", + "reviewer": "reviewer-agent-2", + "independent": true, + "review_modes": ["record", "method"], + "reviewed_claim_ids": ["C1", "C2"], + "reviewed_artifacts": [ + {"path": "metrics.json", "sha256": "..."} + ], + "findings": [ + { + "id": "F-001", + "severity": "major", + "code": "wrong-denominator", + "message": "The report uses enrolled participants instead of analyzed participants.", + "affected_claim_id": "C2", + "evidence": [ + { + "artifact_path": "study-table.json", + "artifact_sha256": "...", + "json_pointer": "/studies/3/analyzed_n" + } + ], + "required_action": "Recalculate the estimate and update the claim.", + "resolution_status": "open" + } + ], + "limitations": [ + "The raw instrument files were unavailable." + ], + "status": "findings" +} +``` + +Finding severities: + +```text +critical +major +minor +suggestion +``` + +Resolution states: + +```text +open +resolved +accepted-risk +not-applicable +``` + +A finding should identify exact evidence whenever possible: artifact path and hash plus page, table, figure, cell, JSON pointer, record ID, line range, or execution ID. + +## Finalize a response + +```bash +python scripts/science_reviewer.py finalize \ + artifacts/run-014/review-task.json \ + artifacts/run-014/reviewer-response.json \ + --output artifacts/run-014/review-receipt-v2.json +``` + +Finalization verifies: + +- task ID and packet fingerprint; +- review modes do not exceed the packet; +- reviewed claim IDs exist; +- omitted material claims have an explicit blocked reason; +- reviewed artifact paths and hashes match the packet; +- at least one artifact is covered; +- finding severity and resolution state; +- limitations are stated; +- independence requirement; +- no unresolved critical/major finding is hidden behind `passed`. + +If independence was required but attested false, the runtime adds `review-not-independent`. Missing claim coverage adds `incomplete-review-coverage`. The resulting status cannot remain passed. + +## Reproduction mode + +A reproduction review needs more than reading files: + +1. separate workspace; +2. recorded input hashes; +3. recorded code/config/model/database revisions; +4. explicit commands; +5. clean or independently constructed environment; +6. new execution logs and output hashes; +7. comparison against the material source values; +8. limitations when exact reproduction is impossible. + +Do not select `reproduction` merely because a reviewer inspected a notebook or reran one derived plotting cell. + +## Review packet transport + +The runtime only writes local JSON. Before sending a packet to another system: + +- inspect input and environment records for sensitive metadata; +- minimize data to the required evidence boundary; +- use an approved connector or transfer route; +- record destination, access scope, retention, and deletion expectations; +- do not include raw patient, participant, proprietary, or credential data unless explicitly approved. + +## Search patterns + +- `## Independence boundary` +- `## Prepare a review packet` +- `## Reviewer response contract` +- `## Finalize a response` +- `## Reproduction mode` +- `## Failure handling` + +## Failure handling + +| Failure | Required response | +| --- | --- | +| packet fingerprint mismatch | stop; prepare or retrieve the exact packet reviewed | +| reviewed artifact hash mismatch | reject the response; do not transfer the review to changed bytes | +| material claim omitted | obtain coverage or record a blocker and leave findings status | +| reviewer not independent | label it a second pass and obtain a separate reviewer if required | +| evidence locator missing | request exact support before resolving a material finding | +| reviewer requests unavailable raw data | record the limitation and decide whether review can proceed | +| reproduction environment unavailable | downgrade to record/method review; do not claim reproduction | +| critical or major finding open | keep review status findings or blocked | +| reviewer identity cannot be authenticated | preserve the attestation limitation; do not claim authenticated identity | + +## Common mistakes + +- Giving the reviewer the intended answer or suspected bug. +- Calling the producer's second pass independent. +- Reviewing only the final report while ignoring raw tables and execution logs. +- Reusing a response after source artifacts change. +- Marking reproduction without a separate execution. +- Omitting review limitations. +- Resolving a finding by editing the receipt rather than changing evidence or claim. +- Treating reviewer success as proof of scientific truth. + +## Evidence boundary + +The reviewer runtime standardizes blinded packets, hash coverage, finding records, and completion gating. It does not authenticate people, guarantee independence, replace domain expertise, or establish scientific truth. diff --git a/skills/science-review/references/index.json b/skills/science-review/references/index.json new file mode 100644 index 00000000..cdd49618 --- /dev/null +++ b/skills/science-review/references/index.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "skill": "science-review", + "default_policy": "load-minimum-required", + "references": [ + { + "id": "review-checklist", + "path": "references/review-checklist.md", + "purpose": "Claim-level checks, severity definitions, finding fields, and the review pass rule.", + "read_when": [ + "assigning finding severity", + "reviewing a material claim", + "deciding whether a review passes" + ], + "required_before": [ + "finding-severity-assignment", + "review-pass-decision" + ], + "search_patterns": [ + "## Severity", + "## Claim-level checks", + "## Pass rule" + ], + "authority": "repository-contract", + "version": "1", + "evidence_boundary": "Defines review procedure; reviewer success does not establish scientific truth." + }, + { + "id": "independent-review-runtime", + "path": "references/independent-review-runtime.md", + "purpose": "Prepare blinded, hash-bound review packets and validate separate reviewer responses without exposing producer-only rationale.", + "read_when": ["before handing a run to a separate reviewer", "before accepting a reviewer response", "before calling a review independent or reproduced"], + "required_before": ["calling scripts/science_reviewer.py", "attaching a packet-derived review receipt to a run"], + "search_patterns": ["## Independence boundary", "## Prepare a review packet", "## Reviewer response contract", "## Reproduction mode", "## Failure handling"], + "authority": "Codex Science first-party review packet contract", + "version": "1", + "evidence_boundary": "The runtime standardizes packet and receipt coverage but does not authenticate reviewer identity or guarantee independence." + } + ] +} diff --git a/src/codex_science/action_connectors.py b/src/codex_science/action_connectors.py new file mode 100644 index 00000000..b2d8c821 --- /dev/null +++ b/src/codex_science/action_connectors.py @@ -0,0 +1,364 @@ +"""Approval-gated contract for optional authenticated connector packs. + +Core Codex Science does not ship provider credentials or provider-specific write +implementations. Optional packs implement ConnectorActionAdapter and receive +credentials out of band from their host. The core contract enforces preview, +optimistic concurrency, idempotency, before/after hashes, and explicit receipts. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Protocol + + +MODES = {"read", "write"} +SECRET_FRAGMENTS = { + "token", + "password", + "secret", + "credential", + "private_key", + "api_key", + "apikey", + "access_key", + "client_secret", + "authorization", + "bearer", + "session_cookie", +} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _sha(value: Any, label: str) -> str: + text = _text(value, label).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{label} must be a SHA-256 digest") + return text + + +def _json_value(value: Any, label: str) -> None: + if value is None or isinstance(value, (str, int, float, bool)): + return + if isinstance(value, list): + for index, item in enumerate(value): + _json_value(item, f"{label}[{index}]") + return + if isinstance(value, Mapping): + for key, item in value.items(): + key_text = str(key) + if any(fragment in key_text.lower() for fragment in SECRET_FRAGMENTS): + raise ValueError(f"credential-like parameter keys are forbidden: {key_text}") + _json_value(item, f"{label}.{key_text}") + return + raise ValueError(f"{label} must contain JSON values") + + +@dataclass(frozen=True) +class ActionSpec: + schema_version: int + connector: str + operation: str + mode: str + target: str + parameters: dict[str, Any] + requested_scopes: tuple[str, ...] + idempotency_key: str + expected_before_sha256: str | None + approval_required: bool + destructive: bool + paid: bool + sensitive_data: bool + compensation: str | None + + @classmethod + def from_payload(cls, payload: Mapping[str, Any]) -> "ActionSpec": + if payload.get("schema_version", 1) != 1: + raise ValueError("unsupported action spec schema") + mode = _text(payload.get("mode"), "mode") + if mode not in MODES: + raise ValueError(f"invalid action mode: {mode}") + parameters = payload.get("parameters", {}) + if not isinstance(parameters, Mapping): + raise ValueError("parameters must be an object") + _json_value(parameters, "parameters") + scopes_raw = payload.get("requested_scopes", []) + if not isinstance(scopes_raw, list): + raise ValueError("requested_scopes must be a list") + scopes = tuple(sorted({_text(item, "requested scope") for item in scopes_raw})) + before = None if payload.get("expected_before_sha256") is None else _sha(payload["expected_before_sha256"], "expected_before_sha256") + spec = cls( + schema_version=1, + connector=_text(payload.get("connector"), "connector"), + operation=_text(payload.get("operation"), "operation"), + mode=mode, + target=_text(payload.get("target"), "target"), + parameters=dict(parameters), + requested_scopes=scopes, + idempotency_key=_text(payload.get("idempotency_key"), "idempotency_key"), + expected_before_sha256=before, + approval_required=bool(payload.get("approval_required", mode == "write")), + destructive=bool(payload.get("destructive", False)), + paid=bool(payload.get("paid", False)), + sensitive_data=bool(payload.get("sensitive_data", False)), + compensation=None if payload.get("compensation") is None else str(payload["compensation"]), + ) + if spec.mode == "write" and not spec.approval_required: + raise ValueError("write actions must require approval") + if spec.destructive and spec.mode != "write": + raise ValueError("destructive actions must use write mode") + if any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.:" for character in spec.idempotency_key): + raise ValueError("idempotency_key contains unsafe characters") + return spec + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["requested_scopes"] = list(self.requested_scopes) + return payload + + @property + def fingerprint(self) -> str: + return _fingerprint(self.to_dict()) + + +class ConnectorActionAdapter(Protocol): + connector_name: str + + def snapshot(self, spec: ActionSpec) -> Mapping[str, Any]: ... + + def preview(self, spec: ActionSpec, before: Mapping[str, Any]) -> Mapping[str, Any]: ... + + def execute(self, spec: ActionSpec, preview: Mapping[str, Any]) -> Mapping[str, Any]: ... + + +def build_preview(adapter: ConnectorActionAdapter, spec: ActionSpec, *, created_at: str | None = None) -> dict[str, Any]: + if adapter.connector_name != spec.connector: + raise ValueError("adapter connector does not match action spec") + before = dict(adapter.snapshot(spec)) + _json_value(before, "provider before state") + before_sha = _fingerprint(before) + if spec.expected_before_sha256 is not None and before_sha != spec.expected_before_sha256: + raise ValueError("current remote state does not match expected_before_sha256") + changes = dict(adapter.preview(spec, before)) + _json_value(changes, "provider proposed changes") + material = { + "schema_version": 1, + "connector": spec.connector, + "operation": spec.operation, + "mode": spec.mode, + "target": spec.target, + "action_spec_sha256": spec.fingerprint, + "idempotency_key": spec.idempotency_key, + "requested_scopes": list(spec.requested_scopes), + "before_state_sha256": before_sha, + "before_state": before, + "proposed_changes": changes, + "approval_required": spec.approval_required, + "destructive": spec.destructive, + "paid": spec.paid, + "sensitive_data": spec.sensitive_data, + "compensation": spec.compensation, + "created_at": created_at or _now(), + "status": "preview", + "executed": False, + "evidence_boundary": "The preview describes a provider-specific proposed action and current state. It does not grant permission, reserve cost, or prove that execution will produce the previewed state." + } + fingerprint = _fingerprint(material) + return {**material, "preview_id": f"preview-{fingerprint[:20]}", "fingerprint": fingerprint} + + +def validate_preview(payload: Mapping[str, Any], spec: ActionSpec | None = None) -> None: + if payload.get("schema_version") != 1 or payload.get("status") != "preview" or payload.get("executed") is not False: + raise ValueError("invalid action preview state") + for field in ("preview_id", "connector", "operation", "target", "action_spec_sha256", "idempotency_key", "before_state_sha256", "created_at", "evidence_boundary"): + _text(payload.get(field), field) + _sha(payload.get("action_spec_sha256"), "action_spec_sha256") + _sha(payload.get("before_state_sha256"), "before_state_sha256") + material = dict(payload) + preview_id = str(material.pop("preview_id")) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint or preview_id != f"preview-{fingerprint[:20]}": + raise ValueError("action preview fingerprint or ID mismatch") + if spec is not None and payload.get("action_spec_sha256") != spec.fingerprint: + raise ValueError("action preview covers a different spec") + + +def build_action_approval(preview: Mapping[str, Any], *, approved_by: str, approved_scopes: list[str], approved_at: str | None = None) -> dict[str, Any]: + validate_preview(preview) + scopes = sorted({_text(item, "approved scope") for item in approved_scopes}) + requested = set(map(str, preview.get("requested_scopes", []))) + if not requested.issubset(scopes): + raise ValueError("approved scopes do not cover requested scopes") + material = { + "schema_version": 1, + "preview_id": preview["preview_id"], + "preview_fingerprint": preview["fingerprint"], + "action_spec_sha256": preview["action_spec_sha256"], + "connector": preview["connector"], + "operation": preview["operation"], + "target": preview["target"], + "before_state_sha256": preview["before_state_sha256"], + "idempotency_key": preview["idempotency_key"], + "approved_by": _text(approved_by, "approved_by"), + "approved_scopes": scopes, + "approved_at": approved_at or _now(), + "approved": True, + "evidence_boundary": "Approval authorizes the exact preview and scopes. It does not endorse the scientific content or waive provider, privacy, cost, or regulatory obligations." + } + return {**material, "fingerprint": _fingerprint(material)} + + +def validate_action_approval(approval: Mapping[str, Any], preview: Mapping[str, Any]) -> None: + validate_preview(preview) + if approval.get("schema_version") != 1 or approval.get("approved") is not True: + raise ValueError("valid action approval is required") + for field in ("preview_id", "preview_fingerprint", "action_spec_sha256", "before_state_sha256", "idempotency_key"): + if approval.get(field) != preview.get(field if field != "preview_fingerprint" else "fingerprint"): + raise ValueError(f"approval {field} does not match preview") + material = dict(approval) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint: + raise ValueError("action approval fingerprint mismatch") + + +class ActionLedger: + def __init__(self, path: Path) -> None: + self.path = path.resolve() + self.path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.path) as connection: + connection.execute("PRAGMA journal_mode=WAL") + connection.execute( + "CREATE TABLE IF NOT EXISTS actions(idempotency_key TEXT PRIMARY KEY, action_spec_sha256 TEXT NOT NULL, receipt_json TEXT NOT NULL, created_at TEXT NOT NULL)" + ) + + def get(self, key: str) -> dict[str, Any] | None: + with sqlite3.connect(self.path) as connection: + row = connection.execute("SELECT receipt_json FROM actions WHERE idempotency_key=?", (key,)).fetchone() + return None if row is None else json.loads(str(row[0])) + + def record(self, *, key: str, spec_sha256: str, receipt: Mapping[str, Any]) -> None: + with sqlite3.connect(self.path) as connection: + existing = connection.execute("SELECT action_spec_sha256, receipt_json FROM actions WHERE idempotency_key=?", (key,)).fetchone() + if existing is not None: + if str(existing[0]) != spec_sha256 or json.loads(str(existing[1])) != dict(receipt): + raise ValueError("idempotency key is already bound to a different action") + return + connection.execute( + "INSERT INTO actions(idempotency_key, action_spec_sha256, receipt_json, created_at) VALUES(?,?,?,?)", + (key, spec_sha256, json.dumps(dict(receipt), sort_keys=True, separators=(",", ":")), _now()), + ) + + +def execute_action( + adapter: ConnectorActionAdapter, + spec: ActionSpec, + preview: Mapping[str, Any], + *, + approval: Mapping[str, Any] | None, + ledger: ActionLedger, + executed_at: str | None = None, +) -> dict[str, Any]: + validate_preview(preview, spec) + if adapter.connector_name != spec.connector: + raise ValueError("adapter connector does not match action spec") + existing = ledger.get(spec.idempotency_key) + if existing is not None: + if existing.get("action_spec_sha256") != spec.fingerprint: + raise ValueError("idempotency key is already bound to a different spec") + return existing + if spec.mode == "write": + if approval is None: + raise ValueError("write action requires explicit approval") + validate_action_approval(approval, preview) + current = dict(adapter.snapshot(spec)) + _json_value(current, "provider current state") + current_sha = _fingerprint(current) + if current_sha != preview["before_state_sha256"]: + raise ValueError("remote state changed after preview; create a new preview") + result = dict(adapter.execute(spec, preview)) + _json_value(result, "provider execution result") + after = dict(result.get("after_state", result)) + after_sha = _fingerprint(after) + material = { + "schema_version": 1, + "action_id": "action-" + uuid.uuid4().hex[:20], + "connector": spec.connector, + "operation": spec.operation, + "mode": spec.mode, + "target": spec.target, + "action_spec_sha256": spec.fingerprint, + "preview_id": preview["preview_id"], + "preview_fingerprint": preview["fingerprint"], + "approval_fingerprint": None if approval is None else approval.get("fingerprint"), + "idempotency_key": spec.idempotency_key, + "before_state_sha256": current_sha, + "after_state_sha256": after_sha, + "after_state": after, + "remote_object_ids": list(result.get("remote_object_ids", [])) if isinstance(result.get("remote_object_ids", []), list) else [], + "provider_receipt": result.get("provider_receipt"), + "compensation": spec.compensation, + "executed_at": executed_at or _now(), + "status": "executed", + "evidence_boundary": "This receipt records a provider adapter action and before/after state hashes. It does not validate the scientific content, provider completeness, legal authorization, or successful downstream interpretation." + } + receipt = {**material, "fingerprint": _fingerprint(material)} + ledger.record(key=spec.idempotency_key, spec_sha256=spec.fingerprint, receipt=receipt) + return receipt + + +def validate_action_receipt(payload: Mapping[str, Any], spec: ActionSpec | None = None) -> None: + if payload.get("schema_version") != 1 or payload.get("status") != "executed": + raise ValueError("invalid action receipt state") + for field in ( + "action_id", + "connector", + "operation", + "mode", + "target", + "action_spec_sha256", + "preview_id", + "preview_fingerprint", + "idempotency_key", + "before_state_sha256", + "after_state_sha256", + "executed_at", + "evidence_boundary", + ): + _text(payload.get(field), field) + for field in ( + "action_spec_sha256", + "preview_fingerprint", + "before_state_sha256", + "after_state_sha256", + ): + _sha(payload.get(field), field) + material = dict(payload) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint: + raise ValueError("action receipt fingerprint mismatch") + if spec is not None and payload.get("action_spec_sha256") != spec.fingerprint: + raise ValueError("action receipt covers a different spec") diff --git a/src/codex_science/advanced_sidecars.py b/src/codex_science/advanced_sidecars.py index 95e0e88f..3ab1503a 100644 --- a/src/codex_science/advanced_sidecars.py +++ b/src/codex_science/advanced_sidecars.py @@ -5,13 +5,25 @@ from pathlib import Path from typing import Any +from codex_science.artifact_store import validate_descriptor +from codex_science.artifact_runtime import ( + stale_selection, + validate_runtime_descriptor, + validate_selection, + validate_transform_proposal, +) from codex_science.collaboration import stale_annotations, validate_annotation from codex_science.evidence_graph_v2 import independent_support_groups, validate_graph_payload from codex_science.literature_v2 import validate_risk_of_bias from codex_science.model_registry_v2 import load_registry_v2, validate_model_receipt_v2 +from codex_science.reference_contract import validate_reference_use_ledger from codex_science.review_receipts import review_receipt_findings, validate_review_receipt -ADVANCED_KINDS = {"evidence-graph-v2", "review-receipt", "review-receipt-v2", "annotation", "model-receipt-v2", "risk-of-bias"} +ADVANCED_KINDS = { + "evidence-graph-v2", "review-receipt", "review-receipt-v2", "annotation", + "model-receipt-v2", "risk-of-bias", "reference-use-ledger", "artifact-descriptor", + "artifact-runtime-descriptor", "artifact-selection", "transform-proposal", +} def _load(path: Path, label: str) -> dict[str, Any]: @@ -28,21 +40,38 @@ def validate_advanced_sidecars( manifest: dict[str, Any], run_dir: Path, verified: dict[str, Path], *, base_sidecars: dict[str, Any] | None = None ) -> dict[str, Any]: result: dict[str, Any] = { - "graph_v2": None, "graph_v2_nodes": {}, "graph_v2_edges": [], "graph_v2_findings": [], - "review_receipts": [], "annotations": [], "model_receipts_v2": [], "risk_of_bias": [], - "artifact_hashes": {str(item["path"]): str(item["sha256"]).lower() for item in manifest.get("artifacts", [])}, + "graph_v2": None, + "graph_v2_nodes": {}, + "graph_v2_edges": [], + "graph_v2_findings": [], + "review_receipts": [], + "annotations": [], + "model_receipts_v2": [], + "risk_of_bias": [], + "reference_use_ledgers": [], + "artifact_descriptors": [], + "runtime_descriptors": [], + "artifact_selections": [], + "transform_proposals": [], + "runtime_descriptor_stale": [], + "artifact_hashes": { + str(item["path"]): str(item["sha256"]).lower() + for item in manifest.get("artifacts", []) + }, "advanced_paths": {}, } seen_graph = False + selections_by_id: dict[str, dict[str, Any]] = {} + pending_proposals: list[dict[str, Any]] = [] for record in manifest.get("artifacts", []): kind, relative = str(record.get("kind", "")), str(record.get("path", "")) if kind not in ADVANCED_KINDS: continue path = verified[relative] + if not path.is_file(): + raise ValueError(f"advanced sidecar must be a file: {relative}") payload = _load(path, kind) if kind == "review-receipt" and not payload.get("review_id"): - # Legacy receipt from schema-v1 bundles: keep it hashed and navigable, - # but do not reinterpret it as the new hash-covered receipt contract. continue result["advanced_paths"].setdefault(kind, []).append(relative) if kind == "evidence-graph-v2": @@ -50,7 +79,12 @@ def validate_advanced_sidecars( raise ValueError("only one evidence-graph-v2 artifact is allowed") seen_graph = True nodes, edges, findings = validate_graph_payload(payload) - result.update({"graph_v2": payload, "graph_v2_nodes": nodes, "graph_v2_edges": edges, "graph_v2_findings": findings}) + result.update({ + "graph_v2": payload, + "graph_v2_nodes": nodes, + "graph_v2_edges": edges, + "graph_v2_findings": findings, + }) elif kind in {"review-receipt", "review-receipt-v2"}: validate_review_receipt(payload) result["review_receipts"].append(payload) @@ -62,6 +96,29 @@ def validate_advanced_sidecars( elif kind == "risk-of-bias": validate_risk_of_bias(payload) result["risk_of_bias"].append(payload) + elif kind == "reference-use-ledger": + validate_reference_use_ledger(payload) + result["reference_use_ledgers"].append(payload) + elif kind == "artifact-descriptor": + result["artifact_descriptors"].append(validate_descriptor(payload).to_dict()) + elif kind == "artifact-runtime-descriptor": + validate_runtime_descriptor(payload) + if result["artifact_hashes"].get(str(payload["artifact_path"])) != str( + payload["artifact_sha256"] + ).lower(): + result["runtime_descriptor_stale"].append(payload) + result["runtime_descriptors"].append(payload) + elif kind == "artifact-selection": + validate_selection(payload) + selection = stale_selection(payload, result["artifact_hashes"]) + result["artifact_selections"].append(selection) + selections_by_id[str(selection["selection_id"])] = payload + elif kind == "transform-proposal": + pending_proposals.append(payload) + for proposal in pending_proposals: + selection = selections_by_id.get(str(proposal.get("selection_id", ""))) + validate_transform_proposal(proposal, selection) + result["transform_proposals"].append(proposal) result["annotations"] = stale_annotations(result["annotations"], result["artifact_hashes"]) if base_sidecars: result["base_sidecars"] = base_sidecars @@ -82,23 +139,78 @@ def review_advanced_sidecars(sidecars: dict[str, Any], *, registry_path: Path | findings.extend(review_receipt_findings(receipt, artifact_hashes, registry_sha256=registry_digest)) if receipt.get("status") != "passed": unresolved = [ - item for item in receipt.get("findings", []) + item + for item in receipt.get("findings", []) if item.get("resolution_status", "open") != "resolved" ] if unresolved: findings.extend(unresolved) else: - findings.append({"code": "review-receipt-not-passed", "severity": "major", "message": f"Review receipt {receipt['review_id']} is {receipt.get('status')}."}) + findings.append({ + "code": "review-receipt-not-passed", + "severity": "major", + "message": f"Review receipt {receipt['review_id']} is {receipt.get('status')}.", + }) for receipt in sidecars.get("model_receipts_v2", []): findings.extend(validate_model_receipt_v2(receipt, registry)) for annotation in sidecars.get("annotations", []): if annotation.get("status") == "stale-anchor": - findings.append({"code": "stale-annotation-anchor", "severity": "minor", "message": f"Annotation {annotation['annotation_id']} points to changed artifact bytes."}) + findings.append({ + "code": "stale-annotation-anchor", + "severity": "minor", + "message": f"Annotation {annotation['annotation_id']} points to changed artifact bytes.", + }) + for descriptor in sidecars.get("runtime_descriptor_stale", []): + findings.append({ + "code": "stale-runtime-descriptor", + "severity": "minor", + "message": ( + "Artifact runtime descriptor covers changed or missing bytes: " + f"{descriptor.get('artifact_path', '')}" + ), + }) + for selection in sidecars.get("artifact_selections", []): + if selection.get("status") == "stale-anchor": + findings.append({ + "code": "stale-artifact-selection", + "severity": "minor", + "message": ( + f"Artifact selection {selection.get('selection_id')} points to " + "changed or missing bytes." + ), + }) + selection_ids = { + str(item.get("selection_id")) + for item in sidecars.get("artifact_selections", []) + } + for proposal in sidecars.get("transform_proposals", []): + if str(proposal.get("selection_id")) not in selection_ids: + findings.append({ + "code": "missing-transform-selection", + "severity": "major", + "message": ( + f"Transform proposal {proposal.get('proposal_id')} references " + "a selection not present in the bundle." + ), + }) base = sidecars.get("base_sidecars", {}) claims = base.get("claim_by_id", {}) if isinstance(base, dict) else {} nodes, edges = sidecars.get("graph_v2_nodes", {}), sidecars.get("graph_v2_edges", []) for claim_id, claim in claims.items(): if claim.get("status") == "replicated" and len(independent_support_groups(claim_id, nodes, edges)) < 2: - findings.append({"code": "dependent-evidence-v2", "severity": "major", "claim_id": claim_id, "message": f"Claim {claim_id} is replicated without two independent evidence components."}) - unique = {(item["code"], item.get("claim_id", ""), item["message"]): item for item in findings} - return sorted(unique.values(), key=lambda item: (item.get("severity", ""), item["code"], item.get("claim_id", ""), item["message"])) + findings.append({ + "code": "dependent-evidence-v2", + "severity": "major", + "claim_id": claim_id, + "message": f"Claim {claim_id} is replicated without two independent evidence components.", + }) + unique = { + (item["code"], item.get("claim_id", ""), item["message"]): item + for item in findings + } + return sorted( + unique.values(), + key=lambda item: ( + item.get("severity", ""), item["code"], item.get("claim_id", ""), item["message"] + ), + ) diff --git a/src/codex_science/artifact_runtime.py b/src/codex_science/artifact_runtime.py new file mode 100644 index 00000000..03e2dde2 --- /dev/null +++ b/src/codex_science/artifact_runtime.py @@ -0,0 +1,748 @@ +"""Bounded native scientific artifact views, selections, and edit proposals. + +The runtime deliberately separates three concepts: + +* a *view* is a bounded, derived description of validated artifact bytes; +* a *selection* is a typed, hash-bound pointer into those bytes; +* a *transform proposal* is a non-executing request that can feed impact and + selective-rerun planning. + +None of these objects are scientific evidence by themselves and no function in +this module mutates the source artifact. +""" + +from __future__ import annotations + +import csv +import hashlib +import html +import io +import json +import struct +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping + +from codex_science.artifact_store import describe_directory, stream_sha256 + + +DEFAULT_MAX_BYTES = 1024 * 1024 +DEFAULT_MAX_RECORDS = 100 + +VIEWER_SELECTIONS: dict[str, frozenset[str]] = { + "structure-3d": frozenset({"atom", "residue", "chain", "ligand", "spatial-region"}), + "molecule-2d": frozenset({"atom", "bond", "substructure", "record"}), + "genome-track": frozenset({"variant", "locus", "interval", "track-record"}), + "single-cell": frozenset({"cell", "cell-set", "feature", "cluster", "donor", "layer"}), + "table": frozenset({"row", "column", "cell", "range"}), + "figure": frozenset({"region", "axis", "legend", "panel"}), + "evidence-graph": frozenset({"node", "edge", "claim", "component"}), + "trajectory": frozenset({"frame", "frame-range", "atom", "atom-set"}), + "directory": frozenset({"entry", "entry-set"}), + "text": frozenset({"line", "line-range", "heading", "json-pointer"}), + "binary": frozenset({"byte-range", "record"}), +} + +MAX_RUNTIME_BYTES = 16 * 1024 * 1024 +MAX_RUNTIME_RECORDS = 10_000 +MAX_SELECTION_BYTES = 64 * 1024 +MAX_PROPOSAL_PARAMETER_BYTES = 64 * 1024 +MAX_REASON_CHARACTERS = 4_000 + +STRUCTURE_SUFFIXES = {".pdb", ".ent", ".cif", ".mmcif"} +MOLECULE_SUFFIXES = {".sdf", ".mol", ".mol2", ".smi", ".smiles"} +GENOME_SUFFIXES = {".vcf", ".bed", ".gff", ".gff3", ".gtf", ".wig", ".bedgraph", ".bigwig", ".bw"} +SINGLE_CELL_SUFFIXES = {".h5ad", ".loom"} +TABLE_SUFFIXES = {".csv", ".tsv", ".parquet", ".feather", ".arrow"} +FIGURE_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".tif", ".tiff"} +GRAPH_SUFFIXES = {".graph.json", ".cyjs"} +TRAJECTORY_SUFFIXES = {".xyz", ".dcd", ".xtc", ".trr", ".nc", ".netcdf"} +TEXT_SUFFIXES = {".txt", ".md", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".py", ".r", ".jl", ".sh"} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _safe_relative(value: str, label: str = "path") -> str: + pure = PurePosixPath(value) + if not value or pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"{label} must be a safe relative path: {value}") + return pure.as_posix() + + +def _sha256(value: Any, label: str) -> str: + text = str(value).strip().lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{label} must be a SHA-256 digest") + return text + + +def _json_value(value: Any, label: str) -> None: + if value is None or isinstance(value, (str, int, float, bool)): + return + if isinstance(value, list): + for index, item in enumerate(value): + _json_value(item, f"{label}[{index}]") + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"{label} object keys must be strings") + _json_value(item, f"{label}.{key}") + return + raise ValueError(f"{label} must contain JSON-compatible values") + + +def _read_head(path: Path, max_bytes: int) -> tuple[bytes, bool]: + if max_bytes < 1: + raise ValueError("max_bytes must be positive") + with path.open("rb") as handle: + data = handle.read(max_bytes + 1) + return data[:max_bytes], len(data) > max_bytes + + +def _decode_text(data: bytes) -> tuple[str, str]: + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + return data.decode(encoding), encoding + except UnicodeDecodeError: + continue + return data.decode("utf-8", errors="replace"), "utf-8-replacement" + + +def detect_viewer(path: Path, *, kind: str = "", media_type: str | None = None) -> str: + if path.is_dir(): + return "directory" + lower = path.name.lower() + suffix = path.suffix.lower() + kind_lower = kind.lower() + media = (media_type or "").lower() + if suffix in STRUCTURE_SUFFIXES or "structure" in kind_lower or "chemical/x-pdb" in media: + return "structure-3d" + if suffix in MOLECULE_SUFFIXES or "molecule" in kind_lower or "chemical/x-mdl" in media: + return "molecule-2d" + if suffix in GENOME_SUFFIXES or any(token in kind_lower for token in ("variant", "genome", "track")): + return "genome-track" + if suffix in SINGLE_CELL_SUFFIXES or "anndata" in kind_lower or "single-cell" in kind_lower: + return "single-cell" + if suffix in TABLE_SUFFIXES or media in {"text/csv", "text/tab-separated-values", "application/vnd.apache.parquet"}: + return "table" + if suffix in FIGURE_SUFFIXES or media.startswith("image/") or "figure" in kind_lower: + return "figure" + if lower.endswith(".graph.json") or suffix in GRAPH_SUFFIXES or "evidence-graph" in kind_lower: + return "evidence-graph" + if suffix in TRAJECTORY_SUFFIXES or "trajectory" in kind_lower: + return "trajectory" + if suffix in TEXT_SUFFIXES or media.startswith("text/") or media.endswith("+json") or media == "application/json": + return "text" + return "binary" + + +def _bounded(items: Iterable[Any], maximum: int) -> tuple[list[Any], bool]: + result: list[Any] = [] + iterator = iter(items) + for _ in range(maximum): + try: + result.append(next(iterator)) + except StopIteration: + return result, False + try: + next(iterator) + except StopIteration: + return result, False + return result, True + + +def _structure_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + data, truncated_bytes = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + warnings: list[str] = [] + atoms: list[dict[str, Any]] = [] + chains: set[str] = set() + residues: set[str] = set() + ligands: set[str] = set() + if path.suffix.lower() in {".pdb", ".ent"}: + for line in text.splitlines(): + if not line.startswith(("ATOM ", "HETATM")): + continue + try: + atom = { + "record": line[:6].strip(), + "serial": int(line[6:11].strip() or 0), + "atom": line[12:16].strip(), + "altloc": line[16:17].strip(), + "resname": line[17:20].strip(), + "chain": line[21:22].strip(), + "resseq": line[22:26].strip(), + "x": float(line[30:38]), + "y": float(line[38:46]), + "z": float(line[46:54]), + "element": line[76:78].strip() if len(line) >= 78 else "", + } + except (ValueError, IndexError): + warnings.append("At least one coordinate record could not be normalized.") + continue + chains.add(atom["chain"] or "_") + residues.add(f"{atom['chain'] or '_'}:{atom['resname']}:{atom['resseq']}") + if atom["record"] == "HETATM" and atom["resname"] not in {"HOH", "WAT"}: + ligands.add(atom["resname"]) + if len(atoms) < max_records: + atoms.append(atom) + preview = { + "format": "pdb", + "encoding": encoding, + "atoms_preview": atoms, + "chains_seen": sorted(chains), + "residues_seen": len(residues), + "ligands_seen": sorted(ligands), + } + else: + categories = sorted({line.split(".", 1)[0] for line in text.splitlines() if line.startswith("_")}) + atom_lines = [line for line in text.splitlines() if line.startswith("ATOM ") or line.startswith("HETATM ")] + preview = { + "format": "mmcif", + "encoding": encoding, + "categories_preview": categories[:max_records], + "coordinate_rows_preview": atom_lines[:max_records], + } + if len(categories) > max_records or len(atom_lines) > max_records: + warnings.append("The mmCIF preview is record bounded.") + if truncated_bytes: + warnings.append("The structure preview is byte bounded; counts describe only the inspected prefix.") + return preview, warnings + + +def _molecule_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + data, truncated_bytes = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + warnings: list[str] = [] + suffix = path.suffix.lower() + if suffix in {".smi", ".smiles"}: + records = [] + for line_number, line in enumerate(text.splitlines(), 1): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + fields = stripped.split() + records.append({"line": line_number, "smiles": fields[0], "name": " ".join(fields[1:])}) + if len(records) >= max_records: + break + preview = {"format": "smiles", "encoding": encoding, "records_preview": records} + else: + blocks = text.split("$$$$") + records = [] + for index, block in enumerate(blocks): + lines = block.strip("\r\n").splitlines() + if not lines: + continue + atom_count = bond_count = None + if len(lines) >= 4: + counts = lines[3] + try: + atom_count = int(counts[0:3]) + bond_count = int(counts[3:6]) + except ValueError: + pass + properties = [line[3:-1] for line in lines if line.startswith("> <") and line.endswith(">")] + records.append({ + "record_index": index, + "name": lines[0].strip() if lines else "", + "atom_count": atom_count, + "bond_count": bond_count, + "properties": properties[:25], + }) + if len(records) >= max_records: + break + preview = {"format": suffix.lstrip(".") or "mol", "encoding": encoding, "records_preview": records} + if truncated_bytes: + warnings.append("The molecular preview is byte bounded and may end inside a record.") + return preview, warnings + + +def _genome_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + if path.suffix.lower() in {".bigwig", ".bw"}: + data, truncated = _read_head(path, min(max_bytes, 4096)) + return ({"format": "bigwig", "magic_hex": data[:4].hex(), "header_bytes_preview": data[:64].hex()}, ["bigWig is represented by bounded binary metadata; track values require a dedicated indexed reader."] + (["Header preview truncated."] if truncated else [])) + data, truncated_bytes = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + headers: list[str] = [] + records: list[dict[str, Any]] = [] + suffix = path.suffix.lower() + for line_number, line in enumerate(text.splitlines(), 1): + if not line: + continue + if line.startswith("#") or line.startswith("track") or line.startswith("browser"): + if len(headers) < 50: + headers.append(line) + continue + fields = line.split("\t") + if suffix == ".vcf" and len(fields) >= 8: + record = {"line": line_number, "chrom": fields[0], "pos": fields[1], "id": fields[2], "ref": fields[3], "alt": fields[4], "qual": fields[5], "filter": fields[6]} + elif len(fields) >= 3: + record = {"line": line_number, "chrom": fields[0], "start": fields[1], "end": fields[2], "fields": fields[3:10]} + else: + record = {"line": line_number, "raw": line[:500]} + records.append(record) + if len(records) >= max_records: + break + warnings = ["The genome-track preview is byte bounded."] if truncated_bytes else [] + return {"format": suffix.lstrip("."), "encoding": encoding, "headers_preview": headers, "records_preview": records}, warnings + + +def _table_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + suffix = path.suffix.lower() + if suffix in {".parquet", ".feather", ".arrow"}: + data, truncated = _read_head(path, min(max_bytes, 8192)) + magic = {"prefix": data[:8].hex(), "suffix_available": False} + if path.stat().st_size >= 8: + with path.open("rb") as handle: + handle.seek(max(0, path.stat().st_size - 8)) + magic["suffix"] = handle.read(8).hex() + magic["suffix_available"] = True + return ({"format": suffix.lstrip("."), "binary_metadata": magic}, ["Column statistics and rows require a dedicated Arrow/Parquet reader; no schema is inferred from filename alone."] + (["Header preview truncated."] if truncated else [])) + data, truncated_bytes = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + delimiter = "\t" if suffix == ".tsv" else "," + try: + dialect = csv.Sniffer().sniff(text[:8192], delimiters=",\t;|") + delimiter = dialect.delimiter + except csv.Error: + pass + reader = csv.reader(io.StringIO(text), delimiter=delimiter) + rows, more = _bounded(reader, max_records + 1) + header = rows[0] if rows else [] + body = rows[1:max_records + 1] if rows else [] + warnings = [] + if truncated_bytes or more: + warnings.append("The table preview is bounded and is not a complete row count.") + return {"format": "delimited-text", "encoding": encoding, "delimiter": delimiter, "columns": header, "rows_preview": body}, warnings + + +def _png_dimensions(data: bytes) -> tuple[int, int] | None: + if len(data) >= 24 and data.startswith(b"\x89PNG\r\n\x1a\n"): + return struct.unpack(">II", data[16:24]) + return None + + +def _jpeg_dimensions(data: bytes) -> tuple[int, int] | None: + if not data.startswith(b"\xff\xd8"): + return None + offset = 2 + while offset + 9 < len(data): + if data[offset] != 0xFF: + offset += 1 + continue + marker = data[offset + 1] + offset += 2 + if marker in {0xD8, 0xD9}: + continue + if offset + 2 > len(data): + break + length = int.from_bytes(data[offset:offset + 2], "big") + if marker in {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} and offset + 7 < len(data): + height = int.from_bytes(data[offset + 3:offset + 5], "big") + width = int.from_bytes(data[offset + 5:offset + 7], "big") + return width, height + if length < 2: + break + offset += length + return None + + +def _figure_preview(path: Path, max_bytes: int) -> tuple[dict[str, Any], list[str]]: + data, truncated = _read_head(path, min(max_bytes, 2 * 1024 * 1024)) + dimensions = _png_dimensions(data) or _jpeg_dimensions(data) + preview: dict[str, Any] = {"format": path.suffix.lower().lstrip("."), "dimensions": None if dimensions is None else {"width": dimensions[0], "height": dimensions[1]}} + warnings: list[str] = [] + if path.suffix.lower() == ".svg": + text, encoding = _decode_text(data) + preview["encoding"] = encoding + preview["markup_preview"] = text[:5000] + warnings.append("SVG markup is escaped in the runtime and is never executed as active content.") + if dimensions is None and path.suffix.lower() not in {".svg"}: + warnings.append("Image dimensions were not decoded by the bounded standard-library reader.") + if truncated: + warnings.append("Figure byte preview was truncated.") + return preview, warnings + + +def _graph_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + data, truncated = _read_head(path, max_bytes) + warnings: list[str] = [] + try: + payload = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return {"format": "json", "parsed": False, "text_preview": _decode_text(data)[0][:5000]}, ["The bounded prefix did not contain a complete JSON graph."] + nodes = payload.get("nodes", []) if isinstance(payload, dict) else [] + edges = payload.get("edges", []) if isinstance(payload, dict) else [] + if not isinstance(nodes, list) or not isinstance(edges, list): + warnings.append("JSON does not expose list-valued nodes and edges.") + nodes, edges = [], [] + preview = { + "format": "evidence-graph-json", + "schema_version": payload.get("schema_version") if isinstance(payload, dict) else None, + "node_count": len(nodes), + "edge_count": len(edges), + "nodes_preview": nodes[:max_records], + "edges_preview": edges[:max_records], + } + if truncated: + warnings.append("Graph JSON happened to parse from a bounded prefix; verify full artifact validation before relying on counts.") + return preview, warnings + + +def _trajectory_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + if path.suffix.lower() != ".xyz": + data, truncated = _read_head(path, min(max_bytes, 4096)) + return ({"format": path.suffix.lower().lstrip("."), "header_hex": data[:128].hex()}, ["Binary trajectory frames require a format-specific reader; only bounded metadata is exposed."] + (["Header preview truncated."] if truncated else [])) + data, truncated = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + lines = text.splitlines() + offset = 0 + frames: list[dict[str, Any]] = [] + while offset < len(lines) and len(frames) < max_records: + try: + atom_count = int(lines[offset].strip()) + except (ValueError, IndexError): + break + comment = lines[offset + 1] if offset + 1 < len(lines) else "" + atom_lines = lines[offset + 2:offset + 2 + atom_count] + if len(atom_lines) < atom_count: + break + frames.append({"frame": len(frames), "atom_count": atom_count, "comment": comment, "atoms_preview": atom_lines[:20]}) + offset += atom_count + 2 + warnings = ["XYZ trajectory preview is byte or frame bounded."] if truncated or offset < len(lines) else [] + return {"format": "xyz", "encoding": encoding, "frames_preview": frames}, warnings + + +def _single_cell_preview(path: Path, max_bytes: int) -> tuple[dict[str, Any], list[str]]: + data, truncated = _read_head(path, min(max_bytes, 8192)) + is_hdf5 = data.startswith(b"\x89HDF\r\n\x1a\n") + return ({"format": path.suffix.lower().lstrip("."), "container": "hdf5" if is_hdf5 else "unknown", "header_hex": data[:128].hex()}, ["AnnData/Loom matrices are not loaded without an explicit HDF5-aware backend; dimensions, layers, donors, and embeddings must not be guessed."] + (["Header preview truncated."] if truncated else [])) + + +def _directory_preview(path: Path, max_records: int, media_type: str | None) -> tuple[dict[str, Any], list[str]]: + descriptor = describe_directory(path, media_type=media_type) + return ({"format": "directory-tree", "root_sha256": descriptor.root_sha256, "total_bytes": descriptor.total_bytes, "entry_count": descriptor.entry_count, "entries_preview": list(descriptor.entries[:max_records])}, ["Directory view is entry bounded."] if descriptor.entry_count > max_records else []) + + +def _text_preview(path: Path, max_bytes: int, max_records: int) -> tuple[dict[str, Any], list[str]]: + data, truncated = _read_head(path, max_bytes) + text, encoding = _decode_text(data) + lines = text.splitlines() + preview: dict[str, Any] = {"encoding": encoding, "line_count_in_prefix": len(lines), "lines_preview": [{"line": index + 1, "text": line} for index, line in enumerate(lines[:max_records])]} + if path.suffix.lower() == ".json" and not truncated: + try: + payload = json.loads(text) + preview["json_type"] = type(payload).__name__ + if isinstance(payload, dict): + preview["top_level_keys"] = sorted(map(str, payload.keys()))[:max_records] + except json.JSONDecodeError: + preview["json_type"] = "invalid" + return preview, ["Text preview is byte or line bounded."] if truncated or len(lines) > max_records else [] + + +def _binary_preview(path: Path, max_bytes: int) -> tuple[dict[str, Any], list[str]]: + data, truncated = _read_head(path, min(max_bytes, 4096)) + return {"header_hex": data[:256].hex(), "header_ascii": "".join(chr(byte) if 32 <= byte < 127 else "." for byte in data[:256])}, ["Binary preview is metadata only."] + (["Header preview truncated."] if truncated else []) + + +@dataclass(frozen=True) +class ArtifactRuntimeDescriptor: + schema_version: int + artifact_path: str + artifact_sha256: str + artifact_kind: str + artifact_type: str + media_type: str | None + size_bytes: int + viewer: str + generated_at: str + bounds: dict[str, int] + preview: dict[str, Any] + warnings: tuple[str, ...] + evidence_boundary: str + fingerprint: str + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["warnings"] = list(self.warnings) + return payload + + +def describe_runtime( + path: Path, + *, + artifact_path: str | None = None, + artifact_sha256: str | None = None, + kind: str = "artifact", + artifact_type: str | None = None, + media_type: str | None = None, + max_bytes: int = DEFAULT_MAX_BYTES, + max_records: int = DEFAULT_MAX_RECORDS, + generated_at: str | None = None, +) -> ArtifactRuntimeDescriptor: + path = path.resolve() + if not path.exists(): + raise ValueError(f"artifact does not exist: {path}") + if max_bytes < 1 or max_records < 1: + raise ValueError("max_bytes and max_records must be positive") + if max_bytes > MAX_RUNTIME_BYTES: + raise ValueError(f"max_bytes exceeds hard runtime ceiling: {MAX_RUNTIME_BYTES}") + if max_records > MAX_RUNTIME_RECORDS: + raise ValueError(f"max_records exceeds hard runtime ceiling: {MAX_RUNTIME_RECORDS}") + relative = _safe_relative(artifact_path or path.name, "artifact_path") + viewer = detect_viewer(path, kind=kind, media_type=media_type) + if path.is_dir(): + directory = describe_directory(path, media_type=media_type) + digest, size, resolved_type = directory.root_sha256, directory.total_bytes, "directory-tree" + else: + digest, size = stream_sha256(path) + resolved_type = artifact_type or "file" + if artifact_sha256 is not None and _sha256(artifact_sha256, "artifact_sha256") != digest: + raise ValueError("artifact bytes do not match artifact_sha256") + parsers = { + "structure-3d": lambda: _structure_preview(path, max_bytes, max_records), + "molecule-2d": lambda: _molecule_preview(path, max_bytes, max_records), + "genome-track": lambda: _genome_preview(path, max_bytes, max_records), + "single-cell": lambda: _single_cell_preview(path, max_bytes), + "table": lambda: _table_preview(path, max_bytes, max_records), + "figure": lambda: _figure_preview(path, max_bytes), + "evidence-graph": lambda: _graph_preview(path, max_bytes, max_records), + "trajectory": lambda: _trajectory_preview(path, max_bytes, max_records), + "directory": lambda: _directory_preview(path, max_records, media_type), + "text": lambda: _text_preview(path, max_bytes, max_records), + "binary": lambda: _binary_preview(path, max_bytes), + } + preview, warnings = parsers[viewer]() + material = { + "schema_version": 1, + "artifact_path": relative, + "artifact_sha256": digest, + "artifact_kind": kind, + "artifact_type": resolved_type, + "media_type": media_type, + "size_bytes": size, + "viewer": viewer, + "generated_at": generated_at or _now(), + "bounds": {"max_bytes": max_bytes, "max_records": max_records}, + "preview": preview, + "warnings": warnings, + "evidence_boundary": "This is a bounded derived view of hash-validated bytes. It does not prove a scientific claim, execute an edit, or replace format-specific validation.", + } + return ArtifactRuntimeDescriptor( + **{**material, "warnings": tuple(warnings)}, + fingerprint=_fingerprint(material), + ) + + +def validate_runtime_descriptor(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported artifact runtime descriptor schema") + _safe_relative(str(payload.get("artifact_path", "")), "artifact_path") + _sha256(payload.get("artifact_sha256"), "artifact_sha256") + viewer = str(payload.get("viewer", "")) + if viewer not in VIEWER_SELECTIONS: + raise ValueError(f"unsupported artifact viewer: {viewer}") + if int(payload.get("size_bytes", -1)) < 0: + raise ValueError("size_bytes must be non-negative") + for field in ("artifact_kind", "artifact_type", "generated_at", "evidence_boundary"): + if not str(payload.get(field, "")).strip(): + raise ValueError(f"{field} is required") + if not isinstance(payload.get("preview"), Mapping) or not isinstance(payload.get("bounds"), Mapping): + raise ValueError("preview and bounds must be objects") + warnings = payload.get("warnings") + if not isinstance(warnings, list) or not all(isinstance(item, str) for item in warnings): + raise ValueError("warnings must be a string list") + material = dict(payload) + fingerprint = _sha256(material.pop("fingerprint", ""), "fingerprint") + if _fingerprint(material) != fingerprint: + raise ValueError("artifact runtime descriptor fingerprint mismatch") + + +def build_selection( + descriptor: Mapping[str, Any], + *, + selector_type: str, + selector: Mapping[str, Any], + selected_by: str, + reason: str, + label: str = "", + created_at: str | None = None, +) -> dict[str, Any]: + validate_runtime_descriptor(descriptor) + viewer = str(descriptor["viewer"]) + if selector_type not in VIEWER_SELECTIONS[viewer]: + raise ValueError(f"selector {selector_type} is not valid for viewer {viewer}") + if not isinstance(selector, Mapping) or not selector: + raise ValueError("selector must be a non-empty object") + _json_value(dict(selector), "selector") + if len(reason) > MAX_REASON_CHARACTERS: + raise ValueError("selection reason is too long") + if len(_canonical_bytes(selector)) > MAX_SELECTION_BYTES: + raise ValueError("selection payload exceeds the runtime limit") + for value, field in ((selected_by, "selected_by"), (reason, "reason")): + if not value.strip(): + raise ValueError(f"{field} is required") + material = { + "schema_version": 1, + "artifact_path": descriptor["artifact_path"], + "artifact_sha256": descriptor["artifact_sha256"], + "viewer": viewer, + "selector_type": selector_type, + "selector": dict(selector), + "selected_by": selected_by, + "reason": reason, + "label": label, + "created_at": created_at or _now(), + "status": "active", + "evidence_boundary": "A selection identifies part of an artifact; it does not validate the selected interpretation.", + } + fingerprint = _fingerprint(material) + return {**material, "selection_id": f"selection-{fingerprint[:20]}", "fingerprint": fingerprint} + + +def validate_selection(payload: Mapping[str, Any], artifact_hashes: Mapping[str, str] | None = None) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported artifact selection schema") + artifact_path = _safe_relative(str(payload.get("artifact_path", "")), "artifact_path") + artifact_sha = _sha256(payload.get("artifact_sha256"), "artifact_sha256") + viewer = str(payload.get("viewer", "")) + selector_type = str(payload.get("selector_type", "")) + if viewer not in VIEWER_SELECTIONS or selector_type not in VIEWER_SELECTIONS[viewer]: + raise ValueError("artifact selection viewer or selector_type is invalid") + if not isinstance(payload.get("selector"), Mapping) or not payload["selector"]: + raise ValueError("artifact selection selector must be non-empty") + for field in ("selection_id", "selected_by", "reason", "created_at", "status", "evidence_boundary"): + if not str(payload.get(field, "")).strip(): + raise ValueError(f"selection {field} is required") + material = dict(payload) + fingerprint = _sha256(material.pop("fingerprint", ""), "selection fingerprint") + material.pop("selection_id", None) + if _fingerprint(material) != fingerprint: + raise ValueError("artifact selection fingerprint mismatch") + if str(payload["selection_id"]) != f"selection-{fingerprint[:20]}": + raise ValueError("artifact selection ID does not match fingerprint") + if artifact_hashes is not None and artifact_hashes.get(artifact_path) != artifact_sha: + raise ValueError("stale artifact selection anchor") + + +def stale_selection(payload: Mapping[str, Any], artifact_hashes: Mapping[str, str]) -> dict[str, Any]: + validate_selection(payload) + result = dict(payload) + if artifact_hashes.get(str(payload["artifact_path"])) != str(payload["artifact_sha256"]).lower(): + result["status"] = "stale-anchor" + return result + + +def build_transform_proposal( + selection: Mapping[str, Any], + *, + operation: str, + parameters: Mapping[str, Any], + reason: str, + affected_steps: Iterable[str], + expected_outputs: Iterable[str], + proposed_by: str, + requires_approval: bool = True, + approval_boundary: str = "artifact mutation and rerun", + created_at: str | None = None, +) -> dict[str, Any]: + validate_selection(selection) + if not operation.strip() or not reason.strip() or not proposed_by.strip(): + raise ValueError("operation, reason, and proposed_by are required") + if not isinstance(parameters, Mapping): + raise ValueError("parameters must be an object") + _json_value(dict(parameters), "parameters") + if len(reason) > MAX_REASON_CHARACTERS: + raise ValueError("proposal reason is too long") + if len(_canonical_bytes(parameters)) > MAX_PROPOSAL_PARAMETER_BYTES: + raise ValueError("proposal parameters exceed the runtime limit") + steps = sorted({_safe_relative(str(item), "affected step") for item in affected_steps}) + outputs = sorted({_safe_relative(str(item), "expected output") for item in expected_outputs}) + if not steps: + raise ValueError("affected_steps must be non-empty") + material = { + "schema_version": 1, + "selection_id": selection["selection_id"], + "selection_fingerprint": selection["fingerprint"], + "artifact_path": selection["artifact_path"], + "artifact_sha256": selection["artifact_sha256"], + "operation": operation, + "parameters": dict(parameters), + "reason": reason, + "affected_steps": steps, + "expected_outputs": outputs, + "proposed_by": proposed_by, + "requires_approval": bool(requires_approval), + "approval_boundary": approval_boundary, + "created_at": created_at or _now(), + "status": "proposed", + "executed": False, + "evidence_boundary": "This proposal describes a possible transform. It does not mutate bytes, execute a workflow, or establish a result.", + } + fingerprint = _fingerprint(material) + return {**material, "proposal_id": f"transform-{fingerprint[:20]}", "fingerprint": fingerprint} + + +def validate_transform_proposal(payload: Mapping[str, Any], selection: Mapping[str, Any] | None = None) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported transform proposal schema") + for field in ("proposal_id", "selection_id", "selection_fingerprint", "artifact_path", "artifact_sha256", "operation", "reason", "proposed_by", "approval_boundary", "created_at", "status", "evidence_boundary"): + if not str(payload.get(field, "")).strip(): + raise ValueError(f"transform proposal {field} is required") + _safe_relative(str(payload["artifact_path"]), "artifact_path") + _sha256(payload["artifact_sha256"], "artifact_sha256") + _sha256(payload["selection_fingerprint"], "selection_fingerprint") + if payload.get("executed") is not False or payload.get("status") != "proposed": + raise ValueError("a transform proposal must remain non-executing and proposed") + for field in ("affected_steps", "expected_outputs"): + if not isinstance(payload.get(field), list) or (field == "affected_steps" and not payload[field]): + raise ValueError(f"{field} must be a {'non-empty ' if field == 'affected_steps' else ''}list") + if not isinstance(payload.get("parameters"), Mapping): + raise ValueError("transform proposal parameters must be an object") + material = dict(payload) + fingerprint = _sha256(material.pop("fingerprint", ""), "transform fingerprint") + material.pop("proposal_id", None) + if _fingerprint(material) != fingerprint or str(payload["proposal_id"]) != f"transform-{fingerprint[:20]}": + raise ValueError("transform proposal fingerprint or ID mismatch") + if selection is not None: + validate_selection(selection) + if payload["selection_id"] != selection["selection_id"] or payload["selection_fingerprint"] != selection["fingerprint"]: + raise ValueError("transform proposal references a different selection") + + +def render_runtime_html(descriptor: Mapping[str, Any], *, title: str | None = None) -> str: + validate_runtime_descriptor(descriptor) + viewer = html.escape(str(descriptor["viewer"])) + artifact = html.escape(str(descriptor["artifact_path"])) + heading = html.escape(title or f"Scientific artifact: {descriptor['artifact_path']}") + preview_json = html.escape(json.dumps(descriptor["preview"], indent=2, sort_keys=True, ensure_ascii=False)) + warnings = "".join(f"
  • {html.escape(str(item))}
    • " for item in descriptor.get("warnings", [])) or "
  • None
    • " + metadata = { + "SHA-256": descriptor["artifact_sha256"], + "Kind": descriptor["artifact_kind"], + "Type": descriptor["artifact_type"], + "Media type": descriptor.get("media_type"), + "Size": descriptor["size_bytes"], + "Viewer": descriptor["viewer"], + "Fingerprint": descriptor["fingerprint"], + } + cards = "".join(f"
    {html.escape(str(key))}
    {html.escape(str(value))}
    " for key, value in metadata.items()) + return f""" + +{heading} + +

    {heading}

    Artifact: {artifact} · Viewer: {viewer}

    +
    {cards}

    Warnings

      {warnings}

    Bounded preview

    {preview_json}
    +

    {html.escape(str(descriptor['evidence_boundary']))}

    \n""" diff --git a/src/codex_science/artifact_store.py b/src/codex_science/artifact_store.py new file mode 100644 index 00000000..a6b23ea0 --- /dev/null +++ b/src/codex_science/artifact_store.py @@ -0,0 +1,305 @@ +"""Streaming and Merkle-hashed descriptors for large scientific artifacts.""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterator, Mapping + +DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024 +ARTIFACT_TYPES = {"file", "chunked-file", "directory-tree", "external-reference"} + + +def _hex_sha256(value: Any, label: str) -> str: + text = str(value).lower().strip() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{label} must be a SHA-256 digest") + return text + + +def stream_sha256(path: Path, *, block_size: int = 1024 * 1024) -> tuple[str, int]: + if block_size <= 0: + raise ValueError("block_size must be positive") + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + while chunk := handle.read(block_size): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +def iter_chunk_hashes(path: Path, *, chunk_size: int = DEFAULT_CHUNK_SIZE) -> Iterator[dict[str, Any]]: + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + offset = 0 + index = 0 + with path.open("rb") as handle: + while chunk := handle.read(chunk_size): + yield { + "index": index, + "offset": offset, + "size": len(chunk), + "sha256": hashlib.sha256(chunk).hexdigest(), + } + index += 1 + offset += len(chunk) + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _validate_relative(value: str) -> str: + pure = PurePosixPath(value) + if not value or pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"artifact entry must be a safe relative path: {value}") + return pure.as_posix() + + +@dataclass(frozen=True) +class ArtifactDescriptor: + schema_version: int + artifact_type: str + root_sha256: str + total_bytes: int + entry_count: int + media_type: str | None + entries: tuple[dict[str, Any], ...] + external: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["entries"] = [dict(item) for item in self.entries] + return payload + + +def describe_file( + path: Path, + *, + media_type: str | None = None, + chunk_size: int | None = None, +) -> ArtifactDescriptor: + if not path.is_file(): + raise ValueError(f"not a file: {path}") + digest, size = stream_sha256(path) + if chunk_size is None: + entries = ({"path": path.name, "size": size, "sha256": digest},) + artifact_type = "file" + else: + chunks = tuple(iter_chunk_hashes(path, chunk_size=chunk_size)) + entries = ( + { + "path": path.name, + "size": size, + "sha256": digest, + "chunk_size": chunk_size, + "chunks": list(chunks), + }, + ) + artifact_type = "chunked-file" + return ArtifactDescriptor(1, artifact_type, digest, size, 1, media_type, entries) + + +def _directory_entries(root: Path, *, follow_symlinks: bool) -> tuple[dict[str, Any], ...]: + entries: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + if not follow_symlinks: + raise ValueError(f"symbolic links are forbidden in directory artifacts: {relative}") + target = path.resolve() + if not target.is_relative_to(root.resolve()): + raise ValueError(f"symbolic link escapes directory artifact: {relative}") + if path.is_dir(): + continue + if not path.is_file(): + raise ValueError(f"unsupported directory artifact entry: {relative}") + digest, size = stream_sha256(path) + entries.append({"path": _validate_relative(relative), "size": size, "sha256": digest}) + return tuple(entries) + + +def describe_directory( + root: Path, + *, + media_type: str | None = None, + follow_symlinks: bool = False, +) -> ArtifactDescriptor: + if not root.is_dir(): + raise ValueError(f"not a directory: {root}") + entries = _directory_entries(root, follow_symlinks=follow_symlinks) + material = {"algorithm": "sha256-merkle-v1", "entries": list(entries)} + root_digest = hashlib.sha256(_canonical_bytes(material)).hexdigest() + return ArtifactDescriptor( + 1, + "directory-tree", + root_digest, + sum(int(item["size"]) for item in entries), + len(entries), + media_type, + entries, + ) + + +def describe_external_reference( + *, + uri: str, + version: str, + sha256: str, + size_bytes: int | None = None, + media_type: str | None = None, + license: str | None = None, +) -> ArtifactDescriptor: + if not uri.strip() or not version.strip(): + raise ValueError("external reference requires uri and version") + digest = _hex_sha256(sha256, "sha256") + external = { + "uri": uri, + "version": version, + "sha256": digest, + "size_bytes": size_bytes, + "license": license, + } + return ArtifactDescriptor( + 1, + "external-reference", + digest, + int(size_bytes or 0), + 0, + media_type, + (), + external, + ) + + +def validate_descriptor(payload: Mapping[str, Any], path: Path | None = None) -> ArtifactDescriptor: + if payload.get("schema_version") != 1: + raise ValueError("unsupported artifact descriptor schema") + artifact_type = str(payload.get("artifact_type", "")) + if artifact_type not in ARTIFACT_TYPES: + raise ValueError(f"unsupported artifact type: {artifact_type}") + root_sha256 = _hex_sha256(payload.get("root_sha256"), "root_sha256") + total_bytes = int(payload.get("total_bytes", 0)) + entry_count = int(payload.get("entry_count", 0)) + entries_raw = payload.get("entries", []) + if not isinstance(entries_raw, (list, tuple)): + raise ValueError("entries must be a list") + entries = tuple(dict(item) for item in entries_raw if isinstance(item, Mapping)) + if len(entries) != len(entries_raw): + raise ValueError("every descriptor entry must be an object") + for item in entries: + _validate_relative(str(item.get("path", ""))) + _hex_sha256(item.get("sha256"), "entry sha256") + if int(item.get("size", -1)) < 0: + raise ValueError("entry size must be non-negative") + chunks = item.get("chunks") + if chunks is not None: + if not isinstance(chunks, list): + raise ValueError("chunks must be a list") + expected_offset = 0 + for index, chunk in enumerate(chunks): + if not isinstance(chunk, Mapping): + raise ValueError("chunk record must be an object") + if int(chunk.get("index", -1)) != index: + raise ValueError("chunk indexes must be contiguous") + if int(chunk.get("offset", -1)) != expected_offset: + raise ValueError("chunk offsets must be contiguous") + size = int(chunk.get("size", -1)) + if size <= 0: + raise ValueError("chunk size must be positive") + _hex_sha256(chunk.get("sha256"), "chunk sha256") + expected_offset += size + if expected_offset != int(item["size"]): + raise ValueError("chunk sizes do not sum to entry size") + descriptor = ArtifactDescriptor( + 1, + artifact_type, + root_sha256, + total_bytes, + entry_count, + None if payload.get("media_type") is None else str(payload.get("media_type")), + entries, + None if payload.get("external") is None else dict(payload.get("external")), + ) + if artifact_type == "external-reference": + external = descriptor.external + if not isinstance(external, dict): + raise ValueError("external-reference requires external metadata") + if _hex_sha256(external.get("sha256"), "external sha256") != root_sha256: + raise ValueError("external reference digest does not match root_sha256") + if not str(external.get("uri", "")).strip() or not str(external.get("version", "")).strip(): + raise ValueError("external reference requires uri and version") + if entries or entry_count != 0: + raise ValueError("external reference cannot contain local entries") + external_size = external.get("size_bytes") + if external_size is not None and int(external_size) != total_bytes: + raise ValueError("external reference size does not match total_bytes") + else: + if len(entries) != entry_count: + raise ValueError("entry_count does not match entries") + if sum(int(item.get("size", 0)) for item in entries) != total_bytes: + raise ValueError("total_bytes does not match entries") + if path is not None: + if artifact_type == "directory-tree": + actual = describe_directory(path, media_type=descriptor.media_type) + elif artifact_type in {"file", "chunked-file"}: + actual = describe_file( + path, + media_type=descriptor.media_type, + chunk_size=int(entries[0].get("chunk_size")) if artifact_type == "chunked-file" else None, + ) + else: + actual = descriptor + if actual.root_sha256 != descriptor.root_sha256 or actual.entries != descriptor.entries: + raise ValueError("artifact bytes do not match descriptor") + return descriptor + + +def write_descriptor(path: Path, descriptor: ArtifactDescriptor) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(descriptor.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +class ContentAddressedStore: + def __init__(self, root: Path) -> None: + self.root = root + self.objects = root / "objects" / "sha256" + + def object_path(self, digest: str) -> Path: + digest = _hex_sha256(digest, "digest") + return self.objects / digest[:2] / digest[2:] + + def add_file(self, source: Path) -> tuple[str, Path]: + digest, _size = stream_sha256(source) + destination = self.object_path(digest) + if destination.is_file(): + existing, _ = stream_sha256(destination) + if existing != digest: + raise ValueError("content-addressed store corruption") + return digest, destination + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=".incoming-", dir=destination.parent) + os.close(descriptor) + temporary = Path(temporary_name) + try: + shutil.copyfile(source, temporary) + copied, _ = stream_sha256(temporary) + if copied != digest: + raise ValueError("copied object digest mismatch") + temporary.chmod(0o444) + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + return digest, destination + + def verify(self, digest: str) -> bool: + path = self.object_path(digest) + if not path.is_file(): + return False + actual, _ = stream_sha256(path) + return actual == digest diff --git a/src/codex_science/artifacts.py b/src/codex_science/artifacts.py index a0afc038..7281947c 100644 --- a/src/codex_science/artifacts.py +++ b/src/codex_science/artifacts.py @@ -1,18 +1,35 @@ """Structured, reproducible analysis artifact manifests and bundle validation.""" from __future__ import annotations -import hashlib import json from pathlib import Path, PurePosixPath from typing import Any -REQUIRED_FIELDS = {"schema_version", "run_id", "question", "plan", "inputs", "code", "executions", "environment", "artifacts", "claims", "review"} +from codex_science.artifact_store import describe_directory, stream_sha256, validate_descriptor + +REQUIRED_FIELDS = { + "schema_version", "run_id", "question", "plan", "inputs", "code", + "executions", "environment", "artifacts", "claims", "review", +} +LOCAL_ARTIFACT_TYPES = {"file", "chunked-file", "directory-tree"} def new_manifest(run_id: str, question: str, plan: list[dict[str, Any]]) -> dict[str, Any]: if not run_id.strip() or not question.strip(): raise ValueError("run_id and question are required") - return {"schema_version": 1, "run_id": run_id, "question": question, "plan": plan, "inputs": [], "code": [], "executions": [], "environment": {}, "artifacts": [], "claims": [], "review": {"status": "pending", "findings": []}} + return { + "schema_version": 1, + "run_id": run_id, + "question": question, + "plan": plan, + "inputs": [], + "code": [], + "executions": [], + "environment": {}, + "artifacts": [], + "claims": [], + "review": {"status": "pending", "findings": []}, + } def _validate_relative_path(value: str) -> None: @@ -25,13 +42,46 @@ def _valid_sha256(value: str) -> bool: return len(value) == 64 and all(character in "0123456789abcdefABCDEF" for character in value) -def add_artifact(manifest: dict[str, Any], path: str, *, kind: str, sha256: str) -> None: +def add_artifact( + manifest: dict[str, Any], + path: str, + *, + kind: str, + sha256: str, + artifact_type: str = "file", + size_bytes: int | None = None, + entry_count: int | None = None, + media_type: str | None = None, + descriptor_path: str | None = None, +) -> None: _validate_relative_path(path) if not _valid_sha256(sha256): raise ValueError("sha256 must be a 64-character hexadecimal digest") if not kind.strip(): raise ValueError("artifact kind is required") - manifest["artifacts"].append({"path": path, "kind": kind, "sha256": sha256.lower()}) + if artifact_type not in LOCAL_ARTIFACT_TYPES: + raise ValueError(f"unsupported local artifact_type: {artifact_type}") + record: dict[str, Any] = { + "path": path, + "kind": kind, + "sha256": sha256.lower(), + } + if artifact_type != "file": + record["artifact_type"] = artifact_type + if size_bytes is not None: + if size_bytes < 0: + raise ValueError("size_bytes must be non-negative") + record["size_bytes"] = size_bytes + if entry_count is not None: + if entry_count < 0: + raise ValueError("entry_count must be non-negative") + record["entry_count"] = entry_count + if media_type is not None: + record["media_type"] = media_type + if descriptor_path is not None: + _validate_relative_path(descriptor_path) + record["descriptor_path"] = descriptor_path + manifest["artifacts"].append(record) def validate_manifest(manifest: dict[str, Any]) -> None: @@ -64,6 +114,14 @@ def validate_manifest(manifest: dict[str, Any]) -> None: raise ValueError(f"Artifact kind is required: {path}") if not _valid_sha256(str(artifact.get("sha256", ""))): raise ValueError("Invalid artifact sha256") + artifact_type = str(artifact.get("artifact_type", "file")) + if artifact_type not in LOCAL_ARTIFACT_TYPES: + raise ValueError(f"Unsupported local artifact_type: {artifact_type}") + for field in ("size_bytes", "entry_count"): + if field in artifact and (isinstance(artifact[field], bool) or int(artifact[field]) < 0): + raise ValueError(f"Artifact {field} must be a non-negative integer: {path}") + if "descriptor_path" in artifact: + _validate_relative_path(str(artifact["descriptor_path"])) claim_ids: set[str] = set() for claim in manifest.get("claims", []): if not isinstance(claim, dict): @@ -76,6 +134,21 @@ def validate_manifest(manifest: dict[str, Any]) -> None: claim_ids.add(claim_id) +def _validate_descriptor_path(record: dict[str, Any], run_dir: Path, target: Path) -> None: + descriptor_relative = record.get("descriptor_path") + if descriptor_relative is None: + return + descriptor_path = run_dir / str(descriptor_relative) + if not descriptor_path.is_file(): + raise ValueError(f"Artifact descriptor is missing: {descriptor_relative}") + payload = json.loads(descriptor_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"Artifact descriptor must be an object: {descriptor_relative}") + descriptor = validate_descriptor(payload, target) + if descriptor.root_sha256 != str(record["sha256"]).lower(): + raise ValueError(f"Artifact descriptor root mismatch: {record['path']}") + + def verify_bundle_artifacts(manifest: dict[str, Any], run_dir: Path) -> dict[str, Path]: validate_manifest(manifest) resolved_run = run_dir.resolve() @@ -83,14 +156,30 @@ def verify_bundle_artifacts(manifest: dict[str, Any], run_dir: Path) -> dict[str for record in manifest["artifacts"]: relative = str(record["path"]) path = run_dir / relative - if not path.is_file(): + artifact_type = str(record.get("artifact_type", "file")) + if artifact_type == "directory-tree": + if not path.is_dir(): + raise ValueError(f"Artifact directory is missing: {relative}") + elif not path.is_file(): raise ValueError(f"Artifact file is missing: {relative}") resolved = path.resolve() if not resolved.is_relative_to(resolved_run): raise ValueError(f"Artifact resolves outside the run bundle: {relative}") - digest = hashlib.sha256(path.read_bytes()).hexdigest() + if artifact_type == "directory-tree": + descriptor = describe_directory(path, media_type=record.get("media_type")) + digest = descriptor.root_sha256 + size_bytes = descriptor.total_bytes + entry_count = descriptor.entry_count + else: + digest, size_bytes = stream_sha256(path) + entry_count = 1 if digest != str(record["sha256"]).lower(): raise ValueError(f"Artifact digest mismatch: {relative}") + if "size_bytes" in record and int(record["size_bytes"]) != size_bytes: + raise ValueError(f"Artifact size mismatch: {relative}") + if "entry_count" in record and int(record["entry_count"]) != entry_count: + raise ValueError(f"Artifact entry count mismatch: {relative}") + _validate_descriptor_path(record, run_dir, path) verified[relative] = path return verified diff --git a/src/codex_science/compute_backends.py b/src/codex_science/compute_backends.py new file mode 100644 index 00000000..c4a9f322 --- /dev/null +++ b/src/codex_science/compute_backends.py @@ -0,0 +1,843 @@ +"""Durable local and Slurm scientific job execution contracts. + +The local backend is fully executable. The Slurm backend submits only after an +explicit, hash-bound approval receipt and uses an existing scheduler +configuration. Neither backend invents credentials, installs software, or +interprets scientific success from a zero process exit code. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Mapping, Protocol + +from codex_science.artifact_store import describe_directory, stream_sha256 + + +TERMINAL_STATES = {"completed", "failed", "timed-out", "cancelled", "lost"} +ALL_STATES = {"submitted", "running", *TERMINAL_STATES} +SECRET_KEY_FRAGMENTS = {"token", "secret", "password", "credential", "private_key", "api_key", "apikey"} +SLURM_STATE_MAP = { + "COMPLETED": "completed", + "CANCELLED": "cancelled", + "TIMEOUT": "timed-out", + "FAILED": "failed", + "NODE_FAIL": "failed", + "OUT_OF_MEMORY": "failed", + "PREEMPTED": "failed", + "BOOT_FAIL": "failed", + "DEADLINE": "timed-out", + "RUNNING": "running", + "PENDING": "submitted", + "CONFIGURING": "submitted", + "COMPLETING": "running", +} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _atomic_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=".state-", suffix=".json", dir=path.parent) + os.close(descriptor) + temporary = Path(temporary_name) + try: + temporary.write_text(json.dumps(dict(payload), indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def _read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON object required: {path}") + return payload + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _safe_relative(value: Any, label: str) -> str: + text = _text(value, label) + pure = PurePosixPath(text) + if pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"{label} must be a safe relative path") + return pure.as_posix() + + +def _reject_secret_environment(environment: Mapping[str, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for key, value in environment.items(): + key_text = _text(key, "environment key") + normalized = key_text.lower() + if any(fragment in normalized for fragment in SECRET_KEY_FRAGMENTS): + raise ValueError(f"secret-like environment keys cannot be recorded in a job spec: {key_text}") + result[key_text] = str(value) + return result + + +def _sha(value: Any, label: str) -> str: + digest = _text(value, label).lower() + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError(f"{label} must be a SHA-256 digest") + return digest + + +def _normalize_inputs(value: Any) -> tuple[dict[str, Any], ...]: + if value is None: + return () + if not isinstance(value, list): + raise ValueError("inputs must be a list") + results: list[dict[str, Any]] = [] + ids: set[str] = set() + for index, raw in enumerate(value): + if not isinstance(raw, Mapping): + raise ValueError(f"input {index} must be an object") + item = dict(raw) + input_id = _text(item.get("id", f"input-{index}"), f"input {index} id") + if input_id in ids: + raise ValueError(f"duplicate input ID: {input_id}") + ids.add(input_id) + item["id"] = input_id + if item.get("path") is not None: + item["path"] = _safe_relative(item["path"], f"input {input_id} path") + elif not str(item.get("identifier", "")).strip(): + raise ValueError(f"input {input_id} needs path or identifier") + if item.get("sha256") is not None: + item["sha256"] = _sha(item["sha256"], f"input {input_id} sha256") + if item.get("root_sha256") is not None: + item["root_sha256"] = _sha(item["root_sha256"], f"input {input_id} root_sha256") + results.append(item) + return tuple(results) + + +def _input_preflight(spec: "JobSpec") -> list[dict[str, Any]]: + checks: list[dict[str, Any]] = [] + working = Path(spec.working_directory).resolve() + for item in spec.inputs: + input_id = str(item["id"]) + if item.get("path") is None: + checks.append( + {"name": f"input:{input_id}", "ready": True, "detail": "external identifier recorded"} + ) + continue + target = (working / str(item["path"])).resolve() + if not target.is_relative_to(working): + checks.append( + {"name": f"input:{input_id}", "ready": False, "detail": "path escapes working directory"} + ) + continue + if not target.exists(): + checks.append({"name": f"input:{input_id}", "ready": False, "detail": "missing"}) + continue + expected = item.get("sha256") or item.get("root_sha256") + if expected is None: + checks.append( + {"name": f"input:{input_id}", "ready": True, "detail": "exists; no digest declared"} + ) + continue + if target.is_file(): + actual, _size = stream_sha256(target) + elif target.is_dir(): + actual = describe_directory(target).root_sha256 + else: + checks.append( + {"name": f"input:{input_id}", "ready": False, "detail": "unsupported filesystem object"} + ) + continue + checks.append( + { + "name": f"input:{input_id}", + "ready": actual == expected, + "detail": "digest matches" if actual == expected else f"digest mismatch: {actual}", + } + ) + return checks + + +@dataclass(frozen=True) +class ResourceRequest: + cpus: int = 1 + memory_mb: int = 1024 + gpus: int = 0 + gpu_type: str | None = None + wall_time_seconds: int = 3600 + partition: str | None = None + account: str | None = None + qos: str | None = None + + def validate(self) -> None: + for field in ("cpus", "memory_mb", "wall_time_seconds"): + value = int(getattr(self, field)) + if value < 1: + raise ValueError(f"resource {field} must be positive") + if self.gpus < 0: + raise ValueError("resource gpus must be non-negative") + + @classmethod + def from_payload(cls, payload: Mapping[str, Any] | None) -> "ResourceRequest": + source = dict(payload or {}) + request = cls( + cpus=int(source.get("cpus", 1)), + memory_mb=int(source.get("memory_mb", 1024)), + gpus=int(source.get("gpus", 0)), + gpu_type=None if source.get("gpu_type") is None else str(source["gpu_type"]), + wall_time_seconds=int(source.get("wall_time_seconds", 3600)), + partition=None if source.get("partition") is None else str(source["partition"]), + account=None if source.get("account") is None else str(source["account"]), + qos=None if source.get("qos") is None else str(source["qos"]), + ) + request.validate() + return request + + +@dataclass(frozen=True) +class JobSpec: + schema_version: int + backend: str + name: str + command: tuple[str, ...] + working_directory: str + environment: dict[str, str] + inherit_environment: bool + inputs: tuple[dict[str, Any], ...] + outputs: tuple[str, ...] + resources: ResourceRequest + timeout_seconds: int + cost_cap: float | None + approval_required: bool + scientific_run_id: str | None + checkpoint_paths: tuple[str, ...] + + @classmethod + def from_payload(cls, payload: Mapping[str, Any]) -> "JobSpec": + if payload.get("schema_version", 1) != 1: + raise ValueError("unsupported job spec schema") + backend = _text(payload.get("backend"), "backend") + if backend not in {"local", "slurm"}: + raise ValueError(f"unsupported job backend: {backend}") + command_raw = payload.get("command") + if not isinstance(command_raw, list) or not command_raw: + raise ValueError("command must be a non-empty argv list") + command = tuple(_text(item, "command item") for item in command_raw) + environment_raw = payload.get("environment", {}) + if not isinstance(environment_raw, Mapping): + raise ValueError("environment must be an object") + inputs_raw = payload.get("inputs", []) + outputs_raw = payload.get("outputs", []) + if not isinstance(outputs_raw, list): + raise ValueError("outputs must be a list") + checkpoints_raw = payload.get("checkpoint_paths", []) + if not isinstance(checkpoints_raw, list): + raise ValueError("checkpoint_paths must be a list") + timeout = int(payload.get("timeout_seconds", 3600)) + if timeout < 1: + raise ValueError("timeout_seconds must be positive") + cost_cap = None if payload.get("cost_cap") is None else float(payload["cost_cap"]) + if cost_cap is not None and cost_cap < 0: + raise ValueError("cost_cap must be non-negative") + spec = cls( + schema_version=1, + backend=backend, + name=_text(payload.get("name"), "name"), + command=command, + working_directory=str(Path(_text(payload.get("working_directory"), "working_directory")).resolve()), + environment=_reject_secret_environment(environment_raw), + inherit_environment=bool(payload.get("inherit_environment", True)), + inputs=_normalize_inputs(inputs_raw), + outputs=tuple(_safe_relative(item, "output") for item in outputs_raw), + resources=ResourceRequest.from_payload(payload.get("resources") if isinstance(payload.get("resources"), Mapping) else None), + timeout_seconds=timeout, + cost_cap=cost_cap, + approval_required=bool(payload.get("approval_required", backend != "local")), + scientific_run_id=None if payload.get("scientific_run_id") is None else str(payload["scientific_run_id"]), + checkpoint_paths=tuple(_safe_relative(item, "checkpoint path") for item in checkpoints_raw), + ) + spec.validate() + return spec + + def validate(self) -> None: + if self.backend not in {"local", "slurm"}: + raise ValueError("unsupported backend") + if not self.command: + raise ValueError("job command cannot be empty") + self.resources.validate() + working = Path(self.working_directory) + if not working.is_dir(): + raise ValueError(f"working_directory is not a directory: {working}") + for item in self.inputs: + if not str(item.get("id", item.get("path", ""))).strip(): + raise ValueError("each input needs id or path") + if item.get("sha256") is not None: + digest = str(item["sha256"]).lower() + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError("input sha256 is invalid") + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["command"] = list(self.command) + payload["inputs"] = [dict(item) for item in self.inputs] + payload["outputs"] = list(self.outputs) + payload["checkpoint_paths"] = list(self.checkpoint_paths) + return payload + + @property + def fingerprint(self) -> str: + return _fingerprint(self.to_dict()) + + +class JobBackend(Protocol): + def preflight(self, spec: JobSpec) -> dict[str, Any]: ... + def submit(self, spec: JobSpec, *, approval: Mapping[str, Any] | None = None) -> dict[str, Any]: ... + def status(self, job_id: str) -> dict[str, Any]: ... + def cancel(self, job_id: str) -> dict[str, Any]: ... + def collect(self, job_id: str) -> dict[str, Any]: ... + + +def build_approval_receipt(spec: JobSpec, *, approved_by: str, target: str, approved_at: str | None = None) -> dict[str, Any]: + material = { + "schema_version": 1, + "job_spec_sha256": spec.fingerprint, + "backend": spec.backend, + "target": _text(target, "target"), + "approved_by": _text(approved_by, "approved_by"), + "approved_at": approved_at or _now(), + "approved": True, + "resource_cap": asdict(spec.resources), + "cost_cap": spec.cost_cap, + "evidence_boundary": "Approval authorizes the described execution boundary; it does not endorse the scientific method or result.", + } + return {**material, "fingerprint": _fingerprint(material)} + + +def validate_approval_receipt(receipt: Mapping[str, Any], spec: JobSpec) -> None: + if receipt.get("schema_version") != 1 or receipt.get("approved") is not True: + raise ValueError("valid explicit approval receipt is required") + if receipt.get("job_spec_sha256") != spec.fingerprint or receipt.get("backend") != spec.backend: + raise ValueError("approval receipt covers a different job spec") + material = dict(receipt) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint: + raise ValueError("approval receipt fingerprint mismatch") + + +def _state_material(spec: JobSpec, *, job_id: str, state: str, submitted_at: str, **updates: Any) -> dict[str, Any]: + if state not in ALL_STATES: + raise ValueError(f"invalid job state: {state}") + payload = { + "schema_version": 1, + "job_id": job_id, + "backend": spec.backend, + "name": spec.name, + "scientific_run_id": spec.scientific_run_id, + "job_spec_sha256": spec.fingerprint, + "state": state, + "submitted_at": submitted_at, + "started_at": None, + "finished_at": None, + "exit_code": None, + "failure_class": None, + "worker_pid": None, + "scheduler_job_id": None, + "stdout_path": "stdout.log", + "stderr_path": "stderr.log", + "resource_request": asdict(spec.resources), + "resource_usage": {}, + "cost": None, + "checkpoint_paths": list(spec.checkpoint_paths), + "outputs": [], + "message": "", + "updated_at": _now(), + } + payload.update(updates) + material = dict(payload) + payload["fingerprint"] = _fingerprint(material) + return payload + + +def validate_job_state(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1 or payload.get("state") not in ALL_STATES: + raise ValueError("invalid job state record") + for field in ("job_id", "backend", "name", "job_spec_sha256", "submitted_at", "updated_at"): + _text(payload.get(field), field) + material = dict(payload) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint: + raise ValueError("job state fingerprint mismatch") + + +def _process_alive(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +class LocalBackend: + def __init__(self, state_dir: Path) -> None: + self.state_dir = state_dir.resolve() + self.jobs_dir = self.state_dir / "jobs" + + def _job_dir(self, job_id: str) -> Path: + safe = _text(job_id, "job_id") + if any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for character in safe): + raise ValueError("job_id contains unsafe characters") + return self.jobs_dir / safe + + def preflight(self, spec: JobSpec) -> dict[str, Any]: + spec.validate() + executable = shutil.which(spec.command[0]) if not Path(spec.command[0]).is_absolute() else spec.command[0] + checks = [ + { + "name": "working-directory", + "ready": Path(spec.working_directory).is_dir(), + "detail": spec.working_directory, + }, + {"name": "executable", "ready": executable is not None, "detail": executable or "not found"}, + { + "name": "resource-request-recorded", + "ready": True, + "detail": asdict(spec.resources), + }, + ] + checks.extend(_input_preflight(spec)) + return { + "schema_version": 1, + "backend": "local", + "ready": all(check["ready"] for check in checks), + "checks": checks, + "executable": executable, + "working_directory": spec.working_directory, + "job_spec_sha256": spec.fingerprint, + "resource_request": asdict(spec.resources), + "warnings": ["Local resource requests are recorded but are not a scheduler-enforced reservation."], + } + + def submit(self, spec: JobSpec, *, approval: Mapping[str, Any] | None = None) -> dict[str, Any]: + if spec.backend != "local": + raise ValueError("LocalBackend requires backend=local") + preflight = self.preflight(spec) + if not preflight["ready"]: + raise ValueError("local preflight failed") + if spec.approval_required: + if approval is None: + raise ValueError("job spec requires explicit approval") + validate_approval_receipt(approval, spec) + job_id = "local-" + uuid.uuid4().hex[:20] + job_dir = self._job_dir(job_id) + job_dir.mkdir(parents=True) + _atomic_json(job_dir / "spec.json", spec.to_dict()) + submitted_at = _now() + initial = _state_material(spec, job_id=job_id, state="submitted", submitted_at=submitted_at) + _atomic_json(job_dir / "state.json", initial) + worker_environment = os.environ.copy() + source_root = str(Path(__file__).resolve().parents[1]) + existing_pythonpath = worker_environment.get("PYTHONPATH", "") + worker_environment["PYTHONPATH"] = source_root + ( + os.pathsep + existing_pythonpath if existing_pythonpath else "" + ) + process = subprocess.Popen( + [sys.executable, "-m", "codex_science.compute_backends", "_worker", str(job_dir)], + env=worker_environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + close_fds=True, + ) + current = _read_json(job_dir / "state.json") + submitted_receipt = _state_material( + spec, + job_id=job_id, + state="submitted", + submitted_at=submitted_at, + worker_pid=process.pid, + message="local worker started", + ) + if current.get("state") == "submitted": + _atomic_json(job_dir / "state.json", submitted_receipt) + return submitted_receipt + + def status(self, job_id: str) -> dict[str, Any]: + job_dir = self._job_dir(job_id) + state = _read_json(job_dir / "state.json") + validate_job_state(state) + if state["state"] in {"submitted", "running"} and state.get("worker_pid") and not _process_alive(int(state["worker_pid"])): + # Give an atomically-written terminal state a short chance to land. + time.sleep(0.05) + state = _read_json(job_dir / "state.json") + validate_job_state(state) + if state["state"] in {"submitted", "running"}: + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + state = _state_material( + spec, + job_id=job_id, + state="lost", + submitted_at=str(state["submitted_at"]), + started_at=state.get("started_at"), + finished_at=_now(), + worker_pid=state.get("worker_pid"), + failure_class="worker-lost", + message="local worker exited without a terminal receipt", + ) + _atomic_json(job_dir / "state.json", state) + return state + + def wait(self, job_id: str, *, timeout_seconds: float, poll_seconds: float = 0.2) -> dict[str, Any]: + if timeout_seconds <= 0 or poll_seconds <= 0: + raise ValueError("wait timeout and poll interval must be positive") + deadline = time.monotonic() + timeout_seconds + while True: + state = self.status(job_id) + if state["state"] in TERMINAL_STATES: + return state + if time.monotonic() >= deadline: + raise TimeoutError(f"job did not reach a terminal state within {timeout_seconds} seconds") + time.sleep(poll_seconds) + + def cancel(self, job_id: str) -> dict[str, Any]: + job_dir = self._job_dir(job_id) + state = self.status(job_id) + if state["state"] in TERMINAL_STATES: + return state + pid = int(state.get("worker_pid") or 0) + if pid > 0 and _process_alive(pid): + try: + os.killpg(pid, signal.SIGTERM) + except ProcessLookupError: + pass + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + cancelled = _state_material( + spec, + job_id=job_id, + state="cancelled", + submitted_at=str(state["submitted_at"]), + started_at=state.get("started_at"), + finished_at=_now(), + worker_pid=pid or None, + failure_class="user-cancelled", + message="job cancelled", + ) + _atomic_json(job_dir / "state.json", cancelled) + return cancelled + + def collect(self, job_id: str) -> dict[str, Any]: + job_dir = self._job_dir(job_id) + state = self.status(job_id) + if state["state"] not in TERMINAL_STATES: + raise ValueError("cannot collect a nonterminal job") + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + root = Path(spec.working_directory).resolve() + outputs: list[dict[str, Any]] = [] + for relative in spec.outputs: + path = (root / relative).resolve() + if not path.is_relative_to(root): + raise ValueError(f"output escapes working directory: {relative}") + if path.is_dir(): + descriptor = describe_directory(path) + outputs.append({"path": relative, "artifact_type": "directory-tree", "sha256": descriptor.root_sha256, "size_bytes": descriptor.total_bytes, "entry_count": descriptor.entry_count}) + elif path.is_file(): + digest, size = stream_sha256(path) + outputs.append({"path": relative, "artifact_type": "file", "sha256": digest, "size_bytes": size, "entry_count": 1}) + else: + outputs.append({"path": relative, "artifact_type": "missing", "sha256": None, "size_bytes": None, "entry_count": 0}) + state = dict(state) + state.pop("fingerprint", None) + state["outputs"] = outputs + state["updated_at"] = _now() + state["fingerprint"] = _fingerprint(state) + _atomic_json(job_dir / "state.json", state) + return state + + +def _worker(job_dir: Path) -> int: + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + initial = _read_json(job_dir / "state.json") + job_id = str(initial["job_id"]) + submitted_at = str(initial["submitted_at"]) + worker_pid = os.getpid() + running = _state_material(spec, job_id=job_id, state="running", submitted_at=submitted_at, started_at=_now(), worker_pid=worker_pid, message="command running") + _atomic_json(job_dir / "state.json", running) + environment = os.environ.copy() if spec.inherit_environment else {} + environment.update(spec.environment) + stdout_path = job_dir / "stdout.log" + stderr_path = job_dir / "stderr.log" + started_monotonic = time.monotonic() + state = "failed" + exit_code: int | None = None + failure_class: str | None = None + message = "" + try: + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run( + list(spec.command), + cwd=spec.working_directory, + env=environment, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + timeout=spec.timeout_seconds, + check=False, + ) + exit_code = int(completed.returncode) + state = "completed" if exit_code == 0 else "failed" + failure_class = None if exit_code == 0 else "nonzero-exit" + message = "command exited successfully" if exit_code == 0 else f"command exited with code {exit_code}" + except subprocess.TimeoutExpired: + state = "timed-out" + failure_class = "wall-time" + message = f"command exceeded timeout_seconds={spec.timeout_seconds}" + except FileNotFoundError as error: + state = "failed" + failure_class = "executable-not-found" + message = str(error) + except BaseException as error: + state = "failed" + failure_class = "worker-exception" + message = f"{type(error).__name__}: {error}" + final = _state_material( + spec, + job_id=job_id, + state=state, + submitted_at=submitted_at, + started_at=running["started_at"], + finished_at=_now(), + exit_code=exit_code, + failure_class=failure_class, + worker_pid=worker_pid, + resource_usage={"wall_seconds": round(time.monotonic() - started_monotonic, 6)}, + message=message, + ) + _atomic_json(job_dir / "state.json", final) + return 0 if state == "completed" else 1 + + +class SlurmBackend: + def __init__(self, state_dir: Path) -> None: + self.state_dir = state_dir.resolve() + self.jobs_dir = self.state_dir / "jobs" + + def _job_dir(self, job_id: str) -> Path: + safe = _text(job_id, "job_id") + if any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for character in safe): + raise ValueError("job_id contains unsafe characters") + return self.jobs_dir / safe + + def preflight(self, spec: JobSpec) -> dict[str, Any]: + spec.validate() + tools = {name: shutil.which(name) for name in ("sbatch", "squeue", "sacct", "scancel")} + checks = [ + { + "name": "working-directory", + "ready": Path(spec.working_directory).is_dir(), + "detail": spec.working_directory, + }, + {"name": "sbatch", "ready": tools["sbatch"] is not None, "detail": tools["sbatch"] or "not found"}, + {"name": "scancel", "ready": tools["scancel"] is not None, "detail": tools["scancel"] or "not found"}, + { + "name": "status-command", + "ready": tools["sacct"] is not None or tools["squeue"] is not None, + "detail": tools["sacct"] or tools["squeue"] or "not found", + }, + ] + checks.extend(_input_preflight(spec)) + return { + "schema_version": 1, + "backend": "slurm", + "ready": all(check["ready"] for check in checks), + "checks": checks, + "tools": tools, + "job_spec_sha256": spec.fingerprint, + "resource_request": asdict(spec.resources), + "warnings": [] if tools["sacct"] else ["sacct is unavailable; terminal accounting may be incomplete."], + } + + def render_script(self, spec: JobSpec, *, job_id: str) -> str: + if spec.backend != "slurm": + raise ValueError("SlurmBackend requires backend=slurm") + resources = spec.resources + hours, remainder = divmod(resources.wall_time_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + directives = [ + f"#SBATCH --job-name={spec.name}", + f"#SBATCH --cpus-per-task={resources.cpus}", + f"#SBATCH --mem={resources.memory_mb}M", + f"#SBATCH --time={hours:02d}:{minutes:02d}:{seconds:02d}", + f"#SBATCH --output={shlex.quote(str(self._job_dir(job_id) / 'stdout.log'))}", + f"#SBATCH --error={shlex.quote(str(self._job_dir(job_id) / 'stderr.log'))}", + ] + if resources.gpus: + gres = f"gpu:{resources.gpu_type}:{resources.gpus}" if resources.gpu_type else f"gpu:{resources.gpus}" + directives.append(f"#SBATCH --gres={gres}") + for flag, value in (("partition", resources.partition), ("account", resources.account), ("qos", resources.qos)): + if value: + directives.append(f"#SBATCH --{flag}={value}") + exports = "\n".join(f"export {shlex.quote(key)}={shlex.quote(value)}" for key, value in sorted(spec.environment.items())) + command = shlex.join(spec.command) + return "\n".join([ + "#!/usr/bin/env bash", + *directives, + "set -euo pipefail", + f"cd {shlex.quote(spec.working_directory)}", + exports, + f"timeout --signal=TERM {spec.timeout_seconds}s {command}", + "", + ]) + + def submit(self, spec: JobSpec, *, approval: Mapping[str, Any] | None = None) -> dict[str, Any]: + if spec.backend != "slurm": + raise ValueError("SlurmBackend requires backend=slurm") + if approval is None: + raise ValueError("Slurm submission requires explicit approval") + validate_approval_receipt(approval, spec) + preflight = self.preflight(spec) + if not preflight["ready"]: + raise ValueError("Slurm preflight failed") + job_id = "slurm-local-" + uuid.uuid4().hex[:16] + job_dir = self._job_dir(job_id) + job_dir.mkdir(parents=True) + _atomic_json(job_dir / "spec.json", spec.to_dict()) + script_path = job_dir / "job.sh" + script_path.write_text(self.render_script(spec, job_id=job_id), encoding="utf-8") + script_path.chmod(0o700) + completed = subprocess.run(["sbatch", "--parsable", str(script_path)], text=True, capture_output=True, check=False) + if completed.returncode != 0: + raise ValueError(f"sbatch failed: {completed.stderr.strip() or completed.stdout.strip()}") + scheduler_job_id = completed.stdout.strip().split(";", 1)[0] + if not scheduler_job_id: + raise ValueError("sbatch returned no scheduler job ID") + submitted_at = _now() + state = _state_material(spec, job_id=job_id, state="submitted", submitted_at=submitted_at, scheduler_job_id=scheduler_job_id, message="submitted to Slurm") + _atomic_json(job_dir / "state.json", state) + return state + + def status(self, job_id: str) -> dict[str, Any]: + job_dir = self._job_dir(job_id) + state = _read_json(job_dir / "state.json") + validate_job_state(state) + if state["state"] in TERMINAL_STATES: + return state + scheduler_job_id = str(state.get("scheduler_job_id") or "") + raw_state = "" + exit_code = None + if shutil.which("sacct"): + completed = subprocess.run(["sacct", "-n", "-P", "-j", scheduler_job_id, "--format=State,ExitCode,Elapsed,MaxRSS"], text=True, capture_output=True, check=False) + first = next((line for line in completed.stdout.splitlines() if line.strip()), "") + if first: + fields = first.split("|") + raw_state = fields[0].split("+", 1)[0].strip().upper() + if len(fields) > 1 and fields[1]: + try: + exit_code = int(fields[1].split(":", 1)[0]) + except ValueError: + exit_code = None + if not raw_state and shutil.which("squeue"): + completed = subprocess.run(["squeue", "-h", "-j", scheduler_job_id, "-o", "%T"], text=True, capture_output=True, check=False) + raw_state = completed.stdout.strip().splitlines()[0].upper() if completed.stdout.strip() else "" + if not raw_state: + return state + mapped = SLURM_STATE_MAP.get(raw_state, "running") + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + updated = _state_material( + spec, + job_id=job_id, + state=mapped, + submitted_at=str(state["submitted_at"]), + started_at=state.get("started_at") or (_now() if mapped == "running" else None), + finished_at=_now() if mapped in TERMINAL_STATES else None, + exit_code=exit_code, + failure_class=None if mapped == "completed" else (raw_state.lower() if mapped in TERMINAL_STATES else None), + scheduler_job_id=scheduler_job_id, + message=f"Slurm state {raw_state}", + ) + _atomic_json(job_dir / "state.json", updated) + return updated + + def cancel(self, job_id: str) -> dict[str, Any]: + job_dir = self._job_dir(job_id) + state = self.status(job_id) + if state["state"] in TERMINAL_STATES: + return state + completed = subprocess.run(["scancel", str(state["scheduler_job_id"])], text=True, capture_output=True, check=False) + if completed.returncode != 0: + raise ValueError(f"scancel failed: {completed.stderr.strip() or completed.stdout.strip()}") + spec = JobSpec.from_payload(_read_json(job_dir / "spec.json")) + cancelled = _state_material( + spec, + job_id=job_id, + state="cancelled", + submitted_at=str(state["submitted_at"]), + started_at=state.get("started_at"), + finished_at=_now(), + scheduler_job_id=state.get("scheduler_job_id"), + failure_class="user-cancelled", + message="Slurm job cancellation requested", + ) + _atomic_json(job_dir / "state.json", cancelled) + return cancelled + + def collect(self, job_id: str) -> dict[str, Any]: + # Shared-filesystem Slurm output collection uses the same hash contract + # as local execution. Remote object transfer belongs to a separate, + # approved staging route. + state = self.status(job_id) + if state["state"] not in TERMINAL_STATES: + raise ValueError("cannot collect a nonterminal Slurm job") + local = LocalBackend(self.state_dir) + return local.collect(job_id) + + +def load_spec(path: Path) -> JobSpec: + return JobSpec.from_payload(_read_json(path)) + + +def backend_for(name: str, state_dir: Path) -> JobBackend: + if name == "local": + return LocalBackend(state_dir) + if name == "slurm": + return SlurmBackend(state_dir) + raise ValueError(f"unsupported backend: {name}") + + +def _worker_main(argv: list[str]) -> int: + if len(argv) != 2 or argv[0] != "_worker": + raise ValueError("invalid worker invocation") + return _worker(Path(argv[1]).resolve()) + + +if __name__ == "__main__": + raise SystemExit(_worker_main(sys.argv[1:])) diff --git a/src/codex_science/experiment_planner.py b/src/codex_science/experiment_planner.py new file mode 100644 index 00000000..dffc8845 --- /dev/null +++ b/src/codex_science/experiment_planner.py @@ -0,0 +1,310 @@ +"""Deterministic, non-executing multi-objective next-experiment planning. + +The planner ranks only declared candidate properties. It does not infer chemical +similarity, biological validity, or calibrated information gain from a model +name. Controls and scientific constraints remain explicit inputs. +""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter, defaultdict +from datetime import datetime, timezone +from typing import Any, Iterable, Mapping + + +DIRECTIONS = {"maximize", "minimize"} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _number(value: Any, label: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{label} must be numeric") + try: + result = float(value) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} must be numeric") from error + if result != result or result in {float("inf"), float("-inf")}: + raise ValueError(f"{label} must be finite") + return result + + +def _validate_objectives(raw: Any) -> list[dict[str, Any]]: + if not isinstance(raw, list) or not raw: + raise ValueError("objectives must be a non-empty list") + objectives: list[dict[str, Any]] = [] + names: set[str] = set() + for index, item in enumerate(raw): + if not isinstance(item, Mapping): + raise ValueError(f"objective {index} must be an object") + name = _text(item.get("name"), f"objective {index} name") + if name in names: + raise ValueError(f"duplicate objective: {name}") + names.add(name) + direction = _text(item.get("direction"), f"objective {name} direction") + if direction not in DIRECTIONS: + raise ValueError(f"invalid objective direction: {direction}") + weight = _number(item.get("weight", 1.0), f"objective {name} weight") + if weight < 0: + raise ValueError(f"objective {name} weight must be non-negative") + objectives.append({"name": name, "direction": direction, "weight": weight, "required": bool(item.get("required", True))}) + if not any(item["weight"] > 0 for item in objectives): + raise ValueError("at least one objective must have positive weight") + return objectives + + +def _validate_candidates(raw: Any, objectives: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not isinstance(raw, list) or not raw: + raise ValueError("candidates must be a non-empty list") + required = {item["name"] for item in objectives if item["required"]} + result: list[dict[str, Any]] = [] + ids: set[str] = set() + for index, item in enumerate(raw): + if not isinstance(item, Mapping): + raise ValueError(f"candidate {index} must be an object") + candidate_id = _text(item.get("id"), f"candidate {index} id") + if candidate_id in ids: + raise ValueError(f"duplicate candidate ID: {candidate_id}") + ids.add(candidate_id) + properties_raw = item.get("properties") + if not isinstance(properties_raw, Mapping): + raise ValueError(f"candidate {candidate_id} properties must be an object") + missing = sorted(required - set(properties_raw)) + if missing: + raise ValueError(f"candidate {candidate_id} lacks required objectives: {', '.join(missing)}") + properties = {str(key): _number(value, f"candidate {candidate_id} property {key}") for key, value in properties_raw.items()} + cost = _number(item.get("cost", 0), f"candidate {candidate_id} cost") + uncertainty = _number(item.get("uncertainty", 0), f"candidate {candidate_id} uncertainty") + if cost < 0 or uncertainty < 0: + raise ValueError(f"candidate {candidate_id} cost and uncertainty must be non-negative") + result.append({ + "id": candidate_id, + "properties": properties, + "cost": cost, + "uncertainty": uncertainty, + "diversity_group": str(item.get("diversity_group", item.get("scaffold_id", candidate_id))), + "control": bool(item.get("control", False)), + "eligible": bool(item.get("eligible", True)), + "exclusion_reason": str(item.get("exclusion_reason", "")), + "metadata": dict(item.get("metadata", {})) if isinstance(item.get("metadata", {}), Mapping) else {}, + }) + return result + + +def _normalize(candidates: list[dict[str, Any]], objectives: list[dict[str, Any]]) -> tuple[dict[str, dict[str, float]], dict[str, float]]: + normalized: dict[str, dict[str, float]] = {item["id"]: {} for item in candidates} + for objective in objectives: + name = objective["name"] + values = [candidate["properties"].get(name) for candidate in candidates if name in candidate["properties"]] + if not values: + continue + minimum, maximum = min(values), max(values) + for candidate in candidates: + if name not in candidate["properties"]: + normalized[candidate["id"]][name] = 0.0 + continue + value = candidate["properties"][name] + score = 1.0 if maximum == minimum else (value - minimum) / (maximum - minimum) + if objective["direction"] == "minimize": + score = 1.0 - score + normalized[candidate["id"]][name] = score + uncertainties = [candidate["uncertainty"] for candidate in candidates] + low, high = min(uncertainties), max(uncertainties) + uncertainty_scores = { + candidate["id"]: (1.0 if high == low and high > 0 else 0.0 if high == low else (candidate["uncertainty"] - low) / (high - low)) + for candidate in candidates + } + return normalized, uncertainty_scores + + +def _dominates(left: Mapping[str, float], right: Mapping[str, float], objective_names: Iterable[str]) -> bool: + values = [(left.get(name, 0.0), right.get(name, 0.0)) for name in objective_names] + return all(a >= b for a, b in values) and any(a > b for a, b in values) + + +def pareto_fronts(candidates: list[dict[str, Any]], normalized: Mapping[str, Mapping[str, float]], objectives: list[dict[str, Any]]) -> list[list[str]]: + remaining = {candidate["id"] for candidate in candidates} + names = [item["name"] for item in objectives if item["weight"] > 0] + fronts: list[list[str]] = [] + while remaining: + front = sorted( + candidate_id for candidate_id in remaining + if not any( + other != candidate_id and _dominates(normalized[other], normalized[candidate_id], names) + for other in remaining + ) + ) + if not front: + raise ValueError("could not construct Pareto front") + fronts.append(front) + remaining.difference_update(front) + return fronts + + +def plan_next_experiment(payload: Mapping[str, Any], *, created_at: str | None = None) -> dict[str, Any]: + if payload.get("schema_version", 1) != 1: + raise ValueError("unsupported next-experiment input schema") + decision = _text(payload.get("decision"), "decision") + objectives = _validate_objectives(payload.get("objectives")) + candidates = _validate_candidates(payload.get("candidates"), objectives) + constraints = payload.get("constraints") + if not isinstance(constraints, Mapping): + raise ValueError("constraints must be an object") + batch_raw = constraints.get("batch_size", 0) + group_raw = constraints.get("diversity_group_cap", constraints.get("scaffold_cap", batch_raw)) + controls_raw = constraints.get("minimum_controls", 0) + if any(isinstance(item, bool) for item in (batch_raw, group_raw, controls_raw)): + raise ValueError("batch size, diversity cap, and minimum controls must be integers") + batch_size = int(batch_raw) + budget = _number(constraints.get("budget", 0), "budget") + group_cap = int(group_raw) + minimum_controls = int(controls_raw) + if batch_size < 1 or budget < 0 or group_cap < 1 or minimum_controls < 0: + raise ValueError("batch size, budget, diversity cap, and minimum controls are invalid") + uncertainty_weight = _number(payload.get("uncertainty_weight", 0.25), "uncertainty_weight") + diversity_bonus = _number(payload.get("diversity_bonus", 0.1), "diversity_bonus") + if uncertainty_weight < 0 or diversity_bonus < 0: + raise ValueError("uncertainty_weight and diversity_bonus must be non-negative") + claim_ids = sorted({_text(item, "claim_id") for item in payload.get("claim_ids", [])}) + + eligible = [item for item in candidates if item["eligible"]] + ineligible = [item for item in candidates if not item["eligible"]] + if not eligible: + raise ValueError("no eligible candidates") + normalized, uncertainty_scores = _normalize(eligible, objectives) + fronts = pareto_fronts(eligible, normalized, objectives) + front_rank = {candidate_id: rank for rank, front in enumerate(fronts) for candidate_id in front} + weight_total = sum(item["weight"] for item in objectives) + by_id = {item["id"]: item for item in candidates} + + def base_utility(candidate: Mapping[str, Any]) -> float: + objective_score = sum( + objective["weight"] * normalized[candidate["id"]].get(objective["name"], 0.0) + for objective in objectives + ) / weight_total + return objective_score + uncertainty_weight * uncertainty_scores[candidate["id"]] + + selected: list[dict[str, Any]] = [] + selected_ids: set[str] = set() + group_counts: Counter[str] = Counter() + total_cost = 0.0 + rejection_reasons: dict[str, list[str]] = defaultdict(list) + + controls = sorted((item for item in eligible if item["control"]), key=lambda item: (item["cost"], item["id"])) + if len(controls) < minimum_controls: + raise ValueError(f"minimum_controls={minimum_controls} but only {len(controls)} eligible controls exist") + for candidate in controls[:minimum_controls]: + if total_cost + candidate["cost"] > budget or len(selected) >= batch_size: + raise ValueError("required controls do not fit the batch or budget") + if group_counts[candidate["diversity_group"]] >= group_cap: + raise ValueError("required controls violate diversity_group_cap") + selected.append(candidate) + selected_ids.add(candidate["id"]) + group_counts[candidate["diversity_group"]] += 1 + total_cost += candidate["cost"] + + while len(selected) < batch_size: + scored: list[tuple[float, int, str, dict[str, Any]]] = [] + for candidate in eligible: + if candidate["id"] in selected_ids: + continue + if total_cost + candidate["cost"] > budget: + rejection_reasons[candidate["id"]].append("budget") + continue + if group_counts[candidate["diversity_group"]] >= group_cap: + rejection_reasons[candidate["id"]].append("diversity-group-cap") + continue + novelty = 1.0 if group_counts[candidate["diversity_group"]] == 0 else 0.0 + score = base_utility(candidate) + diversity_bonus * novelty - 0.01 * front_rank[candidate["id"]] + scored.append((score, -front_rank[candidate["id"]], candidate["id"], candidate)) + if not scored: + break + scored.sort(key=lambda item: (-item[0], item[1], item[2])) + candidate = scored[0][3] + selected.append(candidate) + selected_ids.add(candidate["id"]) + group_counts[candidate["diversity_group"]] += 1 + total_cost += candidate["cost"] + + selected_records = [] + for candidate in selected: + selected_records.append({ + **candidate, + "normalized_objectives": normalized[candidate["id"]], + "uncertainty_score": uncertainty_scores[candidate["id"]], + "pareto_front": front_rank[candidate["id"]], + "base_utility": base_utility(candidate), + }) + for candidate in ineligible: + rejection_reasons[candidate["id"]].append(candidate["exclusion_reason"] or "ineligible") + for candidate in eligible: + if candidate["id"] not in selected_ids and not rejection_reasons[candidate["id"]]: + rejection_reasons[candidate["id"]].append("not-selected-within-batch") + + expected_information_gain = sum(uncertainty_scores[item["id"]] for item in selected) / len(selected) if selected else 0.0 + material = { + "schema_version": 1, + "decision": decision, + "status": "proposed", + "executed": False, + "created_at": created_at or _now(), + "objectives": objectives, + "constraints": { + "batch_size": batch_size, + "budget": budget, + "diversity_group_cap": group_cap, + "minimum_controls": minimum_controls, + }, + "uncertainty_weight": uncertainty_weight, + "diversity_bonus": diversity_bonus, + "selected": selected_records, + "selected_count": len(selected_records), + "total_cost": total_cost, + "remaining_budget": budget - total_cost, + "diversity_groups": dict(sorted(group_counts.items())), + "pareto_fronts": fronts, + "expected_information_gain_proxy": expected_information_gain, + "rejected": [ + {"id": candidate_id, "reasons": sorted(set(reasons)), "candidate": by_id[candidate_id]} + for candidate_id, reasons in sorted(rejection_reasons.items()) + ], + "claim_ids": claim_ids, + "required_controls_satisfied": sum(item["control"] for item in selected) >= minimum_controls, + "approval_required": True, + "evidence_boundary": "This deterministic proposal ranks only declared candidate properties and constraints. It does not infer chemical similarity, validate uncertainty calibration, execute an experiment, or prove expected information gain." + } + fingerprint = _fingerprint(material) + return {**material, "proposal_id": f"experiment-{fingerprint[:20]}", "fingerprint": fingerprint} + + +def validate_experiment_proposal(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1 or payload.get("status") != "proposed" or payload.get("executed") is not False: + raise ValueError("invalid experiment proposal state") + for field in ("proposal_id", "decision", "created_at", "evidence_boundary"): + _text(payload.get(field), field) + if not isinstance(payload.get("selected"), list) or not isinstance(payload.get("rejected"), list): + raise ValueError("selected and rejected must be lists") + material = dict(payload) + proposal_id = str(material.pop("proposal_id")) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint or proposal_id != f"experiment-{fingerprint[:20]}": + raise ValueError("experiment proposal fingerprint or ID mismatch") diff --git a/src/codex_science/fair_export.py b/src/codex_science/fair_export.py new file mode 100644 index 00000000..cfcb06fd --- /dev/null +++ b/src/codex_science/fair_export.py @@ -0,0 +1,231 @@ +"""Portable FAIR-oriented exports from a validated Codex Science run. + +The exporters produce conservative RO-Crate-, W3C PROV-, and scientific-BOM +records. They do not claim formal certification, regulatory compliance, or +portability of bytes that are not included in the source bundle. +""" + +from __future__ import annotations + +import hashlib +import json +import mimetypes +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from codex_science.artifact_store import describe_directory, stream_sha256 +from codex_science.artifacts import validate_bundle + + +RO_CRATE_CONTEXT = "https://w3id.org/ro/crate/1.1/context" +PROV_CONTEXT = "https://www.w3.org/ns/prov#" + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha(value: Any) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + + +def _media_type(path: str, record: Mapping[str, Any]) -> str: + if record.get("media_type"): + return str(record["media_type"]) + guessed, _encoding = mimetypes.guess_type(path) + return guessed or "application/octet-stream" + + +def _artifact_entity(record: Mapping[str, Any]) -> dict[str, Any]: + path = str(record["path"]) + entity = { + "@id": path, + "@type": "Dataset" if record.get("artifact_type") == "directory-tree" else "File", + "name": path, + "encodingFormat": _media_type(path, record), + "sha256": str(record["sha256"]), + "additionalType": str(record.get("kind", "artifact")), + } + for source, target in (("size_bytes", "contentSize"), ("artifact_type", "artifactType"), ("root_sha256", "merkleRoot"), ("entry_count", "entryCount")): + if record.get(source) is not None: + entity[target] = record[source] + return entity + + +def build_ro_crate(manifest: Mapping[str, Any], *, exported_at: str) -> dict[str, Any]: + run_id = str(manifest["run_id"]) + artifacts = [_artifact_entity(item) for item in manifest.get("artifacts", []) if isinstance(item, Mapping)] + root = { + "@id": "./", + "@type": "Dataset", + "name": f"Codex Science run {run_id}", + "description": str(manifest.get("question", "")), + "dateModified": exported_at, + "hasPart": [{"@id": item["@id"]} for item in artifacts], + "mainEntity": {"@id": "manifest.json"}, + "conformsTo": [ + {"@id": "https://w3id.org/ro/crate/1.1"}, + {"@id": "https://github.com/eightmm/codex-science"} + ], + "reviewStatus": str(manifest.get("review", {}).get("status", "unknown")), + } + manifest_entity = { + "@id": "manifest.json", + "@type": "File", + "name": "Codex Science artifact manifest", + "encodingFormat": "application/json", + "about": {"@id": "./"}, + } + claims = [] + for claim in manifest.get("claims", []): + if not isinstance(claim, Mapping) or not claim.get("id"): + continue + claim_id = f"#claim-{claim['id']}" + claims.append({ + "@id": claim_id, + "@type": "CreativeWork", + "identifier": str(claim["id"]), + "text": str(claim.get("text", "")), + "citation": [{"@id": str(item)} for item in claim.get("evidence", [])], + }) + root["about"] = [{"@id": item["@id"]} for item in claims] + return { + "@context": RO_CRATE_CONTEXT, + "@graph": [ + {"@id": "ro-crate-metadata.json", "@type": "CreativeWork", "about": {"@id": "./"}, "conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"}}, + root, + manifest_entity, + *artifacts, + *claims, + ], + "codexScienceBoundary": "This export is RO-Crate-oriented metadata over a validated local bundle. External or omitted bytes remain external and formal RO-Crate conformance is not certified by this exporter." + } + + +def build_prov(manifest: Mapping[str, Any], *, exported_at: str) -> dict[str, Any]: + run_id = str(manifest["run_id"]) + entities: dict[str, Any] = {} + activities: dict[str, Any] = {} + used: dict[str, Any] = {} + generated: dict[str, Any] = {} + for index, item in enumerate(manifest.get("inputs", [])): + if not isinstance(item, Mapping): + continue + identifier = str(item.get("id") or item.get("path") or item.get("identifier") or f"input-{index}") + entities[f"input:{identifier}"] = {"prov:type": "Input", "codex:record": dict(item)} + for item in manifest.get("artifacts", []): + if isinstance(item, Mapping): + entities[f"artifact:{item['path']}"] = {"prov:type": "Artifact", "prov:label": str(item["path"]), "codex:sha256": str(item["sha256"]), "codex:kind": str(item.get("kind", "artifact"))} + for index, execution in enumerate(manifest.get("executions", [])): + if not isinstance(execution, Mapping): + continue + activity_id = f"execution:{execution.get('id', index)}" + activities[activity_id] = {"prov:type": "Execution", "prov:label": str(execution.get("command", activity_id)), "codex:exitCode": execution.get("exit_code")} + for entity_id in entities: + if entity_id.startswith("input:"): + used[f"used:{activity_id}:{entity_id}"] = {"prov:activity": activity_id, "prov:entity": entity_id} + run_activity = f"run:{run_id}" + activities[run_activity] = {"prov:type": "ScientificRun", "prov:label": str(manifest.get("question", run_id)), "prov:endedAtTime": exported_at, "codex:reviewStatus": str(manifest.get("review", {}).get("status", "unknown"))} + for entity_id in entities: + if entity_id.startswith("artifact:"): + generated[f"generated:{entity_id}"] = {"prov:entity": entity_id, "prov:activity": run_activity} + return { + "prefix": {"prov": PROV_CONTEXT, "codex": "https://github.com/eightmm/codex-science#"}, + "entity": entities, + "activity": activities, + "used": used, + "wasGeneratedBy": generated, + "codexScienceBoundary": "This is a deterministic W3C-PROV-oriented export of recorded lineage. It does not authenticate agents or reconstruct omitted execution events." + } + + +def build_scientific_bom(manifest: Mapping[str, Any], sidecars: Mapping[str, Any], *, exported_at: str) -> dict[str, Any]: + components: list[dict[str, Any]] = [] + for index, item in enumerate(manifest.get("code", [])): + if isinstance(item, Mapping): + components.append({"component_type": "code", "id": str(item.get("path") or item.get("module") or f"code-{index}"), "record": dict(item)}) + environment = manifest.get("environment", {}) + if isinstance(environment, Mapping): + for key in ("container_digest", "lockfile_sha256", "code_revision", "model_registry_sha256"): + if environment.get(key) is not None: + components.append({"component_type": key, "id": str(environment[key]), "record": {"value": environment[key]}}) + packages = environment.get("packages", []) + if isinstance(packages, list): + for index, package in enumerate(packages): + components.append({"component_type": "package", "id": str(package if not isinstance(package, Mapping) else package.get("name", f"package-{index}")), "record": package}) + for index, item in enumerate(manifest.get("inputs", [])): + if isinstance(item, Mapping): + component_type = "dataset" if item.get("source") or item.get("identifier") else "input" + components.append({"component_type": component_type, "id": str(item.get("id") or item.get("identifier") or item.get("path") or f"input-{index}"), "record": dict(item)}) + for receipt in sidecars.get("model_receipts_v2", []): + if isinstance(receipt, Mapping): + components.append({"component_type": "model-receipt", "id": str(receipt.get("model_id", "model")), "record": dict(receipt)}) + for record in manifest.get("artifacts", []): + if isinstance(record, Mapping) and record.get("kind") in {"database-snapshot", "model-weight", "container", "license-bom"}: + components.append({"component_type": str(record["kind"]), "id": str(record["path"]), "record": dict(record)}) + normalized = sorted(components, key=lambda item: (item["component_type"], item["id"], _sha(item["record"]))) + return { + "schema_version": 1, + "run_id": str(manifest["run_id"]), + "generated_at": exported_at, + "components": normalized, + "component_count": len(normalized), + "limitations": [ + "Only dependencies recorded in the manifest, environment, and recognized receipts are included.", + "A missing license record is not evidence that reuse is permitted.", + "Container, model, weight, database, and dataset identities remain source-specific." + ], + "evidence_boundary": "The scientific BOM supports dependency and license review; it is not a legal opinion, software vulnerability scan, or proof of complete transitive dependency capture." + } + + +def export_run(manifest_path: Path, output_dir: Path, *, exported_at: str | None = None) -> dict[str, Any]: + manifest_path = manifest_path.resolve() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("manifest must be an object") + sidecars = validate_bundle(manifest, manifest_path.parent) + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + timestamp = exported_at or _now() + ro_crate = build_ro_crate(manifest, exported_at=timestamp) + prov = build_prov(manifest, exported_at=timestamp) + bom = build_scientific_bom(manifest, sidecars, exported_at=timestamp) + files = { + "ro_crate": output_dir / "ro-crate-metadata.json", + "prov": output_dir / "prov.json", + "scientific_bom": output_dir / "scientific-bom.json", + } + _write_json(files["ro_crate"], ro_crate) + _write_json(files["prov"], prov) + _write_json(files["scientific_bom"], bom) + exports = [] + for name, path in files.items(): + digest, size = stream_sha256(path) + exports.append({"name": name, "path": path.name, "sha256": digest, "size_bytes": size}) + manifest_digest, _manifest_size = stream_sha256(manifest_path) + receipt_material = { + "schema_version": 1, + "run_id": manifest["run_id"], + "source_manifest_path": str(manifest_path), + "source_manifest_sha256": manifest_digest, + "generated_at": timestamp, + "exports": exports, + "status": "generated", + "certified": False, + "regulatory_compliance_claimed": False, + "evidence_boundary": "The export packages validated recorded metadata. It does not certify RO-Crate, W3C PROV, regulatory, or legal compliance." + } + receipt = {**receipt_material, "fingerprint": _sha(receipt_material)} + _write_json(output_dir / "export-receipt.json", receipt) + return receipt diff --git a/src/codex_science/mcp_server.py b/src/codex_science/mcp_server.py index bf8ef740..5287c5ca 100644 --- a/src/codex_science/mcp_server.py +++ b/src/codex_science/mcp_server.py @@ -10,6 +10,8 @@ from codex_science.connector_contract import QueryRequest, execute_connector from codex_science.connector_sources import SOURCE_BY_KEY, SOURCE_BY_TOOL, SOURCE_SPECS from codex_science.life_science import plan_life_science_research +from codex_science.source_operations_v3 import SOURCE_OPERATIONS_V3 +from codex_science.transport_v3 import execute_paginated from codex_science.version import MCP_VERSION PROTOCOL_VERSION = "2025-06-18" @@ -30,7 +32,6 @@ } ) LEGACY_SOURCE_SPECS = tuple(spec for spec in SOURCE_SPECS if spec.tool_name in LEGACY_TOOL_NAMES) -# Public compatibility contract used by existing tests and downstream integrations. CONNECTOR_SPECS = tuple((spec.tool_name, spec.description, spec.factory) for spec in LEGACY_SOURCE_SPECS) @@ -54,7 +55,7 @@ def _legacy_tool(name: str, description: str) -> dict[str, Any]: def _v2_tool() -> dict[str, Any]: return { "name": "science_query_source_v2", - "description": "Execute a bounded typed public-source query and return normalized records plus a canonical replay receipt.", + "description": "Execute a bounded normalized public-source query and return records plus a canonical v2 receipt.", "inputSchema": { "type": "object", "properties": { @@ -73,10 +74,31 @@ def _v2_tool() -> dict[str, Any]: } +def _v3_tool() -> dict[str, Any]: + return { + "name": "science_query_source_v3", + "description": "Execute a true-paginated source operation with HTTP headers, retries, rate-limit state, page hashes, and explicit completeness semantics.", + "inputSchema": { + "type": "object", + "properties": { + "source": {"type": "string", "enum": sorted(SOURCE_OPERATIONS_V3)}, + "operation": {"type": "string", "enum": ["search"], "default": "search"}, + "parameters": {"type": "object", "additionalProperties": True}, + "page_size": {"type": "integer", "minimum": 1, "maximum": 100, "default": 25}, + "max_pages": {"type": "integer", "minimum": 1, "maximum": 100, "default": 1}, + "evidence_cutoff": {"type": ["string", "null"]}, + }, + "required": ["source", "parameters"], + "additionalProperties": False, + }, + "annotations": {"readOnlyHint": True, "destructiveHint": False, "openWorldHint": True}, + } + + def _contracts_tool() -> dict[str, Any]: return { "name": "science_list_source_contracts", - "description": "List public-source operations, query semantics, and maturity states.", + "description": "List public-source operations, query semantics, maturity states, and v3 transport availability.", "inputSchema": {"type": "object", "properties": {}, "additionalProperties": False}, "annotations": {"readOnlyHint": True, "destructiveHint": False, "openWorldHint": False}, } @@ -87,6 +109,7 @@ def _contracts_tool() -> dict[str, Any]: *(_legacy_tool(spec.tool_name, spec.description) for spec in SOURCE_SPECS), _legacy_tool("science_plan_life_science_research", "Plan bounded normalization, evidence lanes, provenance, and synthesis."), _v2_tool(), + _v3_tool(), _contracts_tool(), ) TOOL_NAMES = frozenset(tool["name"] for tool in TOOLS) @@ -95,9 +118,7 @@ def _contracts_tool() -> dict[str, Any]: class CodexScienceMCP: def __init__(self, inventory_path: Path) -> None: self.inventory_path = inventory_path - # Keep the historical tool-name-keyed registry stable. self.connectors = {name: factory() for name, _description, factory in CONNECTOR_SPECS} - # Use a distinct source-key registry for typed v2 and newly cataloged adapters. self.sources = {spec.key: spec.factory() for spec in SOURCE_SPECS} @staticmethod @@ -128,7 +149,7 @@ def handle(self, request: Any) -> dict[str, Any] | None: "protocolVersion": PROTOCOL_VERSION, "capabilities": {"tools": {"listChanged": False}}, "serverInfo": {"name": "codex-science", "version": MCP_VERSION}, - "instructions": "Read-only scientific catalog and public-source queries. Prefer science_query_source_v2 for material evidence and replay receipts.", + "instructions": "Read-only scientific catalog and public-source queries. Prefer science_query_source_v3 when available for material evidence; use v2 for sources not yet migrated.", }, ) if method == "ping": @@ -149,7 +170,22 @@ def _call_tool(self, request_id: Any, params: Any) -> dict[str, Any]: if name == "science_list_source_contracts": if arguments: raise ValueError("science_list_source_contracts takes no arguments") - payload = [spec.public_contract() for spec in SOURCE_SPECS] + payload = [] + for spec in SOURCE_SPECS: + record = spec.public_contract() + record["v3_operations"] = ["search"] if spec.key in SOURCE_OPERATIONS_V3 else [] + payload.append(record) + elif name == "science_query_source_v3": + extra = set(arguments) - { + "source", "operation", "parameters", "page_size", "max_pages", "evidence_cutoff", + } + if extra: + raise ValueError(f"Unexpected arguments: {', '.join(sorted(extra))}") + source = arguments.get("source") + if source not in SOURCE_OPERATIONS_V3: + raise ValueError(f"Source has no v3 operation: {source}") + request = QueryRequest.from_payload({**arguments, "source_contract_version": "3"}) + payload = execute_paginated(SOURCE_OPERATIONS_V3[str(source)], request).to_dict() elif name == "science_query_source_v2": extra = set(arguments) - { "source", "operation", "parameters", "page_size", "max_pages", diff --git a/src/codex_science/pipeline_compiler.py b/src/codex_science/pipeline_compiler.py new file mode 100644 index 00000000..928f0ee1 --- /dev/null +++ b/src/codex_science/pipeline_compiler.py @@ -0,0 +1,455 @@ +"""Promote a passed scientific run into a reviewable skill draft. + +The compiler creates a draft only. It never modifies the audited catalog, +activates a skill, copies private inputs, or claims that one successful run is a +validated reusable method. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from codex_science.artifact_store import stream_sha256 +from codex_science.artifacts import validate_bundle +from codex_science.review import review_manifest + + +SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +SECRET_ARGUMENT_FLAGS = { + "--token", + "--password", + "--secret", + "--credential", + "--api-key", + "--apikey", + "--private-key", + "--access-key", + "--client-secret", + "--authorization", +} +SECRET_KEY_FRAGMENTS = { + "token", + "password", + "secret", + "credential", + "private_key", + "api_key", + "apikey", + "access_key", + "client_secret", + "authorization", +} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _redact_mapping(value: Any) -> Any: + if isinstance(value, Mapping): + result: dict[str, Any] = {} + for key, item in value.items(): + key_text = str(key) + if any(fragment in key_text.lower() for fragment in SECRET_KEY_FRAGMENTS): + result[key_text] = "[REDACTED]" + else: + result[key_text] = _redact_mapping(item) + return result + if isinstance(value, list): + return [_redact_mapping(item) for item in value] + return value + + +def _validate_command(command: list[str], index: int) -> None: + for argument in command: + lower = argument.lower() + if any(lower == flag or lower.startswith(flag + "=") for flag in SECRET_ARGUMENT_FLAGS): + raise ValueError(f"command_contract[{index}] contains a credential-bearing argument") + if "://" in argument and any( + token in lower + for token in ( + "token=", + "password=", + "secret=", + "api_key=", + "apikey=", + "access_key=", + "client_secret=", + "authorization=", + ) + ): + raise ValueError(f"command_contract[{index}] contains a credential-bearing URL") + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + + +def _artifact_summary(manifest: Mapping[str, Any]) -> list[dict[str, Any]]: + return [ + { + "path": str(item.get("path", "")), + "kind": str(item.get("kind", "artifact")), + "sha256": str(item.get("sha256", "")), + "artifact_type": str(item.get("artifact_type", "file")), + } + for item in manifest.get("artifacts", []) + if isinstance(item, Mapping) + ] + + +def _schema_inputs(manifest: Mapping[str, Any]) -> dict[str, Any]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input.schema.json", + "title": "Reusable scientific pipeline input", + "type": "object", + "properties": { + "question": {"type": "string", "minLength": 1}, + "parameters": {"type": "object", "additionalProperties": True}, + "inputs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string", "minLength": 1}, + "path": {"type": "string"}, + "identifier": {"type": "string"}, + "sha256": {"type": "string", "pattern": "^[0-9a-fA-F]{64}$"}, + "source": {"type": "string"}, + "release": {"type": "string"} + }, + "required": ["id"], + "anyOf": [{"required": ["path"]}, {"required": ["identifier"]}], + "additionalProperties": True + } + }, + "acceptance": { + "type": "object", + "properties": { + "criteria": {"type": "array", "items": {"type": "object"}}, + "review_required": {"const": True} + }, + "required": ["criteria", "review_required"], + "additionalProperties": False + } + }, + "required": ["question", "parameters", "inputs", "acceptance"], + "additionalProperties": False, + "x-source-run-input-count": len(manifest.get("inputs", [])), + "x-source-run-input-fields": sorted( + { + str(key) + for item in manifest.get("inputs", []) + if isinstance(item, Mapping) + for key in item.keys() + } + ), + } + + +def _schema_outputs() -> dict[str, Any]: + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output.schema.json", + "title": "Reusable scientific pipeline output", + "type": "object", + "properties": { + "run_id": {"type": "string", "minLength": 1}, + "manifest_path": {"type": "string", "minLength": 1}, + "manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "claims": {"type": "array", "items": {"type": "object"}}, + "artifacts": {"type": "array", "items": {"type": "object"}}, + "review": { + "type": "object", + "properties": {"status": {"const": "passed"}}, + "required": ["status"], + "additionalProperties": True + }, + "limitations": {"type": "array", "items": {"type": "string"}} + }, + "required": ["run_id", "manifest_path", "manifest_sha256", "claims", "artifacts", "review", "limitations"], + "additionalProperties": False + } + + +def _skill_text(name: str, description: str, source_run: str, workflow_steps: list[dict[str, Any]]) -> str: + step_lines = [] + for index, step in enumerate(workflow_steps, 1): + step_id = str(step.get("id", f"step-{index}")) + detail = str(step.get("description", step_id)).strip() + step_lines.append(f"{index}. `{step_id}` — {detail}") + steps = "\n".join(step_lines) or "1. Reconstruct the approved workflow from a new decision contract." + return f'''--- +name: {name} +description: "{description.replace('"', "'")}" +--- + +# {name} + +> Draft generated from reviewed run `{source_run}`. It is not active, cataloged, or validated for new inputs until its fixtures, seeded failures, environment, and independent review pass. + +## Decision contract + +Record the new question, intended decision, population or system, input identities and hashes, parameter boundaries, primary acceptance criteria, uncertainty rule, non-goals, model/source revisions, cost and approval boundaries, and what would falsify each material claim. Do not inherit the source run's conclusion. + +## Reference usage + +Before executing, **MUST** read [the generated pipeline contract](references/pipeline.md). Do not guess commands, input fields, environment, or output semantics from this summary. The source run is precedent for structure only, not evidence for a new run. + +## Workflow + +{steps} + +Create a new run ID, initialize provenance, execute only after required approvals, preserve failed attempts and null results, and run `$science-review` against the new artifacts. Use `$science-provenance` for every command, environment, input, output, claim, and receipt. + +## Outputs + +Return a new hash-validated artifact manifest satisfying `output.schema.json`, with explicit claims, uncertainty, limitations, model/source revisions, execution logs, and an independent review receipt when independence is available. Never return the source run as the new output. + +## Boundaries + +- Do not activate or catalog this draft automatically. +- Do not copy private, sensitive, proprietary, or credential-bearing source inputs into the draft. +- Do not assume the source environment, model, database, or endpoint is still available. +- Do not reuse a source review receipt for changed inputs or code. +- Do not claim generalization from one successful run. +- Do not execute destructive, remote, paid, or write-capable operations without the normal approval gate. +''' + + +def _reference_text( + *, + name: str, + source_manifest_path: Path, + source_manifest_sha256: str, + manifest: Mapping[str, Any], + command_contract: list[list[str]], + limitations: list[str], +) -> str: + commands = "\n".join(f"- `{json.dumps(command, ensure_ascii=False)}`" for command in command_contract) or "- No command was promoted. Reconstruct commands from an approved method before validation." + steps = "\n".join( + f"- `{step.get('id', '')}` — {step.get('description', step.get('id', ''))}" + for step in manifest.get("plan", []) + if isinstance(step, Mapping) + ) or "- No source plan steps were recorded." + limitations_text = "\n".join(f"- {item}" for item in limitations) + return f'''# Generated pipeline contract + +This reference was generated from a reviewed run and is a **draft implementation contract**, not scientific evidence and not an activation decision. + +## Source record + +- source run: `{manifest.get('run_id')}` +- source manifest: `{source_manifest_path}` +- source manifest SHA-256: `{source_manifest_sha256}` +- source review status: `{manifest.get('review', {}).get('status', 'unknown')}` +- generated skill: `{name}` + +The source manifest path may be local-machine-specific. The SHA-256 is the stable identity. Never use this reference to bypass source bundle validation. + +## Inputs + +Use `input.schema.json`. Each new input needs an ID plus either a local path or canonical external identifier. Record a checksum when practical, source/release for external data, and all transformations. Do not reuse source paths blindly. + +## Environment + +Reconstruct and review the runtime, lockfile/container, code revision, model and weight revisions, databases, hardware, seed, determinism settings, and material non-secret environment variables. The source environment was: + +```json +{json.dumps(_redact_mapping(manifest.get('environment', {})), indent=2, sort_keys=True, ensure_ascii=False)} +``` + +## Source workflow shape + +{steps} + +## Command contract + +Commands are represented as argv arrays. They are not executed by reading this file. + +{commands} + +Before using a promoted command: + +1. resolve every executable and file path; +2. replace source-run paths with new run inputs; +3. pin or record model, database, and code revisions; +4. run the smallest smoke test; +5. verify exit status and output schema; +6. preserve logs and failures; +7. evaluate scientific acceptance separately from process success. + +## Output contract + +Use `output.schema.json`. The new run must contain a different run ID and manifest hash, current claims and limitations, current artifact hashes, and a current passed review. A process receipt alone is not an output contract. + +## Validation and promotion + +The generated draft starts below active maturity. To promote it: + +1. add a redistributable acceptance fixture; +2. seed material failure cases; +3. add deterministic tests; +4. record exact source/model/database drift behavior; +5. run independent review; +6. add `quality.json` with an honest maturity declaration; +7. pass the repository skill, reference, inventory, wrapper, and contract gates; +8. activate it only through normal catalog policy. + +## Failure handling + +| Failure | Required response | +| --- | --- | +| source manifest changed | stop; restore or identify the reviewed source bytes | +| command contains unavailable paths | revise the draft and test in the approved environment | +| model/database revision unavailable | select a reviewed replacement and create a new acceptance result | +| output schema mismatch | keep the failed run and correct code/schema before retry | +| source result does not reproduce | report non-reproduction; do not lower the threshold post hoc | +| private input referenced | replace it with an approved immutable reference or remove the dependency | +| new operation requires approval | stop at the approval gate | + +## Common mistakes + +- Treating a generated draft as active. +- Copying a source conclusion into the new question. +- Reusing source review receipts. +- Reconstructing commands from prose while ignoring argv and environment details. +- Omitting failed source steps because the final run passed. +- Claiming generalization from a single fixture. + +## Known limitations + +{limitations_text} + +## Evidence boundary + +This reference preserves workflow shape and explicit source lineage. It does not validate new inputs, authenticate the producer, guarantee reproducibility, or establish scientific truth. +''' + + +def compile_pipeline_draft( + *, + manifest_path: Path, + output_dir: Path, + name: str, + description: str, + command_contract: list[list[str]] | None = None, + limitations: list[str] | None = None, + generated_at: str | None = None, +) -> dict[str, Any]: + name = _text(name, "name") + if not SKILL_NAME_RE.fullmatch(name): + raise ValueError("name must be lowercase kebab-case") + description = _text(description, "description") + manifest_path = manifest_path.resolve() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("manifest must be an object") + sidecars = validate_bundle(manifest, manifest_path.parent) + review = review_manifest(manifest, manifest_path.parent, sidecars=sidecars) + if manifest.get("review", {}).get("status") != "passed" or review.get("status") != "passed": + raise ValueError("only a currently passed run may be promoted to a skill draft") + if any(step.get("status") != "completed" for step in manifest.get("plan", []) if isinstance(step, Mapping)): + raise ValueError("source run has incomplete plan steps") + output_dir = output_dir.resolve() + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"output directory is not empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + references = output_dir / "references" + references.mkdir() + manifest_sha, _size = stream_sha256(manifest_path) + commands = command_contract or [] + for index, command in enumerate(commands): + if not isinstance(command, list) or not command or not all(isinstance(item, str) and item for item in command): + raise ValueError(f"command_contract[{index}] must be a non-empty argv list") + _validate_command(command, index) + limit_values = limitations or [ + "Generated from one reviewed run and not validated for a new input distribution.", + "External sources, models, weights, databases, and endpoints may drift.", + "The draft contains no automatic activation or remote/write permission." + ] + (output_dir / "SKILL.md").write_text( + _skill_text(name, description, str(manifest.get("run_id", "unknown")), [dict(item) for item in manifest.get("plan", []) if isinstance(item, Mapping)]), + encoding="utf-8", + ) + _write_json(output_dir / "input.schema.json", _schema_inputs(manifest)) + _write_json(output_dir / "output.schema.json", _schema_outputs()) + (references / "pipeline.md").write_text( + _reference_text( + name=name, + source_manifest_path=manifest_path, + source_manifest_sha256=manifest_sha, + manifest=manifest, + command_contract=commands, + limitations=limit_values, + ), + encoding="utf-8", + ) + _write_json( + references / "index.json", + { + "schema_version": 1, + "skill": name, + "references": [ + { + "id": "pipeline", + "path": "references/pipeline.md", + "purpose": "Exact generated workflow shape, input/output schemas, promotion path, failure handling, and source lineage.", + "read_when": ["before using the generated draft", "before reconstructing commands", "before validating or promoting the draft"], + "required_before": ["executing any promoted command", "declaring the draft reusable", "adding it to an audited catalog"], + "search_patterns": ["## Command contract", "## Output contract", "## Validation and promotion", "## Failure handling"], + "authority": "Generated from a hash-validated Codex Science run", + "version": "1", + "evidence_boundary": "The generated reference preserves workflow structure but is not evidence for a new run." + } + ] + }, + ) + promotion_material = { + "schema_version": 1, + "status": "draft", + "activated": False, + "cataloged": False, + "generated_at": generated_at or _now(), + "skill": name, + "source_run_id": manifest.get("run_id"), + "source_manifest_path": str(manifest_path), + "source_manifest_sha256": manifest_sha, + "source_review_status": manifest.get("review", {}).get("status"), + "source_artifacts": _artifact_summary(manifest), + "command_count": len(commands), + "limitations": limit_values, + "required_next_steps": [ + "add redistributable acceptance fixture", + "seed material failures", + "add deterministic tests", + "run independent review", + "declare honest maturity", + "pass catalog activation policy" + ] + } + promotion = {**promotion_material, "fingerprint": _fingerprint(promotion_material)} + _write_json(output_dir / "promotion.json", promotion) + return promotion diff --git a/src/codex_science/project_store.py b/src/codex_science/project_store.py new file mode 100644 index 00000000..b4a3583d --- /dev/null +++ b/src/codex_science/project_store.py @@ -0,0 +1,599 @@ +"""Local project-level evidence memory over immutable scientific run bundles. + +The SQLite store is an index and decision ledger. A run manifest and its hashed +artifacts remain authoritative. Import, fork, compare, and merge-plan operations +never rewrite a run bundle. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Iterator, Mapping + +from codex_science.artifact_store import stream_sha256 +from codex_science.artifacts import validate_bundle +from codex_science.collaboration import diff_runs + + +SCHEMA_VERSION = 1 +POLARITIES = {"supports", "contradicts", "qualifies", "neutral", "unavailable"} +BRANCH_STATUSES = {"active", "candidate", "merged", "abandoned"} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256_json(value: Any) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _sha(value: Any, label: str) -> str: + text = _text(value, label).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{label} must be a SHA-256 digest") + return text + + +def _relative(value: Any, label: str) -> str: + text = _text(value, label) + pure = PurePosixPath(text) + if pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"{label} must be a safe relative path") + return pure.as_posix() + + +def _json_object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{label} must be an object") + payload = dict(value) + try: + json.dumps(payload, ensure_ascii=False) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} must be JSON-compatible") from error + return payload + + +def _manifest_claims(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + return {str(item["id"]): dict(item) for item in manifest.get("claims", []) if isinstance(item, Mapping) and item.get("id")} + + +def _manifest_artifacts(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + return {str(item["path"]): dict(item) for item in manifest.get("artifacts", []) if isinstance(item, Mapping) and item.get("path")} + + +@dataclass(frozen=True) +class ImportedRun: + project_id: str + run_id: str + branch_name: str + parent_run_id: str | None + manifest_path: str + manifest_sha256: str + review_status: str + imported_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "project_id": self.project_id, + "run_id": self.run_id, + "branch_name": self.branch_name, + "parent_run_id": self.parent_run_id, + "manifest_path": self.manifest_path, + "manifest_sha256": self.manifest_sha256, + "review_status": self.review_status, + "imported_at": self.imported_at, + } + + +class ProjectStore: + def __init__(self, path: Path) -> None: + self.path = path.resolve() + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + self.path.parent.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(self.path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA journal_mode = WAL") + try: + yield connection + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def initialize(self) -> None: + with self.connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS projects ( + project_id TEXT PRIMARY KEY, + title TEXT NOT NULL, + question TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS runs ( + project_id TEXT NOT NULL, + run_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + parent_run_id TEXT, + manifest_path TEXT NOT NULL, + manifest_sha256 TEXT NOT NULL, + question TEXT NOT NULL, + review_status TEXT NOT NULL, + imported_at TEXT NOT NULL, + PRIMARY KEY (project_id, run_id), + FOREIGN KEY (project_id) REFERENCES projects(project_id), + FOREIGN KEY (project_id, parent_run_id) REFERENCES runs(project_id, run_id) + ); + CREATE TABLE IF NOT EXISTS run_artifacts ( + project_id TEXT NOT NULL, + run_id TEXT NOT NULL, + artifact_path TEXT NOT NULL, + sha256 TEXT NOT NULL, + kind TEXT NOT NULL, + artifact_type TEXT NOT NULL, + payload_json TEXT NOT NULL, + PRIMARY KEY (project_id, run_id, artifact_path), + FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, run_id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS run_claims ( + project_id TEXT NOT NULL, + run_id TEXT NOT NULL, + claim_id TEXT NOT NULL, + payload_json TEXT NOT NULL, + PRIMARY KEY (project_id, run_id, claim_id), + FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, run_id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS branches ( + project_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + base_run_id TEXT NOT NULL, + head_run_id TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (project_id, branch_name), + FOREIGN KEY (project_id, base_run_id) REFERENCES runs(project_id, run_id), + FOREIGN KEY (project_id, head_run_id) REFERENCES runs(project_id, run_id) + ); + CREATE TABLE IF NOT EXISTS assertions ( + project_id TEXT NOT NULL, + assertion_id TEXT NOT NULL, + run_id TEXT NOT NULL, + claim_id TEXT NOT NULL, + source_id TEXT NOT NULL, + polarity TEXT NOT NULL, + effect_measure TEXT, + estimate REAL, + interval_low REAL, + interval_high REAL, + unit TEXT, + sample_size INTEGER, + population TEXT, + independence_group TEXT NOT NULL, + risk_of_bias_ref TEXT, + locator_json TEXT NOT NULL, + created_at TEXT NOT NULL, + fingerprint TEXT NOT NULL, + PRIMARY KEY (project_id, assertion_id), + FOREIGN KEY (project_id, run_id) REFERENCES runs(project_id, run_id), + FOREIGN KEY (project_id, run_id, claim_id) REFERENCES run_claims(project_id, run_id, claim_id) + ); + CREATE TABLE IF NOT EXISTS merge_plans ( + project_id TEXT NOT NULL, + plan_id TEXT NOT NULL, + source_branch TEXT NOT NULL, + target_branch TEXT NOT NULL, + base_run_id TEXT NOT NULL, + source_run_id TEXT NOT NULL, + target_run_id TEXT NOT NULL, + status TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + fingerprint TEXT NOT NULL, + PRIMARY KEY (project_id, plan_id) + ); + CREATE TABLE IF NOT EXISTS events ( + project_id TEXT NOT NULL, + sequence INTEGER NOT NULL, + event_type TEXT NOT NULL, + payload_json TEXT NOT NULL, + previous_event_sha256 TEXT, + event_sha256 TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (project_id, sequence) + ); + """ + ) + current = connection.execute("SELECT value FROM metadata WHERE key='schema_version'").fetchone() + if current is None: + connection.execute("INSERT INTO metadata(key, value) VALUES('schema_version', ?)", (str(SCHEMA_VERSION),)) + elif int(current["value"]) != SCHEMA_VERSION: + raise ValueError(f"unsupported project store schema: {current['value']}") + + def _require_project(self, connection: sqlite3.Connection, project_id: str) -> None: + if connection.execute("SELECT 1 FROM projects WHERE project_id=?", (project_id,)).fetchone() is None: + raise ValueError(f"unknown project: {project_id}") + + def _require_run(self, connection: sqlite3.Connection, project_id: str, run_id: str) -> sqlite3.Row: + row = connection.execute("SELECT * FROM runs WHERE project_id=? AND run_id=?", (project_id, run_id)).fetchone() + if row is None: + raise ValueError(f"unknown run: {project_id}/{run_id}") + return row + + def _require_branch(self, connection: sqlite3.Connection, project_id: str, branch_name: str) -> sqlite3.Row: + row = connection.execute("SELECT * FROM branches WHERE project_id=? AND branch_name=?", (project_id, branch_name)).fetchone() + if row is None: + raise ValueError(f"unknown branch: {project_id}/{branch_name}") + return row + + def _append_event(self, connection: sqlite3.Connection, project_id: str, event_type: str, payload: Mapping[str, Any]) -> str: + row = connection.execute("SELECT sequence, event_sha256 FROM events WHERE project_id=? ORDER BY sequence DESC LIMIT 1", (project_id,)).fetchone() + sequence = 1 if row is None else int(row["sequence"]) + 1 + previous = None if row is None else str(row["event_sha256"]) + created_at = _now() + material = { + "project_id": project_id, + "sequence": sequence, + "event_type": event_type, + "payload": dict(payload), + "previous_event_sha256": previous, + "created_at": created_at, + } + digest = _sha256_json(material) + connection.execute( + "INSERT INTO events(project_id, sequence, event_type, payload_json, previous_event_sha256, event_sha256, created_at) VALUES(?,?,?,?,?,?,?)", + (project_id, sequence, event_type, json.dumps(dict(payload), sort_keys=True), previous, digest, created_at), + ) + return digest + + def create_project(self, *, project_id: str, title: str, question: str) -> dict[str, Any]: + self.initialize() + project_id = _text(project_id, "project_id") + title = _text(title, "title") + question = _text(question, "question") + created_at = _now() + with self.connect() as connection: + existing = connection.execute("SELECT * FROM projects WHERE project_id=?", (project_id,)).fetchone() + if existing is not None: + if existing["title"] == title and existing["question"] == question: + return dict(existing) + raise ValueError(f"project already exists with different metadata: {project_id}") + connection.execute( + "INSERT INTO projects(project_id, title, question, created_at, status) VALUES(?,?,?,?,?)", + (project_id, title, question, created_at, "active"), + ) + self._append_event(connection, project_id, "project-created", {"title": title, "question": question}) + return {"project_id": project_id, "title": title, "question": question, "created_at": created_at, "status": "active"} + + def import_run( + self, + *, + project_id: str, + manifest_path: Path, + branch_name: str = "main", + parent_run_id: str | None = None, + ) -> ImportedRun: + self.initialize() + project_id = _text(project_id, "project_id") + branch_name = _text(branch_name, "branch_name") + manifest_path = manifest_path.resolve() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("run manifest must be an object") + validate_bundle(manifest, manifest_path.parent) + run_id = _text(manifest.get("run_id"), "manifest run_id") + digest, _size = stream_sha256(manifest_path) + question = _text(manifest.get("question"), "manifest question") + review_status = str(manifest.get("review", {}).get("status", "unknown")) + imported_at = _now() + with self.connect() as connection: + self._require_project(connection, project_id) + if parent_run_id is not None: + self._require_run(connection, project_id, parent_run_id) + existing = connection.execute("SELECT * FROM runs WHERE project_id=? AND run_id=?", (project_id, run_id)).fetchone() + if existing is not None: + if existing["manifest_sha256"] != digest: + raise ValueError(f"run ID already exists with different manifest bytes: {run_id}") + return ImportedRun( + project_id, run_id, str(existing["branch_name"]), existing["parent_run_id"], + str(existing["manifest_path"]), digest, str(existing["review_status"]), str(existing["imported_at"]), + ) + branch = connection.execute("SELECT * FROM branches WHERE project_id=? AND branch_name=?", (project_id, branch_name)).fetchone() + if branch is not None and parent_run_id is None: + parent_run_id = str(branch["head_run_id"]) + connection.execute( + "INSERT INTO runs(project_id, run_id, branch_name, parent_run_id, manifest_path, manifest_sha256, question, review_status, imported_at) VALUES(?,?,?,?,?,?,?,?,?)", + (project_id, run_id, branch_name, parent_run_id, str(manifest_path), digest, question, review_status, imported_at), + ) + for artifact_path, artifact in sorted(_manifest_artifacts(manifest).items()): + connection.execute( + "INSERT INTO run_artifacts(project_id, run_id, artifact_path, sha256, kind, artifact_type, payload_json) VALUES(?,?,?,?,?,?,?)", + ( + project_id, run_id, artifact_path, _sha(artifact.get("sha256"), "artifact sha256"), + str(artifact.get("kind", "artifact")), str(artifact.get("artifact_type", "file")), + json.dumps(artifact, sort_keys=True), + ), + ) + for claim_id, claim in sorted(_manifest_claims(manifest).items()): + connection.execute( + "INSERT INTO run_claims(project_id, run_id, claim_id, payload_json) VALUES(?,?,?,?)", + (project_id, run_id, claim_id, json.dumps(claim, sort_keys=True)), + ) + if branch is None: + branch_base = parent_run_id or run_id + connection.execute( + "INSERT INTO branches(project_id, branch_name, base_run_id, head_run_id, status, created_at, updated_at) VALUES(?,?,?,?,?,?,?)", + (project_id, branch_name, branch_base, run_id, "active", imported_at, imported_at), + ) + else: + connection.execute( + "UPDATE branches SET head_run_id=?, updated_at=? WHERE project_id=? AND branch_name=?", + (run_id, imported_at, project_id, branch_name), + ) + self._append_event( + connection, + project_id, + "run-imported", + {"run_id": run_id, "branch_name": branch_name, "parent_run_id": parent_run_id, "manifest_sha256": digest, "review_status": review_status}, + ) + return ImportedRun(project_id, run_id, branch_name, parent_run_id, str(manifest_path), digest, review_status, imported_at) + + def fork_branch(self, *, project_id: str, source_run_id: str, branch_name: str) -> dict[str, Any]: + self.initialize() + project_id = _text(project_id, "project_id") + source_run_id = _text(source_run_id, "source_run_id") + branch_name = _text(branch_name, "branch_name") + created_at = _now() + with self.connect() as connection: + self._require_project(connection, project_id) + self._require_run(connection, project_id, source_run_id) + if connection.execute("SELECT 1 FROM branches WHERE project_id=? AND branch_name=?", (project_id, branch_name)).fetchone() is not None: + raise ValueError(f"branch already exists: {branch_name}") + connection.execute( + "INSERT INTO branches(project_id, branch_name, base_run_id, head_run_id, status, created_at, updated_at) VALUES(?,?,?,?,?,?,?)", + (project_id, branch_name, source_run_id, source_run_id, "active", created_at, created_at), + ) + self._append_event(connection, project_id, "branch-forked", {"branch_name": branch_name, "source_run_id": source_run_id}) + return {"project_id": project_id, "branch_name": branch_name, "base_run_id": source_run_id, "head_run_id": source_run_id, "status": "active", "created_at": created_at} + + def add_assertion( + self, + *, + project_id: str, + run_id: str, + claim_id: str, + source_id: str, + polarity: str, + locator: Mapping[str, Any], + independence_group: str, + effect_measure: str | None = None, + estimate: float | None = None, + interval_low: float | None = None, + interval_high: float | None = None, + unit: str | None = None, + sample_size: int | None = None, + population: str | None = None, + risk_of_bias_ref: str | None = None, + ) -> dict[str, Any]: + self.initialize() + for value, label in ((project_id, "project_id"), (run_id, "run_id"), (claim_id, "claim_id"), (source_id, "source_id"), (independence_group, "independence_group")): + _text(value, label) + if polarity not in POLARITIES: + raise ValueError(f"invalid assertion polarity: {polarity}") + locator_payload = _json_object(locator, "locator") + artifact_path = _relative(locator_payload.get("artifact_path"), "locator artifact_path") + artifact_sha = _sha(locator_payload.get("artifact_sha256"), "locator artifact_sha256") + if not any(locator_payload.get(field) not in (None, "") for field in ("page", "table", "figure", "cell", "json_pointer", "line_range", "record_id")): + raise ValueError("assertion locator needs a page, table, figure, cell, JSON pointer, line range, or record ID") + locator_payload["artifact_path"] = artifact_path + locator_payload["artifact_sha256"] = artifact_sha + if sample_size is not None and (isinstance(sample_size, bool) or sample_size < 0): + raise ValueError("sample_size must be a non-negative integer") + if (interval_low is None) != (interval_high is None): + raise ValueError("both interval bounds must be provided together") + if interval_low is not None and interval_high is not None and interval_low > interval_high: + raise ValueError("interval_low cannot exceed interval_high") + created_at = _now() + material = { + "project_id": project_id, + "run_id": run_id, + "claim_id": claim_id, + "source_id": source_id, + "polarity": polarity, + "effect_measure": effect_measure, + "estimate": estimate, + "interval_low": interval_low, + "interval_high": interval_high, + "unit": unit, + "sample_size": sample_size, + "population": population, + "independence_group": independence_group, + "risk_of_bias_ref": risk_of_bias_ref, + "locator": locator_payload, + "created_at": created_at, + } + fingerprint = _sha256_json(material) + assertion_id = f"assertion-{fingerprint[:20]}" + with self.connect() as connection: + self._require_run(connection, project_id, run_id) + if connection.execute("SELECT 1 FROM run_claims WHERE project_id=? AND run_id=? AND claim_id=?", (project_id, run_id, claim_id)).fetchone() is None: + raise ValueError(f"claim is not present in imported run: {claim_id}") + artifact = connection.execute( + "SELECT sha256 FROM run_artifacts WHERE project_id=? AND run_id=? AND artifact_path=?", + (project_id, run_id, artifact_path), + ).fetchone() + if artifact is None or str(artifact["sha256"]) != artifact_sha: + raise ValueError("assertion locator does not match an imported artifact hash") + existing = connection.execute("SELECT fingerprint FROM assertions WHERE project_id=? AND assertion_id=?", (project_id, assertion_id)).fetchone() + if existing is not None: + return {**material, "assertion_id": assertion_id, "fingerprint": fingerprint} + connection.execute( + """INSERT INTO assertions( + project_id, assertion_id, run_id, claim_id, source_id, polarity, + effect_measure, estimate, interval_low, interval_high, unit, + sample_size, population, independence_group, risk_of_bias_ref, + locator_json, created_at, fingerprint + ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + project_id, assertion_id, run_id, claim_id, source_id, polarity, + effect_measure, estimate, interval_low, interval_high, unit, + sample_size, population, independence_group, risk_of_bias_ref, + json.dumps(locator_payload, sort_keys=True), created_at, fingerprint, + ), + ) + self._append_event(connection, project_id, "assertion-added", {"assertion_id": assertion_id, "run_id": run_id, "claim_id": claim_id, "fingerprint": fingerprint}) + return {**material, "assertion_id": assertion_id, "fingerprint": fingerprint} + + def _load_manifest_for_run(self, connection: sqlite3.Connection, project_id: str, run_id: str) -> dict[str, Any]: + row = self._require_run(connection, project_id, run_id) + path = Path(str(row["manifest_path"])) + digest, _size = stream_sha256(path) + if digest != str(row["manifest_sha256"]): + raise ValueError(f"imported run manifest changed on disk: {run_id}") + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"run manifest is not an object: {run_id}") + return payload + + def compare_runs(self, *, project_id: str, previous_run_id: str, current_run_id: str) -> dict[str, Any]: + self.initialize() + with self.connect() as connection: + self._require_project(connection, project_id) + previous = self._load_manifest_for_run(connection, project_id, previous_run_id) + current = self._load_manifest_for_run(connection, project_id, current_run_id) + result = diff_runs(previous, current) + result["project_id"] = project_id + return result + + @staticmethod + def _three_way_conflicts(base: Mapping[str, Any], source: Mapping[str, Any], target: Mapping[str, Any], key: str) -> list[str]: + base_items = {str(item.get(key)): item for item in base.values() if isinstance(item, Mapping)} if isinstance(base, Mapping) else {} + source_items = {str(item.get(key)): item for item in source.values() if isinstance(item, Mapping)} if isinstance(source, Mapping) else {} + target_items = {str(item.get(key)): item for item in target.values() if isinstance(item, Mapping)} if isinstance(target, Mapping) else {} + conflicts = [] + for identifier in sorted(source_items.keys() & target_items.keys()): + source_changed = source_items[identifier] != base_items.get(identifier) + target_changed = target_items[identifier] != base_items.get(identifier) + if source_changed and target_changed and source_items[identifier] != target_items[identifier]: + conflicts.append(identifier) + return conflicts + + def prepare_merge_plan(self, *, project_id: str, source_branch: str, target_branch: str) -> dict[str, Any]: + self.initialize() + project_id = _text(project_id, "project_id") + created_at = _now() + with self.connect() as connection: + source = self._require_branch(connection, project_id, source_branch) + target = self._require_branch(connection, project_id, target_branch) + source_run = str(source["head_run_id"]) + target_run = str(target["head_run_id"]) + base_candidates = {str(source["base_run_id"]), str(target["base_run_id"])} + if str(source["base_run_id"]) == str(target["base_run_id"]): + base_run = str(source["base_run_id"]) + elif str(source["base_run_id"]) == target_run: + base_run = target_run + elif str(target["base_run_id"]) == source_run: + base_run = source_run + else: + # The store does not invent a lowest common ancestor. Choose the + # source fork point and report the ambiguity explicitly. + base_run = str(source["base_run_id"]) + base_manifest = self._load_manifest_for_run(connection, project_id, base_run) + source_manifest = self._load_manifest_for_run(connection, project_id, source_run) + target_manifest = self._load_manifest_for_run(connection, project_id, target_run) + source_diff = diff_runs(base_manifest, source_manifest) + target_diff = diff_runs(base_manifest, target_manifest) + base_claims, source_claims, target_claims = (_manifest_claims(item) for item in (base_manifest, source_manifest, target_manifest)) + base_artifacts, source_artifacts, target_artifacts = (_manifest_artifacts(item) for item in (base_manifest, source_manifest, target_manifest)) + claim_conflicts = sorted( + claim_id for claim_id in source_claims.keys() & target_claims.keys() + if source_claims[claim_id] != base_claims.get(claim_id) + and target_claims[claim_id] != base_claims.get(claim_id) + and source_claims[claim_id] != target_claims[claim_id] + ) + artifact_conflicts = sorted( + artifact_path for artifact_path in source_artifacts.keys() & target_artifacts.keys() + if source_artifacts[artifact_path] != base_artifacts.get(artifact_path) + and target_artifacts[artifact_path] != base_artifacts.get(artifact_path) + and source_artifacts[artifact_path] != target_artifacts[artifact_path] + ) + payload = { + "schema_version": 1, + "project_id": project_id, + "source_branch": source_branch, + "target_branch": target_branch, + "base_run_id": base_run, + "source_run_id": source_run, + "target_run_id": target_run, + "source_diff": source_diff, + "target_diff": target_diff, + "claim_conflicts": claim_conflicts, + "artifact_conflicts": artifact_conflicts, + "base_ambiguity": len(base_candidates) > 1 and base_run not in {source_run, target_run}, + "review_receipts_invalidated": bool( + source_diff.get("review_invalidated") or target_diff.get("review_invalidated") + ), + "requires_scientific_review": True, + "status": "blocked-conflicts" if claim_conflicts or artifact_conflicts else "candidate", + "executed": False, + "created_at": created_at, + "evidence_boundary": "A merge plan compares immutable run records. It does not combine artifact bytes, resolve scientific disagreements, or mark a branch merged.", + } + fingerprint = _sha256_json(payload) + plan_id = f"merge-{fingerprint[:20]}" + connection.execute( + "INSERT OR IGNORE INTO merge_plans(project_id, plan_id, source_branch, target_branch, base_run_id, source_run_id, target_run_id, status, payload_json, created_at, fingerprint) VALUES(?,?,?,?,?,?,?,?,?,?,?)", + (project_id, plan_id, source_branch, target_branch, base_run, source_run, target_run, payload["status"], json.dumps(payload, sort_keys=True), created_at, fingerprint), + ) + self._append_event(connection, project_id, "merge-plan-created", {"plan_id": plan_id, "source_branch": source_branch, "target_branch": target_branch, "fingerprint": fingerprint}) + return {**payload, "plan_id": plan_id, "fingerprint": fingerprint} + + def summary(self, *, project_id: str) -> dict[str, Any]: + self.initialize() + with self.connect() as connection: + project = connection.execute("SELECT * FROM projects WHERE project_id=?", (project_id,)).fetchone() + if project is None: + raise ValueError(f"unknown project: {project_id}") + branches = [dict(row) for row in connection.execute("SELECT * FROM branches WHERE project_id=? ORDER BY branch_name", (project_id,))] + runs = [dict(row) for row in connection.execute("SELECT run_id, branch_name, parent_run_id, manifest_sha256, review_status, imported_at FROM runs WHERE project_id=? ORDER BY imported_at, run_id", (project_id,))] + assertion_count = int(connection.execute("SELECT COUNT(*) AS n FROM assertions WHERE project_id=?", (project_id,)).fetchone()["n"]) + merge_count = int(connection.execute("SELECT COUNT(*) AS n FROM merge_plans WHERE project_id=?", (project_id,)).fetchone()["n"]) + last_event = connection.execute("SELECT sequence, event_sha256, created_at FROM events WHERE project_id=? ORDER BY sequence DESC LIMIT 1", (project_id,)).fetchone() + return { + "schema_version": 1, + "project": dict(project), + "branches": branches, + "runs": runs, + "assertion_count": assertion_count, + "merge_plan_count": merge_count, + "event_chain_head": None if last_event is None else dict(last_event), + "evidence_boundary": "The project store indexes imported immutable run bundles; it is not a substitute for their manifests, reviews, or scientific evidence.", + } diff --git a/src/codex_science/reference_contract.py b/src/codex_science/reference_contract.py new file mode 100644 index 00000000..c55ee62b --- /dev/null +++ b/src/codex_science/reference_contract.py @@ -0,0 +1,368 @@ +"""Progressive-disclosure reference contracts for scientific Agent Skills.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping + +INDEX_NAME = "index.json" +REFERENCE_SECTION_RE = re.compile(r"^##\s+Reference (?:usage|map)\s*$", re.I | re.M) +MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\((references/[^)]+)\)") + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _string_list(value: Any, label: str, *, required: bool = False) -> tuple[str, ...]: + if value is None and not required: + return () + if not isinstance(value, list): + raise ValueError(f"{label} must be a list") + result = tuple(_text(item, f"{label} item") for item in value) + if required and not result: + raise ValueError(f"{label} must be non-empty") + return result + + +@dataclass(frozen=True) +class ReferenceEntry: + id: str + path: str + purpose: str + read_when: tuple[str, ...] + required_before: tuple[str, ...] + search_patterns: tuple[str, ...] = () + authority: str = "repository-contract" + version: str = "1" + evidence_boundary: str = "" + + @classmethod + def from_payload(cls, payload: Mapping[str, Any], *, index: int) -> "ReferenceEntry": + path = _text(payload.get("path"), f"references[{index}].path") + pure = PurePosixPath(path) + if pure.is_absolute() or ".." in pure.parts or len(pure.parts) != 2 or pure.parts[0] != "references": + raise ValueError( + f"references[{index}].path must be one level below references/: {path}" + ) + if pure.name == INDEX_NAME: + raise ValueError("references/index.json cannot index itself") + return cls( + id=_text(payload.get("id"), f"references[{index}].id"), + path=path, + purpose=_text(payload.get("purpose"), f"references[{index}].purpose"), + read_when=_string_list(payload.get("read_when"), f"references[{index}].read_when", required=True), + required_before=_string_list( + payload.get("required_before"), + f"references[{index}].required_before", + required=True, + ), + search_patterns=_string_list( + payload.get("search_patterns"), f"references[{index}].search_patterns" + ), + authority=_text( + payload.get("authority", "repository-contract"), + f"references[{index}].authority", + ), + version=_text(payload.get("version", "1"), f"references[{index}].version"), + evidence_boundary=str(payload.get("evidence_boundary", "")).strip(), + ) + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + for field in ("read_when", "required_before", "search_patterns"): + payload[field] = list(payload[field]) + return payload + + +@dataclass(frozen=True) +class ReferenceIndex: + schema_version: int + skill: str + references: tuple[ReferenceEntry, ...] + default_policy: str = "load-minimum-required" + + @classmethod + def from_payload(cls, payload: Mapping[str, Any]) -> "ReferenceIndex": + if payload.get("schema_version") != 1: + raise ValueError("unsupported reference index schema") + raw = payload.get("references") + if not isinstance(raw, list) or not raw: + raise ValueError("reference index requires a non-empty references list") + references = tuple( + ReferenceEntry.from_payload(item, index=index) + for index, item in enumerate(raw) + if isinstance(item, Mapping) + ) + if len(references) != len(raw): + raise ValueError("each reference index entry must be an object") + ids = [item.id for item in references] + paths = [item.path for item in references] + if len(ids) != len(set(ids)): + raise ValueError("reference IDs must be unique") + if len(paths) != len(set(paths)): + raise ValueError("reference paths must be unique") + return cls( + schema_version=1, + skill=_text(payload.get("skill"), "skill"), + references=references, + default_policy=_text( + payload.get("default_policy", "load-minimum-required"), "default_policy" + ), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "skill": self.skill, + "default_policy": self.default_policy, + "references": [item.to_dict() for item in self.references], + } + + +@dataclass(frozen=True) +class ReferenceUseReceipt: + schema_version: int + skill: str + reference_id: str + path: str + sha256: str + read_reason: str + used_for: tuple[str, ...] + sections: tuple[str, ...] + search_terms: tuple[str, ...] + loaded_at: str + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + for field in ("used_for", "sections", "search_terms"): + payload[field] = list(payload[field]) + return payload + + +def load_reference_index(skill_dir: Path) -> ReferenceIndex | None: + path = skill_dir / "references" / INDEX_NAME + if not path.is_file(): + return None + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("reference index must be a JSON object") + return ReferenceIndex.from_payload(payload) + + +def validate_reference_index(skill_dir: Path, *, strict: bool = False) -> dict[str, Any]: + skill_path = skill_dir / "SKILL.md" + if not skill_path.is_file(): + raise ValueError(f"missing SKILL.md: {skill_dir}") + skill_text = skill_path.read_text(encoding="utf-8") + references_dir = skill_dir / "references" + files = sorted( + path + for path in references_dir.glob("*") + if path.is_file() and path.name != INDEX_NAME and not path.name.startswith(".") + ) if references_dir.is_dir() else [] + index = load_reference_index(skill_dir) + findings: list[dict[str, str]] = [] + if files and index is None: + findings.append({ + "code": "missing-reference-index", + "severity": "major" if strict else "minor", + "message": f"{skill_dir.name} has reference files but no references/index.json", + }) + return {"skill": skill_dir.name, "index": None, "findings": findings, "references": []} + if index is None: + return {"skill": skill_dir.name, "index": None, "findings": [], "references": []} + if index.skill != skill_dir.name: + findings.append({ + "code": "reference-skill-mismatch", + "severity": "major", + "message": f"reference index skill {index.skill} does not match {skill_dir.name}", + }) + if not REFERENCE_SECTION_RE.search(skill_text): + findings.append({ + "code": "missing-reference-usage-section", + "severity": "major", + "message": f"{skill_dir.name} must contain a '## Reference usage' or '## Reference map' section", + }) + linked = set(MARKDOWN_LINK_RE.findall(skill_text)) + indexed_paths = {item.path for item in index.references} + actual_paths = {path.relative_to(skill_dir).as_posix() for path in files} + for path in sorted(indexed_paths - actual_paths): + findings.append({ + "code": "missing-reference-file", + "severity": "major", + "message": f"indexed reference is missing: {path}", + }) + for path in sorted(actual_paths - indexed_paths): + findings.append({ + "code": "unindexed-reference-file", + "severity": "major" if strict else "minor", + "message": f"reference file is not indexed: {path}", + }) + for entry in index.references: + path = skill_dir / entry.path + if path.is_file() and len(path.read_text(encoding="utf-8")) > 12000 and not entry.search_patterns: + findings.append({ + "code": "large-reference-without-search-patterns", + "severity": "minor", + "message": f"large reference needs search_patterns: {entry.path}", + }) + if entry.path not in linked: + findings.append({ + "code": "reference-not-linked-from-skill", + "severity": "major" if strict else "minor", + "message": f"SKILL.md does not link indexed reference: {entry.path}", + }) + return { + "skill": skill_dir.name, + "index": index.to_dict(), + "findings": findings, + "references": [ + { + **entry.to_dict(), + "sha256": _sha256(skill_dir / entry.path) if (skill_dir / entry.path).is_file() else None, + } + for entry in index.references + ], + } + + +def select_references( + index: ReferenceIndex, + *, + route: str = "", + query: str = "", + limit: int | None = None, +) -> list[ReferenceEntry]: + tokens = set(re.findall(r"[a-z0-9_-]+", f"{route} {query}".lower())) + ranked: list[tuple[int, str, ReferenceEntry]] = [] + for entry in index.references: + haystack = " ".join( + [entry.id, entry.purpose, *entry.read_when, *entry.required_before, *entry.search_patterns] + ).lower() + score = sum(3 for token in tokens if token and token in haystack) + if route and any(route.lower() in item.lower() for item in entry.required_before): + score += 20 + if route and any(route.lower() in item.lower() for item in entry.read_when): + score += 10 + if not tokens: + score = 1 + if score > 0: + ranked.append((score, entry.id, entry)) + ranked.sort(key=lambda item: (-item[0], item[1])) + entries = [item[2] for item in ranked] + return entries if limit is None else entries[:limit] + + +def build_reference_use_receipt( + skill_dir: Path, + entry: ReferenceEntry, + *, + read_reason: str, + used_for: Iterable[str], + sections: Iterable[str] = (), + search_terms: Iterable[str] = (), + loaded_at: str | None = None, +) -> ReferenceUseReceipt: + path = skill_dir / entry.path + if not path.is_file(): + raise ValueError(f"reference file is missing: {entry.path}") + return ReferenceUseReceipt( + schema_version=1, + skill=skill_dir.name, + reference_id=entry.id, + path=entry.path, + sha256=_sha256(path), + read_reason=_text(read_reason, "read_reason"), + used_for=tuple(_text(item, "used_for item") for item in used_for), + sections=tuple(str(item).strip() for item in sections if str(item).strip()), + search_terms=tuple(str(item).strip() for item in search_terms if str(item).strip()), + loaded_at=loaded_at or _now(), + ) + + +def audit_reference_roots( + roots: Iterable[Path], + *, + strict_names: Iterable[str] = (), +) -> dict[str, Any]: + strict_set = set(strict_names) + records: list[dict[str, Any]] = [] + for root in roots: + if not root.is_dir(): + continue + for skill_dir in sorted(path for path in root.iterdir() if path.is_dir()): + if not (skill_dir / "SKILL.md").is_file(): + continue + records.append(validate_reference_index(skill_dir, strict=skill_dir.name in strict_set)) + findings = [ + {"skill": record["skill"], **finding} + for record in records + for finding in record["findings"] + ] + return { + "schema_version": 1, + "summary": { + "skills": len(records), + "indexed_skills": sum(record["index"] is not None for record in records), + "references": sum(len(record["references"]) for record in records), + "findings": len(findings), + "major_findings": sum(item["severity"] == "major" for item in findings), + }, + "skills": records, + "findings": findings, + } + + +def validate_reference_use_receipt(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported reference use receipt schema") + for field in ("skill", "reference_id", "path", "sha256", "read_reason", "loaded_at"): + _text(payload.get(field), field) + reference_path = PurePosixPath(str(payload["path"])) + if reference_path.is_absolute() or ".." in reference_path.parts or reference_path.parts[0] != "references": + raise ValueError("reference use receipt path must stay under references/") + digest = str(payload["sha256"]).lower() + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + raise ValueError("reference use receipt sha256 is invalid") + for field in ("used_for", "sections", "search_terms"): + value = payload.get(field, []) + if not isinstance(value, (list, tuple)) or not all(isinstance(item, str) for item in value): + raise ValueError(f"reference use receipt {field} must be strings") + + +def validate_reference_use_ledger(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported reference use ledger schema") + _text(payload.get("skill"), "skill") + uses = payload.get("uses") + if not isinstance(uses, list) or not uses: + raise ValueError("reference use ledger requires a non-empty uses list") + seen: set[tuple[str, str]] = set() + for item in uses: + if not isinstance(item, Mapping): + raise ValueError("reference use ledger entries must be objects") + validate_reference_use_receipt(item) + key = (str(item["reference_id"]), str(item["sha256"])) + if key in seen: + raise ValueError(f"duplicate reference use receipt: {item['reference_id']}") + seen.add(key) diff --git a/src/codex_science/review_receipts.py b/src/codex_science/review_receipts.py index 3a7daf56..cb637896 100644 --- a/src/codex_science/review_receipts.py +++ b/src/codex_science/review_receipts.py @@ -24,7 +24,14 @@ def _sha(value: Any, label: str) -> str: def canonical_sha256(payload: Mapping[str, Any]) -> str: - return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + return hashlib.sha256( + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + ).hexdigest() def validate_review_receipt(payload: dict[str, Any]) -> None: diff --git a/src/codex_science/reviewer_runtime.py b/src/codex_science/reviewer_runtime.py new file mode 100644 index 00000000..0a1642b6 --- /dev/null +++ b/src/codex_science/reviewer_runtime.py @@ -0,0 +1,324 @@ +"""Blinded scientific review packets and hash-covered review response intake. + +The runtime prepares deterministic packets for a separate human or agent. It +never sends data to a model service, claims reviewer independence, or exposes +private chain-of-thought. Reviewer identity and independence remain attestations +that must be governed by the calling system. +""" + +from __future__ import annotations + +import hashlib +import json +import urllib.parse +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from codex_science.artifact_store import stream_sha256 +from codex_science.artifacts import validate_bundle +from codex_science.review_receipts import build_review_receipt, canonical_sha256, validate_review_receipt + + +REVIEW_MODES = {"record", "source", "method", "reproduction"} +SEVERITIES = {"critical", "major", "minor", "suggestion"} +SENSITIVE_FRAGMENTS = {"token", "password", "secret", "credential", "private_key", "api_key", "apikey", "chain_of_thought", "hidden_rationale"} +EXCLUDED_PRODUCER_FIELDS = {"intended_conclusion", "producer_rationale", "chain_of_thought", "private_scratchpad", "hidden_answer", "suspected_bug"} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _fingerprint(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _text(value: Any, label: str) -> str: + text = str(value).strip() + if not text: + raise ValueError(f"{label} is required") + return text + + +def _sha(value: Any, label: str) -> str: + text = _text(value, label).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{label} must be a SHA-256 digest") + return text + + +def _redact(value: Any) -> Any: + if isinstance(value, Mapping): + result: dict[str, Any] = {} + for key, item in value.items(): + key_text = str(key) + normalized = key_text.lower() + if key_text in EXCLUDED_PRODUCER_FIELDS: + continue + if any(fragment in normalized for fragment in SENSITIVE_FRAGMENTS): + result[key_text] = "[REDACTED]" + else: + result[key_text] = _redact(item) + return result + if isinstance(value, list): + return [_redact(item) for item in value] + if isinstance(value, str) and "://" in value: + try: + parsed = urllib.parse.urlsplit(value) + query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) + if query: + query = [ + ( + key, + "[REDACTED]" + if any(fragment in key.lower() for fragment in SENSITIVE_FRAGMENTS) + else item, + ) + for key, item in query + ] + return urllib.parse.urlunsplit( + ( + parsed.scheme, + parsed.netloc, + parsed.path, + urllib.parse.urlencode(query), + parsed.fragment, + ) + ) + except ValueError: + return "[REDACTED-INVALID-URL]" + return value + + +def _material_claim_ids(manifest: Mapping[str, Any], sidecars: Mapping[str, Any]) -> list[str]: + claim_register = sidecars.get("claim_by_id") + if isinstance(claim_register, Mapping) and claim_register: + return sorted(str(key) for key in claim_register) + return sorted(str(item.get("id")) for item in manifest.get("claims", []) if isinstance(item, Mapping) and item.get("id")) + + +def build_review_packet( + manifest_path: Path, + *, + review_modes: list[str], + independent_required: bool = True, + review_questions: list[str] | None = None, + created_at: str | None = None, +) -> dict[str, Any]: + manifest_path = manifest_path.resolve() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("manifest must be an object") + sidecars = validate_bundle(manifest, manifest_path.parent) + modes = sorted({_text(item, "review mode") for item in review_modes}) + unknown = sorted(set(modes) - REVIEW_MODES) + if unknown or not modes: + raise ValueError(f"invalid or empty review modes: {', '.join(unknown)}") + digest, _size = stream_sha256(manifest_path) + artifact_hashes = [ + {"path": str(item["path"]), "sha256": str(item["sha256"]).lower(), "kind": str(item.get("kind", "artifact"))} + for item in manifest.get("artifacts", []) + if isinstance(item, Mapping) + ] + claims = sidecars.get("claim_register", {}).get("claims") if isinstance(sidecars.get("claim_register"), Mapping) else None + if not isinstance(claims, list): + claims = [dict(item) for item in manifest.get("claims", []) if isinstance(item, Mapping)] + graph = sidecars.get("graph_v2") or sidecars.get("evidence_graph") + questions = review_questions or [ + "Does each material claim have direct, correctly attributed evidence?", + "Do the design, controls, splits, metrics, uncertainty, and inference level support the claim?", + "Are duplicated sources, shared cohorts or samples, and model-training overlap represented correctly?", + "Do reported figures, tables, and values agree with the saved artifacts and execution record?", + "Which alternative explanations or counterevidence remain unresolved?" + ] + material = { + "schema_version": 1, + "source_run_id": str(manifest["run_id"]), + "source_manifest_path": str(manifest_path), + "source_manifest_sha256": digest, + "created_at": created_at or _now(), + "review_modes": modes, + "independent_required": bool(independent_required), + "producer_context_excluded": sorted(EXCLUDED_PRODUCER_FIELDS), + "decision_contract": { + "question": str(manifest.get("question", "")), + "plan": _redact(manifest.get("plan", [])), + "inputs": _redact(manifest.get("inputs", [])), + "code": _redact(manifest.get("code", [])), + "executions": _redact(manifest.get("executions", [])), + "environment": _redact(manifest.get("environment", {})), + }, + "claims": _redact(claims), + "material_claim_ids": _material_claim_ids(manifest, sidecars), + "artifacts": artifact_hashes, + "evidence_graph": _redact(graph) if isinstance(graph, Mapping) else None, + "study_table": _redact(sidecars.get("study_table")) if sidecars.get("study_table") else None, + "query_records": _redact(sidecars.get("query_records", [])), + "lane_receipts": _redact(sidecars.get("lane_receipts", [])), + "model_receipts": _redact(sidecars.get("model_receipts_v2", sidecars.get("model_receipts", []))), + "review_questions": [_text(item, "review question") for item in questions], + "response_contract": { + "reviewer": "required", + "independent": "boolean attestation", + "review_modes": modes, + "reviewed_claim_ids": "all material claims unless an explicit blocked reason is supplied", + "reviewed_artifacts": "path and SHA-256 pairs from this packet", + "findings": "stable ID, severity, code, message, evidence, required action, and resolution status", + "limitations": "required" + }, + "evidence_boundary": "This packet exposes recorded run evidence while excluding producer-only rationale fields. It does not authenticate the reviewer, guarantee true independence, or include unrecorded evidence." + } + fingerprint = _fingerprint(material) + return {**material, "review_task_id": f"review-task-{fingerprint[:20]}", "fingerprint": fingerprint} + + +def validate_review_packet(payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("unsupported review packet schema") + for field in ("review_task_id", "source_run_id", "source_manifest_path", "source_manifest_sha256", "created_at", "evidence_boundary"): + _text(payload.get(field), field) + _sha(payload.get("source_manifest_sha256"), "source_manifest_sha256") + modes = payload.get("review_modes") + if not isinstance(modes, list) or not modes or set(modes) - REVIEW_MODES: + raise ValueError("invalid review packet modes") + if not isinstance(payload.get("artifacts"), list) or not isinstance(payload.get("material_claim_ids"), list): + raise ValueError("review packet artifacts and material_claim_ids must be lists") + material = dict(payload) + task_id = str(material.pop("review_task_id")) + fingerprint = str(material.pop("fingerprint", "")).lower() + if _fingerprint(material) != fingerprint or task_id != f"review-task-{fingerprint[:20]}": + raise ValueError("review packet fingerprint or task ID mismatch") + + +def _validate_finding(item: Mapping[str, Any], index: int) -> dict[str, Any]: + finding_id = _text(item.get("id"), f"finding {index} id") + severity = _text(item.get("severity"), f"finding {finding_id} severity") + if severity not in SEVERITIES: + raise ValueError(f"invalid finding severity: {severity}") + code = _text(item.get("code"), f"finding {finding_id} code") + message = _text(item.get("message", item.get("rationale")), f"finding {finding_id} message") + resolution = str(item.get("resolution_status", "open")) + if resolution not in {"open", "resolved", "accepted-risk", "not-applicable"}: + raise ValueError(f"invalid finding resolution status: {resolution}") + evidence = item.get("evidence", []) + if not isinstance(evidence, list): + raise ValueError(f"finding {finding_id} evidence must be a list") + required_action = str(item.get("required_action", "")).strip() + return { + **dict(item), + "id": finding_id, + "severity": severity, + "code": code, + "message": message, + "resolution_status": resolution, + "evidence": evidence, + "required_action": required_action, + } + + +def finalize_review_response(packet: Mapping[str, Any], response: Mapping[str, Any]) -> dict[str, Any]: + validate_review_packet(packet) + if response.get("schema_version", 1) != 1: + raise ValueError("unsupported review response schema") + if response.get("review_task_id") != packet["review_task_id"] or response.get("packet_fingerprint") != packet["fingerprint"]: + raise ValueError("review response covers a different packet") + reviewer = _text(response.get("reviewer"), "reviewer") + independent = response.get("independent") + if not isinstance(independent, bool): + raise ValueError("independent must be a boolean attestation") + response_modes_raw = response.get("review_modes", []) + if not isinstance(response_modes_raw, list): + raise ValueError("response review_modes must be a list") + response_modes = sorted({_text(item, "response review mode") for item in response_modes_raw}) + if not response_modes or set(response_modes) - set(packet["review_modes"]): + raise ValueError("response review modes are empty or exceed the packet") + reviewed_claims_raw = response.get("reviewed_claim_ids", []) + if not isinstance(reviewed_claims_raw, list): + raise ValueError("reviewed_claim_ids must be a list") + reviewed_claims = sorted({_text(item, "reviewed claim ID") for item in reviewed_claims_raw}) + material_claims = set(map(str, packet["material_claim_ids"])) + unknown_claims = sorted(set(reviewed_claims) - material_claims) + if unknown_claims: + raise ValueError(f"response references unknown claims: {', '.join(unknown_claims)}") + blocked_reason = str(response.get("blocked_reason", "")).strip() + missing_claims = sorted(material_claims - set(reviewed_claims)) + if missing_claims and not blocked_reason: + raise ValueError(f"review omits material claims without blocked_reason: {', '.join(missing_claims)}") + packet_artifacts = {str(item["path"]): str(item["sha256"]).lower() for item in packet["artifacts"]} + reviewed_artifacts_raw = response.get("reviewed_artifacts", []) + if not isinstance(reviewed_artifacts_raw, list): + raise ValueError("reviewed_artifacts must be a list") + covered_artifacts: list[dict[str, str]] = [] + for index, item in enumerate(reviewed_artifacts_raw): + if not isinstance(item, Mapping): + raise ValueError(f"reviewed artifact {index} must be an object") + path = _text(item.get("path"), f"reviewed artifact {index} path") + digest = _sha(item.get("sha256"), f"reviewed artifact {path} sha256") + if packet_artifacts.get(path) != digest: + raise ValueError(f"reviewed artifact does not match packet: {path}") + covered_artifacts.append({"path": path, "sha256": digest}) + if not covered_artifacts: + raise ValueError("review response must cover at least one artifact") + findings_raw = response.get("findings", []) + if not isinstance(findings_raw, list): + raise ValueError("findings must be a list") + findings = [_validate_finding(item, index) for index, item in enumerate(findings_raw) if isinstance(item, Mapping)] + if len(findings) != len(findings_raw): + raise ValueError("all findings must be objects") + if packet.get("independent_required") and not independent: + findings.append({ + "id": "F-review-independence", + "severity": "major", + "code": "review-not-independent", + "message": "The review packet required independence but the reviewer attested independent=false.", + "resolution_status": "open", + "evidence": [], + "required_action": "Obtain a separate reviewer or explicitly downgrade the completion claim." + }) + if missing_claims: + findings.append({ + "id": "F-review-coverage", + "severity": "major", + "code": "incomplete-review-coverage", + "message": f"Material claims were not reviewed: {', '.join(missing_claims)}. Blocked reason: {blocked_reason}", + "resolution_status": "open", + "evidence": [], + "required_action": "Resolve the blocker or review the missing claims." + }) + unresolved_blocking = any(item["severity"] in {"critical", "major"} and item["resolution_status"] not in {"resolved", "not-applicable"} for item in findings) + requested_status = str(response.get("status", "findings")) + if requested_status not in {"passed", "findings", "blocked"}: + raise ValueError("invalid review response status") + status = "findings" if unresolved_blocking else requested_status + if status == "passed" and (missing_claims or (packet.get("independent_required") and not independent)): + status = "findings" + limitations = response.get("limitations", []) + if not isinstance(limitations, list) or not all(isinstance(item, str) and item.strip() for item in limitations): + raise ValueError("limitations must be a non-empty string list") + if not limitations: + raise ValueError("review response must state limitations") + receipt = build_review_receipt( + review_id=f"review-{str(packet['review_task_id']).removeprefix('review-task-')}", + reviewer=reviewer, + independent=independent, + review_modes=response_modes, + status=status, + covered_artifacts=covered_artifacts, + covered_claim_ids=reviewed_claims, + findings=findings, + covered_registry_sha256=None if response.get("covered_registry_sha256") is None else _sha(response["covered_registry_sha256"], "covered_registry_sha256"), + limitations=list(limitations), + ) + receipt["review_task_id"] = packet["review_task_id"] + receipt["packet_fingerprint"] = packet["fingerprint"] + # Rebuild the receipt fingerprint after adding packet linkage. + material = dict(receipt) + material.pop("fingerprint", None) + receipt["fingerprint"] = canonical_sha256(material) + validate_review_receipt(receipt) + return receipt diff --git a/src/codex_science/skill_maturity.py b/src/codex_science/skill_maturity.py new file mode 100644 index 00000000..41dfc6d5 --- /dev/null +++ b/src/codex_science/skill_maturity.py @@ -0,0 +1,295 @@ +"""Machine-readable maturity auditing for first-party scientific skills.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping + + +LEVELS = {f"L{index}": index for index in range(5)} +ROLES = {"conductor", "retriever", "executor", "analyzer", "reviewer", "infrastructure"} +REQUIRED_DECISION_SECTIONS = ("## Decision contract", "## Workflow", "## Outputs", "## Boundaries") +CORE_SECTION_ALIASES = { + "codex-science": ("## Research contract and evidence graph", "## Workflow", "## Completion test", "## Boundaries"), + "science-provenance": ("## Run contract", "## Inputs and retrieval", "## Outputs and claims", "## Safety and integrity"), + "science-review": ("## Inputs and review mode", "## Review workflow", "## Receipt and independence", "## Boundary"), + "remote-scientific-compute": ( + "## Decision contract", + "## Execute and monitor", + "## Outputs", + "## Boundaries", + ), +} +FRONTMATTER_RE = re.compile(r"\A---\s*\n(?P.*?)\n---\s*\n", re.DOTALL) +FIELD_RE = re.compile(r"^(?P[A-Za-z0-9_-]+):\s*(?P.+?)\s*$", re.MULTILINE) + + +def _safe_repo_path(root: Path, value: Any, label: str) -> Path: + text = str(value).strip() + pure = PurePosixPath(text) + if not text or pure.is_absolute() or ".." in pure.parts: + raise ValueError(f"{label} must be a safe repository-relative path: {text}") + path = (root / pure.as_posix()).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError(f"{label} escapes repository root") + return path + + +def _frontmatter(text: str) -> dict[str, str]: + match = FRONTMATTER_RE.match(text) + if match is None: + return {} + result: dict[str, str] = {} + for item in FIELD_RE.finditer(match.group("body")): + value = item.group("value").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {chr(34), chr(39)}: + value = value[1:-1] + result[item.group("key")] = value + return result + + +def _load_json(path: Path, label: str) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{label} must be a JSON object: {path}") + return payload + + +def _reference_contract(skill_dir: Path) -> tuple[bool, list[str]]: + index_path = skill_dir / "references" / "index.json" + if not index_path.is_file(): + return False, ["references/index.json is missing"] + try: + payload = _load_json(index_path, "reference index") + except (OSError, ValueError, json.JSONDecodeError) as error: + return False, [str(error)] + entries = payload.get("references") + if not isinstance(entries, list) or not entries: + return False, ["reference index contains no references"] + errors: list[str] = [] + ids: set[str] = set() + paths: set[str] = set() + for index, raw in enumerate(entries): + if not isinstance(raw, Mapping): + errors.append(f"reference {index} is not an object") + continue + reference_id = str(raw.get("id", "")).strip() + relative = str(raw.get("path", "")).strip() + if not reference_id or reference_id in ids: + errors.append(f"reference {index} has missing or duplicate id") + ids.add(reference_id) + if not relative or relative in paths: + errors.append(f"reference {index} has missing or duplicate path") + paths.add(relative) + try: + pure = PurePosixPath(relative) + if pure.is_absolute() or ".." in pure.parts or len(pure.parts) != 2 or pure.parts[0] != "references": + raise ValueError("must be a one-level references path") + target = skill_dir / relative + if not target.is_file(): + errors.append(f"indexed reference is missing: {relative}") + except ValueError as error: + errors.append(f"invalid reference path {relative}: {error}") + if not isinstance(raw.get("read_when"), list) or not raw.get("read_when"): + errors.append(f"reference {reference_id or index} has no read_when routes") + if raw.get("required_before") is None: + errors.append(f"reference {reference_id or index} has no required_before field") + if not str(raw.get("evidence_boundary", "")).strip(): + errors.append(f"reference {reference_id or index} has no evidence_boundary") + return not errors, errors + + +def _path_list(payload: Mapping[str, Any], field: str) -> list[str]: + value = payload.get(field, []) + if not isinstance(value, list) or not all(isinstance(item, str) and item.strip() for item in value): + raise ValueError(f"quality {field} must be a string list") + return [item.strip() for item in value] + + +def validate_quality_declaration(root: Path, skill_dir: Path, payload: Mapping[str, Any]) -> dict[str, Any]: + if payload.get("schema_version") != 1: + raise ValueError("unsupported quality declaration schema") + skill = str(payload.get("skill", "")).strip() + if not skill: + raise ValueError("quality skill is required") + roles = _path_list(payload, "roles") + unknown_roles = sorted(set(roles) - ROLES) + if unknown_roles: + raise ValueError(f"unknown skill roles: {', '.join(unknown_roles)}") + declared = str(payload.get("declared_maturity", "")).strip() + if declared not in LEVELS: + raise ValueError(f"invalid declared maturity: {declared}") + output_schemas = _path_list(payload, "output_schemas") + fixtures = _path_list(payload, "acceptance_fixtures") + seeded_failures = _path_list(payload, "seeded_failures") + tests = _path_list(payload, "test_files") + limitations = _path_list(payload, "limitations") + for path in fixtures + tests: + if not _safe_repo_path(root, path, "quality dependency").exists(): + raise ValueError(f"quality dependency does not exist: {path}") + if LEVELS[declared] >= 3 and not output_schemas: + raise ValueError("L3+ skills require output_schemas") + if LEVELS[declared] >= 4 and (not fixtures or not seeded_failures or not tests): + raise ValueError("L4 skills require acceptance fixtures, seeded failures, and tests") + return { + "schema_version": 1, + "skill": skill, + "roles": roles, + "declared_maturity": declared, + "output_schemas": output_schemas, + "acceptance_fixtures": fixtures, + "seeded_failures": seeded_failures, + "test_files": tests, + "limitations": limitations, + "quality_path": str((skill_dir / "quality.json").relative_to(root)), + } + + +def _sections_for(skill_dir: Path, name: str, text: str) -> tuple[bool, list[str]]: + required = CORE_SECTION_ALIASES.get(name, REQUIRED_DECISION_SECTIONS) + missing = [section for section in required if section not in text] + return not missing, missing + + +def evaluate_skill(root: Path, skill_dir: Path) -> dict[str, Any]: + skill_path = skill_dir / "SKILL.md" + relative = str(skill_dir.relative_to(root)) + record: dict[str, Any] = { + "path": relative, + "name": skill_dir.name, + "computed_maturity": "L0", + "declared_maturity": None, + "roles": [], + "reasons": [], + "output_schemas": [], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [], + "reference_contract": False, + } + if not skill_path.is_file(): + record["reasons"].append("SKILL.md is missing") + return record + text = skill_path.read_text(encoding="utf-8") + metadata = _frontmatter(text) + name = metadata.get("name", "").strip() + description = metadata.get("description", "").strip() + if not name or not description or "# " not in text: + record["reasons"].append("frontmatter name/description or title is missing") + return record + record["name"] = name + computed = 1 + complete_sections, missing_sections = _sections_for(skill_dir, skill_dir.name, text) + if skill_dir.name in {"codex-science", "science-provenance", "science-review"}: + # These three skills jointly define the first-party handoff contract; + # requiring circular self-references would not add evidence. + has_provenance = True + has_review = True + else: + has_provenance = "$science-provenance" in text + has_review = "$science-review" in text + if complete_sections and has_provenance and has_review: + computed = 2 + else: + record["reasons"].extend(f"missing instruction contract: {section}" for section in missing_sections) + if not has_provenance: + record["reasons"].append("science provenance handoff is missing") + if not has_review: + record["reasons"].append("science review handoff is missing") + reference_ok, reference_errors = _reference_contract(skill_dir) + record["reference_contract"] = reference_ok + if not reference_ok and (skill_dir / "references").is_dir(): + record["reasons"].extend(reference_errors) + quality_path = skill_dir / "quality.json" + declaration: dict[str, Any] | None = None + if quality_path.is_file(): + try: + declaration = validate_quality_declaration(root, skill_dir, _load_json(quality_path, "quality declaration")) + except (OSError, ValueError, json.JSONDecodeError) as error: + record["reasons"].append(f"invalid quality declaration: {error}") + if declaration: + record.update({key: declaration[key] for key in ("declared_maturity", "roles", "output_schemas", "acceptance_fixtures", "seeded_failures", "test_files", "limitations")}) + if computed >= 2 and reference_ok and declaration["output_schemas"]: + computed = 3 + if computed >= 3 and declaration["acceptance_fixtures"] and declaration["seeded_failures"] and declaration["test_files"]: + computed = 4 + record["computed_maturity"] = f"L{computed}" + if record["declared_maturity"] and LEVELS[record["computed_maturity"]] < LEVELS[record["declared_maturity"]]: + record["reasons"].append(f"declared {record['declared_maturity']} but computed {record['computed_maturity']}") + return record + + +def load_policy(root: Path, path: Path | None = None) -> dict[str, Any]: + policy_path = path or root / "catalog" / "native-skill-policy.json" + payload = _load_json(policy_path, "native skill policy") + if payload.get("schema_version") != 1 or not isinstance(payload.get("requirements"), list): + raise ValueError("unsupported native skill policy") + return payload + + +def audit_native_skills(root: Path, *, policy_path: Path | None = None) -> dict[str, Any]: + root = root.resolve() + records = [] + for parent in (root / "skills", root / "authored-skills"): + if not parent.is_dir(): + continue + for skill_dir in sorted(path for path in parent.iterdir() if path.is_dir() and (path / "SKILL.md").is_file()): + records.append(evaluate_skill(root, skill_dir)) + by_name = {record["name"]: record for record in records} + policy = load_policy(root, policy_path) + findings: list[dict[str, str]] = [] + for requirement in policy["requirements"]: + if not isinstance(requirement, Mapping): + raise ValueError("native skill policy requirements must be objects") + name = str(requirement.get("skill", "")) + minimum = str(requirement.get("minimum_maturity", "")) + if minimum not in LEVELS: + raise ValueError(f"invalid policy maturity: {minimum}") + record = by_name.get(name) + if record is None: + findings.append({"code": "missing-policy-skill", "severity": "major", "skill": name, "message": f"Policy skill is missing: {name}"}) + continue + if LEVELS[record["computed_maturity"]] < LEVELS[minimum]: + findings.append({"code": "skill-below-maturity", "severity": "major", "skill": name, "message": f"{name} computed {record['computed_maturity']} below required {minimum}."}) + required_roles = set(requirement.get("required_roles", [])) + missing_roles = sorted(required_roles - set(record["roles"])) + if missing_roles: + findings.append({"code": "missing-skill-role", "severity": "major", "skill": name, "message": f"{name} is missing roles: {', '.join(missing_roles)}"}) + if bool(requirement.get("require_reference_contract", False)) and not record["reference_contract"]: + findings.append({"code": "missing-reference-contract", "severity": "major", "skill": name, "message": f"{name} has no valid progressive reference contract."}) + summary = {level: sum(record["computed_maturity"] == level for record in records) for level in LEVELS} + return { + "schema_version": 1, + "skill_count": len(records), + "summary": summary, + "status": "findings" if findings else "passed", + "findings": sorted(findings, key=lambda item: (item["code"], item["skill"])), + "skills": records, + "policy": policy, + } + + +def render_maturity_markdown(report: Mapping[str, Any]) -> str: + lines = [ + "# Native skill maturity", + "", + f"Status: **{report['status']}** · skills: **{report['skill_count']}**", + "", + "| Skill | Path | Computed | Declared | Roles | References | Findings |", + "| --- | --- | ---: | ---: | --- | --- | --- |", + ] + for record in report["skills"]: + reasons = "; ".join(record["reasons"]).replace("|", "\\|") + lines.append( + f"| {record['name']} | `{record['path']}` | {record['computed_maturity']} | {record['declared_maturity'] or ''} | " + f"{', '.join(record['roles'])} | {'yes' if record['reference_contract'] else 'no'} | {reasons} |" + ) + if report["findings"]: + lines.extend(["", "## Policy findings", ""]) + lines.extend(f"- **{item['code']}** `{item['skill']}` — {item['message']}" for item in report["findings"]) + lines.append("") + return "\n".join(lines) diff --git a/src/codex_science/source_operations_v3.py b/src/codex_science/source_operations_v3.py new file mode 100644 index 00000000..d211dccf --- /dev/null +++ b/src/codex_science/source_operations_v3.py @@ -0,0 +1,239 @@ +"""Source-specific true-pagination operations for Connector Contract v3.""" +from __future__ import annotations + +import json +import urllib.parse +from dataclasses import dataclass +from typing import Any, Mapping + +from codex_science.connector_contract import QueryRequest +from codex_science.transport_v3 import PageRequest + + +def _query(request: QueryRequest) -> str: + value = request.parameters.get("query") + if not isinstance(value, str) or not value.strip(): + raise ValueError("parameters.query is required") + return value.strip() + + +def _integer(value: Any, label: str, *, minimum: int = 0) -> int: + result = int(value) + if result < minimum: + raise ValueError(f"{label} must be >= {minimum}") + return result + + +@dataclass(frozen=True) +class PubMedSearchV3: + source: str = "pubmed" + operation: str = "search" + + def first_request(self, request: QueryRequest) -> PageRequest: + params: dict[str, Any] = { + "db": "pubmed", + "term": _query(request), + "retmode": "json", + "retstart": 0, + "retmax": request.page_size, + "usehistory": "y", + } + if request.evidence_cutoff: + params.update({"datetype": "pdat", "maxdate": request.evidence_cutoff.replace("-", "/")}) + for key in ("mindate", "maxdate", "datetype", "sort"): + if request.parameters.get(key) not in (None, ""): + params[key] = request.parameters[key] + url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?" + urllib.parse.urlencode(params) + return PageRequest("GET", url, {}, cursor="0") + + def next_request(self, request: QueryRequest, page_request: PageRequest, payload: Any, page_index: int) -> PageRequest | None: + result = payload.get("esearchresult") + if not isinstance(result, dict): + raise ValueError("missing esearchresult") + count = _integer(result.get("count", 0), "count") + current = _integer(result.get("retstart", page_index * request.page_size), "retstart") + returned = len(result.get("idlist") or []) + next_start = current + returned + if returned == 0 or next_start >= count: + return None + parsed = urllib.parse.urlsplit(page_request.url) + params = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + params["retstart"] = str(next_start) + url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urllib.parse.urlencode(params), parsed.fragment)) + return PageRequest("GET", url, {}, cursor=str(next_start)) + + def normalize(self, payload: Any) -> list[dict[str, Any]]: + result = payload.get("esearchresult") + if not isinstance(result, dict): + raise ValueError("missing esearchresult") + return [ + {"id": str(identifier), "title": "", "url": f"https://pubmed.ncbi.nlm.nih.gov/{identifier}/"} + for identifier in result.get("idlist", []) + ] + + def source_release(self, payload: Any, headers: Mapping[str, str]) -> str | None: + return headers.get("last-modified") or headers.get("date") + + +@dataclass(frozen=True) +class EuropePMCSearchV3: + source: str = "europepmc" + operation: str = "search" + + def first_request(self, request: QueryRequest) -> PageRequest: + query = _query(request) + if request.evidence_cutoff: + query = f"({query}) AND FIRST_PDATE:[* TO {request.evidence_cutoff}]" + params = { + "query": query, + "pageSize": request.page_size, + "format": "json", + "cursorMark": "*", + "resultType": str(request.parameters.get("result_type", "core")), + } + url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search?" + urllib.parse.urlencode(params) + return PageRequest("GET", url, {}, cursor="*") + + def next_request(self, request: QueryRequest, page_request: PageRequest, payload: Any, page_index: int) -> PageRequest | None: + cursor = payload.get("nextCursorMark") + returned = payload.get("resultList", {}).get("result", []) if isinstance(payload.get("resultList"), dict) else [] + if not isinstance(cursor, str) or not cursor or not returned or cursor == page_request.cursor: + return None + parsed = urllib.parse.urlsplit(page_request.url) + params = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + params["cursorMark"] = cursor + url = urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urllib.parse.urlencode(params), parsed.fragment)) + return PageRequest("GET", url, {}, cursor=cursor) + + def normalize(self, payload: Any) -> list[dict[str, Any]]: + result_list = payload.get("resultList") + if not isinstance(result_list, dict): + raise ValueError("missing resultList") + records = [] + for item in result_list.get("result", []): + identifier = str(item.get("id") or "") + source = str(item.get("source") or "") + if identifier: + records.append({ + "id": identifier, + "title": str(item.get("title") or identifier), + "source": source, + "doi": str(item.get("doi") or ""), + "publication_date": str(item.get("firstPublicationDate") or ""), + "url": f"https://europepmc.org/article/{source}/{identifier}", + }) + return records + + def source_release(self, payload: Any, headers: Mapping[str, str]) -> str | None: + return headers.get("last-modified") or headers.get("date") + + +@dataclass(frozen=True) +class ChEMBLMoleculeSearchV3: + source: str = "chembl" + operation: str = "search" + + def first_request(self, request: QueryRequest) -> PageRequest: + params = {"q": _query(request), "limit": request.page_size, "offset": 0} + url = "https://www.ebi.ac.uk/chembl/api/data/molecule/search.json?" + urllib.parse.urlencode(params) + return PageRequest("GET", url, {}, cursor="0") + + def next_request(self, request: QueryRequest, page_request: PageRequest, payload: Any, page_index: int) -> PageRequest | None: + page_meta = payload.get("page_meta") + if not isinstance(page_meta, dict): + raise ValueError("missing page_meta") + next_url = page_meta.get("next") + if not isinstance(next_url, str) or not next_url: + return None + if next_url.startswith("/"): + next_url = "https://www.ebi.ac.uk" + next_url + parsed = urllib.parse.urlsplit(next_url) + cursor = dict(urllib.parse.parse_qsl(parsed.query)).get("offset") + return PageRequest("GET", next_url, {}, cursor=cursor) + + def normalize(self, payload: Any) -> list[dict[str, Any]]: + molecules = payload.get("molecules") + if not isinstance(molecules, list): + raise ValueError("missing molecules") + records = [] + for item in molecules: + identifier = str(item.get("molecule_chembl_id") or "") + if not identifier: + continue + structures = item.get("molecule_structures") or {} + records.append({ + "id": identifier, + "title": str(item.get("pref_name") or identifier), + "canonical_smiles": str(structures.get("canonical_smiles") or ""), + "url": f"https://www.ebi.ac.uk/chembl/explore/compound/{identifier}", + }) + return records + + def source_release(self, payload: Any, headers: Mapping[str, str]) -> str | None: + return headers.get("last-modified") or headers.get("date") + + +@dataclass(frozen=True) +class PDBSearchV3: + source: str = "pdb" + operation: str = "search" + + def _body(self, request: QueryRequest, start: int) -> bytes: + payload = { + "query": {"type": "terminal", "service": "full_text", "parameters": {"value": _query(request)}}, + "return_type": "entry", + "request_options": {"paginate": {"start": start, "rows": request.page_size}}, + } + return json.dumps(payload, separators=(",", ":")).encode("utf-8") + + def first_request(self, request: QueryRequest) -> PageRequest: + return PageRequest( + "POST", + "https://search.rcsb.org/rcsbsearch/v2/query", + {"Content-Type": "application/json"}, + body=self._body(request, 0), + cursor="0", + ) + + def next_request(self, request: QueryRequest, page_request: PageRequest, payload: Any, page_index: int) -> PageRequest | None: + result_set = payload.get("result_set") + if not isinstance(result_set, list): + raise ValueError("missing result_set") + total = _integer(payload.get("total_count", len(result_set)), "total_count") + start = _integer(page_request.cursor or 0, "cursor") + next_start = start + len(result_set) + if not result_set or next_start >= total: + return None + return PageRequest( + "POST", + page_request.url, + {"Content-Type": "application/json"}, + body=self._body(request, next_start), + cursor=str(next_start), + ) + + def normalize(self, payload: Any) -> list[dict[str, Any]]: + result_set = payload.get("result_set") + if not isinstance(result_set, list): + raise ValueError("missing result_set") + return [ + { + "id": str(item["identifier"]), + "title": "", + "score": item.get("score"), + "url": f"https://www.rcsb.org/structure/{item['identifier']}", + } + for item in result_set + if item.get("identifier") + ] + + def source_release(self, payload: Any, headers: Mapping[str, str]) -> str | None: + return headers.get("last-modified") or headers.get("date") + + +SOURCE_OPERATIONS_V3 = { + "pubmed": PubMedSearchV3(), + "europepmc": EuropePMCSearchV3(), + "chembl": ChEMBLMoleculeSearchV3(), + "pdb": PDBSearchV3(), +} diff --git a/src/codex_science/transport_v3.py b/src/codex_science/transport_v3.py new file mode 100644 index 00000000..5b6a1c58 --- /dev/null +++ b/src/codex_science/transport_v3.py @@ -0,0 +1,426 @@ +"""Bounded HTTP transport, true pagination, and replay receipts for public science sources.""" +from __future__ import annotations + +import hashlib +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Protocol + +from codex_science.connector_contract import QueryRequest, sha256_json + +SECRET_KEYS = {"authorization", "proxy-authorization", "x-api-key", "api-key", "token", "key"} +TERMINAL_STATUSES = { + "complete", + "bounded-by-user", + "negative", + "filtered-empty", + "partial-next-cursor", + "partial-rate-limit", + "source-unavailable", + "authentication-required", + "schema-drift", + "normalization-failed", +} + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _redact_url(url: str) -> str: + parsed = urllib.parse.urlsplit(url) + pairs = [] + for key, value in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True): + pairs.append((key, "[REDACTED]" if key.lower() in SECRET_KEYS else value)) + return urllib.parse.urlunsplit( + (parsed.scheme, parsed.netloc, parsed.path, urllib.parse.urlencode(pairs), parsed.fragment) + ) + + +def _safe_headers(headers: Mapping[str, str]) -> dict[str, str]: + return { + str(key).lower(): "[REDACTED]" if str(key).lower() in SECRET_KEYS else str(value) + for key, value in headers.items() + } + + +@dataclass(frozen=True) +class TransportPolicy: + timeout_seconds: float = 30.0 + max_attempts: int = 3 + backoff_seconds: tuple[float, ...] = (0.0, 0.5, 1.5) + max_response_bytes: int = 50 * 1024 * 1024 + user_agent: str = "codex-science/0.4 (+https://github.com/eightmm/codex-science)" + + def validate(self) -> None: + if self.timeout_seconds <= 0 or self.max_attempts < 1 or self.max_response_bytes < 1: + raise ValueError("transport policy values must be positive") + if len(self.backoff_seconds) < self.max_attempts: + raise ValueError("backoff_seconds must cover every attempt") + + +@dataclass(frozen=True) +class PageRequest: + method: str + url: str + headers: dict[str, str] + body: bytes | None = None + cursor: str | None = None + + @property + def request_sha256(self) -> str: + material = { + "method": self.method.upper(), + "url": _redact_url(self.url), + "headers": _safe_headers(self.headers), + "body_sha256": None if self.body is None else hashlib.sha256(self.body).hexdigest(), + "cursor": self.cursor, + } + return sha256_json(material) + + +@dataclass(frozen=True) +class TransportResponse: + status_code: int + url: str + headers: dict[str, str] + body: bytes + media_type: str + retrieved_at: str + attempts: int + + @property + def body_sha256(self) -> str: + return hashlib.sha256(self.body).hexdigest() + + def json(self) -> Any: + return json.loads(self.body.decode("utf-8")) + + +class Transport(Protocol): + def send(self, request: PageRequest) -> TransportResponse: ... + + +class TransportError(RuntimeError): + def __init__(self, message: str, *, status_code: int | None = None, retryable: bool = False) -> None: + super().__init__(message) + self.status_code = status_code + self.retryable = retryable + + +class UrllibTransport: + def __init__(self, policy: TransportPolicy | None = None) -> None: + self.policy = policy or TransportPolicy() + self.policy.validate() + + def send(self, request: PageRequest) -> TransportResponse: + last_error: BaseException | None = None + for attempt in range(1, self.policy.max_attempts + 1): + delay = self.policy.backoff_seconds[attempt - 1] + if delay: + time.sleep(delay) + headers = {"User-Agent": self.policy.user_agent, "Accept": "application/json", **request.headers} + urllib_request = urllib.request.Request( + request.url, + data=request.body, + headers=headers, + method=request.method.upper(), + ) + try: + with urllib.request.urlopen(urllib_request, timeout=self.policy.timeout_seconds) as response: + body = response.read(self.policy.max_response_bytes + 1) + if len(body) > self.policy.max_response_bytes: + raise TransportError("response exceeded max_response_bytes") + response_headers = {str(key).lower(): str(value) for key, value in response.headers.items()} + media_type = response_headers.get("content-type", "application/octet-stream").split(";", 1)[0] + return TransportResponse( + status_code=int(response.status), + url=_redact_url(response.geturl()), + headers=_safe_headers(response_headers), + body=body, + media_type=media_type, + retrieved_at=_now(), + attempts=attempt, + ) + except urllib.error.HTTPError as error: + last_error = error + retryable = error.code == 429 or 500 <= error.code < 600 + if retryable and attempt < self.policy.max_attempts: + continue + raise TransportError( + f"HTTP {error.code} for {_redact_url(request.url)}", + status_code=error.code, + retryable=retryable, + ) from error + except (urllib.error.URLError, TimeoutError, OSError) as error: + last_error = error + if attempt < self.policy.max_attempts: + continue + raise TransportError( + f"transport failed for {_redact_url(request.url)}: {error}", + retryable=True, + ) from error + raise TransportError(f"transport failed: {last_error}", retryable=True) + + +class SourceOperation(Protocol): + source: str + operation: str + + def first_request(self, request: QueryRequest) -> PageRequest: ... + def next_request( + self, + request: QueryRequest, + page_request: PageRequest, + payload: Any, + page_index: int, + ) -> PageRequest | None: ... + def normalize(self, payload: Any) -> list[dict[str, Any]]: ... + def source_release(self, payload: Any, headers: Mapping[str, str]) -> str | None: ... + + +@dataclass(frozen=True) +class PageReceiptV3: + page_index: int + request_sha256: str + method: str + request_url: str + request_body_sha256: str | None + cursor_in: str | None + cursor_out: str | None + status_code: int + response_sha256: str + response_bytes: int + media_type: str + etag: str | None + last_modified: str | None + rate_limit_limit: str | None + rate_limit_remaining: str | None + rate_limit_reset: str | None + retrieved_at: str + attempts: int + normalized_record_count: int + snapshot_body_path: str | None + snapshot_metadata_path: str | None + + +@dataclass(frozen=True) +class QueryReceiptV3: + schema_version: int + query_id: str + canonical_request_sha256: str + source: str + operation: str + source_contract_version: str + source_release: str | None + status: str + completeness: str + requested_max_pages: int + fetched_pages: int + records_returned: int + returned_record_ids: tuple[str, ...] + normalized_records_sha256: str + missingness: str + warnings: tuple[str, ...] + pages: tuple[PageReceiptV3, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class QueryResultV3: + request: QueryRequest + records: tuple[dict[str, Any], ...] + receipt: QueryReceiptV3 + + def to_dict(self) -> dict[str, Any]: + return {"records": [dict(item) for item in self.records], "receipt": self.receipt.to_dict()} + + +def _snapshot_page( + snapshot_dir: Path | None, + *, + query_id: str, + page_index: int, + page_request: PageRequest, + response: TransportResponse, +) -> tuple[str | None, str | None]: + if snapshot_dir is None: + return None, None + query_dir = snapshot_dir / query_id + query_dir.mkdir(parents=True, exist_ok=True) + body_name = f"page-{page_index:04d}.body" + meta_name = f"page-{page_index:04d}.json" + body_path = query_dir / body_name + meta_path = query_dir / meta_name + body_path.write_bytes(response.body) + metadata = { + "schema_version": 1, + "request": { + "method": page_request.method, + "url": _redact_url(page_request.url), + "headers": _safe_headers(page_request.headers), + "body_sha256": None if page_request.body is None else hashlib.sha256(page_request.body).hexdigest(), + "cursor": page_request.cursor, + "request_sha256": page_request.request_sha256, + }, + "response": { + "status_code": response.status_code, + "url": response.url, + "headers": response.headers, + "body_sha256": response.body_sha256, + "body_bytes": len(response.body), + "media_type": response.media_type, + "retrieved_at": response.retrieved_at, + "attempts": response.attempts, + "body_path": body_name, + }, + } + meta_path.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return (body_path.relative_to(snapshot_dir).as_posix(), meta_path.relative_to(snapshot_dir).as_posix()) + + +def _status_from_error(error: TransportError) -> str: + if error.status_code in {401, 403}: + return "authentication-required" + if error.status_code == 429: + return "partial-rate-limit" + return "source-unavailable" + + +def execute_paginated( + operation: SourceOperation, + request: QueryRequest, + *, + transport: Transport | None = None, + snapshot_dir: Path | None = None, +) -> QueryResultV3: + request.validate() + if request.source != operation.source or request.operation != operation.operation: + raise ValueError( + f"operation mismatch: request={request.source}.{request.operation}, " + f"handler={operation.source}.{operation.operation}" + ) + transport = transport or UrllibTransport() + page_request = operation.first_request(request) + records: list[dict[str, Any]] = [] + receipts: list[PageReceiptV3] = [] + warnings: list[str] = [] + source_release: str | None = None + next_exists = False + terminal_status = "complete" + + for page_index in range(request.max_pages): + try: + response = transport.send(page_request) + except TransportError as error: + terminal_status = _status_from_error(error) + warnings.append(str(error)) + break + try: + payload = response.json() + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"schema-drift: source returned invalid JSON on page {page_index}") from error + try: + page_records = operation.normalize(payload) + except (KeyError, TypeError, ValueError) as error: + raise ValueError(f"normalization-failed on page {page_index}: {error}") from error + if not isinstance(page_records, list) or not all(isinstance(item, dict) for item in page_records): + raise ValueError("normalization-failed: operation must return record objects") + next_request = operation.next_request(request, page_request, payload, page_index) + next_exists = next_request is not None + source_release = source_release or operation.source_release(payload, response.headers) + body_path, metadata_path = _snapshot_page( + snapshot_dir, + query_id=request.query_id, + page_index=page_index, + page_request=page_request, + response=response, + ) + receipts.append( + PageReceiptV3( + page_index=page_index, + request_sha256=page_request.request_sha256, + method=page_request.method.upper(), + request_url=_redact_url(page_request.url), + request_body_sha256=None if page_request.body is None else hashlib.sha256(page_request.body).hexdigest(), + cursor_in=page_request.cursor, + cursor_out=None if next_request is None else next_request.cursor, + status_code=response.status_code, + response_sha256=response.body_sha256, + response_bytes=len(response.body), + media_type=response.media_type, + etag=response.headers.get("etag"), + last_modified=response.headers.get("last-modified"), + rate_limit_limit=response.headers.get("x-ratelimit-limit"), + rate_limit_remaining=response.headers.get("x-ratelimit-remaining"), + rate_limit_reset=response.headers.get("x-ratelimit-reset"), + retrieved_at=response.retrieved_at, + attempts=response.attempts, + normalized_record_count=len(page_records), + snapshot_body_path=body_path, + snapshot_metadata_path=metadata_path, + ) + ) + records.extend(dict(item) for item in page_records) + if next_request is None: + next_exists = False + break + page_request = next_request + + if terminal_status == "complete": + if next_exists and len(receipts) >= request.max_pages: + terminal_status = "bounded-by-user" + elif not records: + terminal_status = "negative" + completeness = terminal_status + missingness = "negative" if not records and terminal_status in {"complete", "negative"} else ( + "unavailable" if terminal_status in {"source-unavailable", "authentication-required"} else + "partial" if terminal_status.startswith("partial") or terminal_status == "bounded-by-user" else + "observed" + ) + record_ids = tuple(str(item.get("id") or f"record-{index}") for index, item in enumerate(records)) + receipt = QueryReceiptV3( + schema_version=3, + query_id=request.query_id, + canonical_request_sha256=request.request_sha256, + source=request.source, + operation=request.operation, + source_contract_version="3", + source_release=source_release, + status=terminal_status, + completeness=completeness, + requested_max_pages=request.max_pages, + fetched_pages=len(receipts), + records_returned=len(records), + returned_record_ids=record_ids, + normalized_records_sha256=sha256_json(records), + missingness=missingness, + warnings=tuple(warnings), + pages=tuple(receipts), + ) + return QueryResultV3(request, tuple(records), receipt) + + +def replay_snapshot_directory(snapshot_dir: Path, query_id: str) -> list[dict[str, Any]]: + query_dir = snapshot_dir / query_id + if not query_dir.is_dir(): + raise ValueError(f"snapshot query directory not found: {query_id}") + pages: list[dict[str, Any]] = [] + for metadata_path in sorted(query_dir.glob("page-*.json")): + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + body_path = query_dir / metadata["response"]["body_path"] + body = body_path.read_bytes() + if hashlib.sha256(body).hexdigest() != metadata["response"]["body_sha256"]: + raise ValueError(f"snapshot body digest mismatch: {body_path.name}") + pages.append({"metadata": metadata, "payload": json.loads(body.decode("utf-8"))}) + if not pages: + raise ValueError(f"no snapshot pages found for query: {query_id}") + return pages diff --git a/src/codex_science/version.py b/src/codex_science/version.py index 28d0762a..4d59d756 100644 --- a/src/codex_science/version.py +++ b/src/codex_science/version.py @@ -1,8 +1,8 @@ """Single source of truth for Codex Science release identities.""" from __future__ import annotations -PACKAGE_VERSION = "0.3.0" -PLUGIN_VERSION = "0.3.0+codex.20260720000100" +PACKAGE_VERSION = "0.4.0" +PLUGIN_VERSION = "0.4.0+codex.20260721000100" MCP_VERSION = PACKAGE_VERSION RELEASE_SCHEMA_VERSION = 1 RUNTIME_AFFECTING_PREFIXES = ( diff --git a/tests/test_action_connectors.py b/tests/test_action_connectors.py new file mode 100644 index 00000000..24c0c4cb --- /dev/null +++ b/tests/test_action_connectors.py @@ -0,0 +1,125 @@ +import copy +import tempfile +import unittest +from pathlib import Path + +from codex_science.action_connectors import ( + ActionLedger, + ActionSpec, + build_action_approval, + build_preview, + execute_action, + validate_action_approval, + validate_preview, +) + + +class FakeAdapter: + connector_name = "fake-lab" + + def __init__(self) -> None: + self.state = {"object_id": "sample-1", "status": "draft", "version": 1} + self.execute_count = 0 + + def snapshot(self, spec: ActionSpec): + return copy.deepcopy(self.state) + + def preview(self, spec: ActionSpec, before): + return {"operation": spec.operation, "changes": {"status": spec.parameters["status"]}, "before_version": before["version"]} + + def execute(self, spec: ActionSpec, preview): + self.execute_count += 1 + self.state = {**self.state, "status": spec.parameters["status"], "version": self.state["version"] + 1} + return {"after_state": copy.deepcopy(self.state), "remote_object_ids": [self.state["object_id"]], "provider_receipt": {"request_id": "provider-1"}} + + +def write_spec() -> ActionSpec: + return ActionSpec.from_payload( + { + "schema_version": 1, + "connector": "fake-lab", + "operation": "publish-sample", + "mode": "write", + "target": "workspace-1/sample-1", + "parameters": {"status": "published"}, + "requested_scopes": ["sample:read", "sample:write"], + "idempotency_key": "publish-sample-1-v1", + "approval_required": True, + "destructive": False, + "paid": False, + "sensitive_data": False, + "compensation": "Set status back to draft through a separately approved action." + } + ) + + +class ActionConnectorTests(unittest.TestCase): + def test_write_preview_approval_execution_and_idempotency(self) -> None: + adapter = FakeAdapter() + spec = write_spec() + preview = build_preview(adapter, spec, created_at="2026-07-21T00:00:00Z") + validate_preview(preview, spec) + self.assertFalse(preview["executed"]) + approval = build_action_approval( + preview, + approved_by="jaemin", + approved_scopes=["sample:read", "sample:write"], + approved_at="2026-07-21T00:01:00Z", + ) + validate_action_approval(approval, preview) + with tempfile.TemporaryDirectory() as tempdir: + ledger = ActionLedger(Path(tempdir) / "actions.sqlite") + receipt = execute_action( + adapter, + spec, + preview, + approval=approval, + ledger=ledger, + executed_at="2026-07-21T00:02:00Z", + ) + self.assertEqual("executed", receipt["status"]) + self.assertNotEqual(receipt["before_state_sha256"], receipt["after_state_sha256"]) + self.assertEqual(["sample-1"], receipt["remote_object_ids"]) + repeated = execute_action(adapter, spec, preview, approval=approval, ledger=ledger) + self.assertEqual(receipt, repeated) + self.assertEqual(1, adapter.execute_count) + + def test_write_requires_exact_approval_and_unchanged_before_state(self) -> None: + adapter = FakeAdapter() + spec = write_spec() + preview = build_preview(adapter, spec, created_at="2026-07-21T00:00:00Z") + with tempfile.TemporaryDirectory() as tempdir: + ledger = ActionLedger(Path(tempdir) / "actions.sqlite") + with self.assertRaisesRegex(ValueError, "requires explicit approval"): + execute_action(adapter, spec, preview, approval=None, ledger=ledger) + approval = build_action_approval(preview, approved_by="jaemin", approved_scopes=["sample:read", "sample:write"]) + adapter.state["version"] = 2 + with self.assertRaisesRegex(ValueError, "changed after preview"): + execute_action(adapter, spec, preview, approval=approval, ledger=ledger) + + def test_approval_scope_and_idempotency_key_cannot_be_reused_for_other_spec(self) -> None: + adapter = FakeAdapter() + spec = write_spec() + preview = build_preview(adapter, spec) + with self.assertRaisesRegex(ValueError, "do not cover"): + build_action_approval(preview, approved_by="jaemin", approved_scopes=["sample:read"]) + approval = build_action_approval(preview, approved_by="jaemin", approved_scopes=["sample:read", "sample:write"]) + with tempfile.TemporaryDirectory() as tempdir: + ledger = ActionLedger(Path(tempdir) / "actions.sqlite") + execute_action(adapter, spec, preview, approval=approval, ledger=ledger) + changed = ActionSpec.from_payload({**spec.to_dict(), "parameters": {"status": "archived"}}) + changed_preview = build_preview(adapter, changed) + changed_approval = build_action_approval(changed_preview, approved_by="jaemin", approved_scopes=["sample:read", "sample:write"]) + with self.assertRaisesRegex(ValueError, "different spec"): + execute_action(adapter, changed, changed_preview, approval=changed_approval, ledger=ledger) + + def test_credentials_are_out_of_band_and_write_cannot_disable_approval(self) -> None: + base = write_spec().to_dict() + with self.assertRaisesRegex(ValueError, "credential-like"): + ActionSpec.from_payload({**base, "parameters": {"api_token": "secret"}}) + with self.assertRaisesRegex(ValueError, "must require approval"): + ActionSpec.from_payload({**base, "approval_required": False}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_artifact_runtime.py b/tests/test_artifact_runtime.py new file mode 100644 index 00000000..8f4fb9b2 --- /dev/null +++ b/tests/test_artifact_runtime.py @@ -0,0 +1,121 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifact_runtime import ( + build_selection, + build_transform_proposal, + describe_runtime, + render_runtime_html, + stale_selection, + validate_runtime_descriptor, + validate_selection, + validate_transform_proposal, +) + + +class ArtifactRuntimeTests(unittest.TestCase): + def test_structure_descriptor_selection_and_proposal_are_hash_bound(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + structure = root / "receptor.pdb" + structure.write_text( + "ATOM 1 N GLY A 145 11.104 13.207 9.101 1.00 20.00 N \n" + "ATOM 2 CA GLY A 145 12.101 12.101 9.500 1.00 20.00 C \n" + "HETATM 3 C1 LIG B 1 14.000 10.000 8.000 1.00 20.00 C \n", + encoding="utf-8", + ) + descriptor = describe_runtime( + structure, + artifact_path="receptor.pdb", + kind="receptor-structure", + media_type="chemical/x-pdb", + max_bytes=4096, + max_records=20, + generated_at="2026-07-21T00:00:00Z", + ).to_dict() + validate_runtime_descriptor(descriptor) + self.assertEqual("structure-3d", descriptor["viewer"]) + self.assertEqual(["A", "B"], descriptor["preview"]["chains_seen"]) + self.assertEqual(["LIG"], descriptor["preview"]["ligands_seen"]) + + selection = build_selection( + descriptor, + selector_type="residue", + selector={"chain": "A", "residue_number": 145}, + selected_by="reviewer", + reason="Inspect the alternate receptor state.", + created_at="2026-07-21T00:01:00Z", + ) + validate_selection(selection, {"receptor.pdb": descriptor["artifact_sha256"]}) + self.assertEqual("active", stale_selection(selection, {"receptor.pdb": descriptor["artifact_sha256"]})["status"]) + self.assertEqual("stale-anchor", stale_selection(selection, {"receptor.pdb": "0" * 64})["status"]) + + proposal = build_transform_proposal( + selection, + operation="exclude-alternate-conformation", + parameters={"altloc": "B"}, + reason="Use one receptor microstate.", + affected_steps=["receptor-preparation", "docking"], + expected_outputs=["prepared/receptor.pdbqt"], + proposed_by="analyst", + created_at="2026-07-21T00:02:00Z", + ) + validate_transform_proposal(proposal, selection) + self.assertFalse(proposal["executed"]) + self.assertTrue(proposal["requires_approval"]) + + def test_table_view_is_bounded_and_html_escapes_title(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + table = root / "metrics.csv" + table.write_text("metric,value\nrmsd,1.2\npr_auc,0.9\nextra,3\n", encoding="utf-8") + descriptor = describe_runtime( + table, + artifact_path="metrics.csv", + kind="metrics", + media_type="text/csv", + max_bytes=1024, + max_records=2, + generated_at="2026-07-21T00:00:00Z", + ).to_dict() + self.assertEqual("table", descriptor["viewer"]) + self.assertEqual(["metric", "value"], descriptor["preview"]["columns"]) + self.assertEqual(2, len(descriptor["preview"]["rows_preview"])) + html = render_runtime_html(descriptor, title="") + self.assertNotIn("", html) + self.assertIn("<script>alert(1)</script>", html) + + def test_unsupported_selector_and_digest_mismatch_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "data.txt" + path.write_text("hello\n", encoding="utf-8") + descriptor = describe_runtime( + path, + artifact_path="data.txt", + kind="text", + max_bytes=100, + max_records=10, + ).to_dict() + with self.assertRaises(ValueError): + build_selection( + descriptor, + selector_type="residue", + selector={"chain": "A"}, + selected_by="reviewer", + reason="wrong viewer", + ) + with self.assertRaises(ValueError): + describe_runtime( + path, + artifact_path="data.txt", + artifact_sha256="f" * 64, + kind="text", + max_bytes=100, + max_records=10, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_artifact_runtime_sidecars.py b/tests/test_artifact_runtime_sidecars.py new file mode 100644 index 00000000..d95e53fa --- /dev/null +++ b/tests/test_artifact_runtime_sidecars.py @@ -0,0 +1,119 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifact_runtime import build_selection, build_transform_proposal, describe_runtime +from codex_science.artifacts import add_artifact, new_manifest, validate_bundle, write_manifest +from codex_science.review import review_manifest + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class ArtifactRuntimeSidecarTests(unittest.TestCase): + def test_runtime_descriptor_selection_and_proposal_validate_as_hashed_sidecars(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) + data = run_dir / "metrics.csv" + data.write_text("metric,value\nrmsd,1.2\n", encoding="utf-8") + descriptor = describe_runtime( + data, + artifact_path="metrics.csv", + kind="metrics", + media_type="text/csv", + max_bytes=1024, + max_records=20, + generated_at="2026-07-21T00:00:00Z", + ).to_dict() + selection = build_selection( + descriptor, + selector_type="cell", + selector={"row": 1, "column": "value"}, + selected_by="reviewer", + reason="Inspect the RMSD threshold input.", + created_at="2026-07-21T00:01:00Z", + ) + proposal = build_transform_proposal( + selection, + operation="recalculate-metric", + parameters={"symmetry_aware": True}, + reason="Use the preregistered symmetry-aware definition.", + affected_steps=["compute-metrics", "review"], + expected_outputs=["metrics.csv"], + proposed_by="reviewer", + created_at="2026-07-21T00:02:00Z", + ) + descriptor_path = run_dir / "metrics.runtime.json" + selection_path = run_dir / "metric.selection.json" + proposal_path = run_dir / "metric.proposal.json" + for path, payload in ((descriptor_path, descriptor), (selection_path, selection), (proposal_path, proposal)): + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + manifest = new_manifest( + "runtime-sidecar-run", + "Does the metric artifact satisfy its declared definition?", + [{"id": "review", "description": "Inspect metric definition", "status": "completed"}], + ) + manifest["claims"] = [{"id": "C1", "text": "The metric is recorded.", "evidence": ["metrics.csv"]}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "metrics.csv", kind="metrics", sha256=digest(data)) + add_artifact(manifest, descriptor_path.name, kind="artifact-runtime-descriptor", sha256=digest(descriptor_path)) + add_artifact(manifest, selection_path.name, kind="artifact-selection", sha256=digest(selection_path)) + add_artifact(manifest, proposal_path.name, kind="transform-proposal", sha256=digest(proposal_path)) + write_manifest(manifest, run_dir / "manifest.json") + + sidecars = validate_bundle(manifest, run_dir) + self.assertEqual(1, len(sidecars["runtime_descriptors"])) + self.assertEqual(1, len(sidecars["artifact_selections"])) + self.assertEqual(1, len(sidecars["transform_proposals"])) + self.assertEqual([], sidecars["advanced_findings"]) + self.assertEqual("passed", review_manifest(manifest, run_dir, sidecars=sidecars)["status"]) + + def test_changed_target_bytes_make_runtime_records_stale_without_rewriting_them(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) + data = run_dir / "metrics.csv" + data.write_text("metric,value\nrmsd,1.2\n", encoding="utf-8") + descriptor = describe_runtime( + data, + artifact_path="metrics.csv", + kind="metrics", + max_bytes=1024, + max_records=20, + generated_at="2026-07-21T00:00:00Z", + ).to_dict() + selection = build_selection( + descriptor, + selector_type="cell", + selector={"row": 1, "column": "value"}, + selected_by="reviewer", + reason="Inspect old bytes.", + created_at="2026-07-21T00:01:00Z", + ) + data.write_text("metric,value\nrmsd,2.4\n", encoding="utf-8") + descriptor_path = run_dir / "metrics.runtime.json" + selection_path = run_dir / "metric.selection.json" + descriptor_path.write_text(json.dumps(descriptor, sort_keys=True) + "\n", encoding="utf-8") + selection_path.write_text(json.dumps(selection, sort_keys=True) + "\n", encoding="utf-8") + + manifest = new_manifest( + "stale-runtime-run", + "Did the inspected metric bytes change?", + [{"id": "inspect", "description": "Inspect stale anchor", "status": "completed"}], + ) + manifest["claims"] = [{"id": "C1", "text": "The current metric is recorded.", "evidence": ["metrics.csv"]}] + manifest["review"] = {"status": "findings", "findings": []} + add_artifact(manifest, "metrics.csv", kind="metrics", sha256=digest(data)) + add_artifact(manifest, descriptor_path.name, kind="artifact-runtime-descriptor", sha256=digest(descriptor_path)) + add_artifact(manifest, selection_path.name, kind="artifact-selection", sha256=digest(selection_path)) + sidecars = validate_bundle(manifest, run_dir) + codes = {item["code"] for item in sidecars["advanced_findings"]} + self.assertIn("stale-runtime-descriptor", codes) + self.assertIn("stale-artifact-selection", codes) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_artifact_store.py b/tests/test_artifact_store.py new file mode 100644 index 00000000..5f62bd84 --- /dev/null +++ b/tests/test_artifact_store.py @@ -0,0 +1,93 @@ +import os +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifact_store import ( + ContentAddressedStore, + describe_directory, + describe_external_reference, + describe_file, + validate_descriptor, +) +from codex_science.artifacts import add_artifact, new_manifest, verify_bundle_artifacts + + +class ArtifactStoreTests(unittest.TestCase): + def test_streaming_chunks_and_merkle_directory_are_stable(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) / "dataset" + root.mkdir() + (root / "a.bin").write_bytes(b"abcdef") + (root / "nested").mkdir() + (root / "nested" / "b.bin").write_bytes(b"ghijkl") + first = describe_directory(root, media_type="application/x-directory") + second = describe_directory(root, media_type="application/x-directory") + self.assertEqual(first.root_sha256, second.root_sha256) + self.assertEqual(2, first.entry_count) + validate_descriptor(first.to_dict(), root) + + chunked = describe_file(root / "a.bin", chunk_size=2) + self.assertEqual(3, len(chunked.entries[0]["chunks"])) + validate_descriptor(chunked.to_dict(), root / "a.bin") + + def test_manifest_validates_directory_without_whole_file_reads(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + run = Path(tempdir) + dataset = run / "trajectory" + dataset.mkdir() + (dataset / "frame-0001.bin").write_bytes(b"frame-one") + descriptor = describe_directory(dataset) + manifest = new_manifest("run-1", "Validate a directory artifact", []) + add_artifact( + manifest, + "trajectory", + kind="trajectory", + sha256=descriptor.root_sha256, + artifact_type="directory-tree", + size_bytes=descriptor.total_bytes, + entry_count=descriptor.entry_count, + ) + verified = verify_bundle_artifacts(manifest, run) + self.assertEqual(dataset, verified["trajectory"]) + (dataset / "frame-0001.bin").write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + verify_bundle_artifacts(manifest, run) + + + def test_external_reference_preserves_immutable_identity(self) -> None: + descriptor = describe_external_reference( + uri="s3://private-bucket/object", + version="version-7", + sha256="a" * 64, + size_bytes=123, + media_type="application/octet-stream", + license="internal-restricted", + ) + validated = validate_descriptor(descriptor.to_dict()) + self.assertEqual("external-reference", validated.artifact_type) + self.assertEqual(123, validated.total_bytes) + + def test_symlink_escape_is_rejected_and_store_is_content_addressed(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + source = root / "source.bin" + source.write_bytes(b"payload") + store = ContentAddressedStore(root / "cache") + digest, first = store.add_file(source) + digest2, second = store.add_file(source) + self.assertEqual((digest, first), (digest2, second)) + self.assertTrue(store.verify(digest)) + + directory = root / "dir" + directory.mkdir() + try: + os.symlink(source, directory / "escape") + except (OSError, NotImplementedError): + return + with self.assertRaisesRegex(ValueError, "symbolic links"): + describe_directory(directory) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_compute_backends.py b/tests/test_compute_backends.py new file mode 100644 index 00000000..11ea07d2 --- /dev/null +++ b/tests/test_compute_backends.py @@ -0,0 +1,148 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + +from codex_science.compute_backends import ( + JobSpec, + LocalBackend, + SlurmBackend, + build_approval_receipt, + validate_approval_receipt, +) + + +def local_spec(working: Path, *, command: list[str], outputs: list[str], timeout: int = 10) -> JobSpec: + return JobSpec.from_payload( + { + "schema_version": 1, + "backend": "local", + "name": "fixture-job", + "command": command, + "working_directory": str(working), + "environment": {"OMP_NUM_THREADS": "1"}, + "inherit_environment": True, + "inputs": [], + "outputs": outputs, + "resources": {"cpus": 1, "memory_mb": 256, "gpus": 0, "wall_time_seconds": timeout}, + "timeout_seconds": timeout, + "cost_cap": None, + "approval_required": False, + "scientific_run_id": "run-fixture", + "checkpoint_paths": [], + } + ) + + +class ComputeBackendTests(unittest.TestCase): + def test_local_job_is_durable_and_outputs_are_hash_collected(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + working = root / "work" + working.mkdir() + state_dir = root / "state" + spec = local_spec( + working, + command=[sys.executable, "-c", "from pathlib import Path; Path('result.json').write_text('{\\\"value\\\": 7}\\n')"], + outputs=["result.json"], + ) + backend = LocalBackend(state_dir) + preflight = backend.preflight(spec) + self.assertTrue(preflight["ready"]) + submitted = backend.submit(spec) + self.assertEqual("submitted", submitted["state"]) + terminal = backend.wait(submitted["job_id"], timeout_seconds=15, poll_seconds=0.05) + self.assertEqual("completed", terminal["state"], json.dumps(terminal, indent=2)) + self.assertEqual(0, terminal["exit_code"]) + collected = backend.collect(submitted["job_id"]) + self.assertEqual("file", collected["outputs"][0]["artifact_type"]) + self.assertEqual(64, len(collected["outputs"][0]["sha256"])) + self.assertEqual(1, collected["outputs"][0]["entry_count"]) + self.assertTrue((state_dir / "jobs" / submitted["job_id"] / "stdout.log").is_file()) + self.assertTrue((state_dir / "jobs" / submitted["job_id"] / "stderr.log").is_file()) + + def test_timeout_remains_a_distinct_terminal_failure(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + working = root / "work" + working.mkdir() + backend = LocalBackend(root / "state") + spec = local_spec( + working, + command=[sys.executable, "-c", "import time; time.sleep(2)"], + outputs=[], + timeout=1, + ) + submitted = backend.submit(spec) + terminal = backend.wait(submitted["job_id"], timeout_seconds=10, poll_seconds=0.05) + self.assertEqual("timed-out", terminal["state"]) + self.assertEqual("wall-time", terminal["failure_class"]) + + def test_approval_is_bound_to_exact_spec(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + working = Path(tempdir) + spec = JobSpec.from_payload( + { + "backend": "slurm", + "name": "approved-job", + "command": ["python", "analysis.py"], + "working_directory": str(working), + "environment": {}, + "inputs": [], + "outputs": ["result.json"], + "resources": {"cpus": 2, "memory_mb": 1024, "gpus": 1, "gpu_type": "a100", "wall_time_seconds": 600, "partition": "gpu"}, + "timeout_seconds": 600, + "approval_required": True, + } + ) + receipt = build_approval_receipt(spec, approved_by="jaemin", target="cluster-a", approved_at="2026-07-21T00:00:00Z") + validate_approval_receipt(receipt, spec) + changed = JobSpec.from_payload({**spec.to_dict(), "timeout_seconds": 601}) + with self.assertRaisesRegex(ValueError, "different job spec"): + validate_approval_receipt(receipt, changed) + + def test_slurm_render_is_nonexecuting_and_resource_explicit(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + spec = JobSpec.from_payload( + { + "backend": "slurm", + "name": "dock", + "command": ["python", "dock.py", "--input", "input.json"], + "working_directory": str(root), + "environment": {"OMP_NUM_THREADS": "4"}, + "inputs": [], + "outputs": ["poses.sdf"], + "resources": {"cpus": 4, "memory_mb": 8192, "gpus": 1, "gpu_type": "a100", "wall_time_seconds": 7200, "partition": "gpu", "account": "science"}, + "timeout_seconds": 7200, + "approval_required": True, + } + ) + backend = SlurmBackend(root / "state") + script = backend.render_script(spec, job_id="preview") + self.assertIn("#SBATCH --cpus-per-task=4", script) + self.assertIn("#SBATCH --mem=8192M", script) + self.assertIn("#SBATCH --gres=gpu:a100:1", script) + self.assertIn("#SBATCH --partition=gpu", script) + self.assertIn("#SBATCH --account=science", script) + self.assertIn("timeout --signal=TERM 7200s", script) + self.assertIn("python dock.py --input input.json", script) + + def test_secret_like_environment_keys_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + with self.assertRaisesRegex(ValueError, "secret-like"): + JobSpec.from_payload( + { + "backend": "local", + "name": "bad", + "command": ["echo", "x"], + "working_directory": tempdir, + "environment": {"API_TOKEN": "secret"}, + "outputs": [], + } + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_experiment_planner.py b/tests/test_experiment_planner.py new file mode 100644 index 00000000..1c1a47b9 --- /dev/null +++ b/tests/test_experiment_planner.py @@ -0,0 +1,68 @@ +import copy +import unittest + +from codex_science.experiment_planner import plan_next_experiment, validate_experiment_proposal + + +INPUT = { + "schema_version": 1, + "decision": "Select the next compound batch to discriminate potency and solubility hypotheses.", + "objectives": [ + {"name": "potency", "direction": "maximize", "weight": 0.6, "required": True}, + {"name": "solubility", "direction": "maximize", "weight": 0.4, "required": True} + ], + "candidates": [ + {"id": "control", "properties": {"potency": 0.2, "solubility": 0.9}, "cost": 2, "uncertainty": 0.1, "diversity_group": "control", "control": True}, + {"id": "a1", "properties": {"potency": 0.95, "solubility": 0.3}, "cost": 4, "uncertainty": 0.8, "diversity_group": "A"}, + {"id": "a2", "properties": {"potency": 0.85, "solubility": 0.4}, "cost": 3, "uncertainty": 0.2, "diversity_group": "A"}, + {"id": "b1", "properties": {"potency": 0.6, "solubility": 0.85}, "cost": 3, "uncertainty": 0.7, "diversity_group": "B"}, + {"id": "c1", "properties": {"potency": 0.55, "solubility": 0.7}, "cost": 10, "uncertainty": 0.5, "diversity_group": "C"}, + {"id": "excluded", "properties": {"potency": 1.0, "solubility": 1.0}, "cost": 1, "uncertainty": 1.0, "diversity_group": "X", "eligible": False, "exclusion_reason": "reactive-substructure"} + ], + "constraints": {"batch_size": 3, "budget": 10, "diversity_group_cap": 1, "minimum_controls": 1}, + "uncertainty_weight": 0.25, + "diversity_bonus": 0.2, + "claim_ids": ["C-potency", "C-solubility"] +} + + +class ExperimentPlannerTests(unittest.TestCase): + def test_controls_budget_diversity_and_pareto_ranking_are_explicit(self) -> None: + proposal = plan_next_experiment(INPUT, created_at="2026-07-21T00:00:00Z") + validate_experiment_proposal(proposal) + selected = [item["id"] for item in proposal["selected"]] + self.assertEqual("control", selected[0]) + self.assertEqual(3, len(selected)) + self.assertLessEqual(proposal["total_cost"], 10) + self.assertEqual(1, proposal["diversity_groups"]["control"]) + self.assertTrue(all(count <= 1 for count in proposal["diversity_groups"].values())) + self.assertTrue(proposal["required_controls_satisfied"]) + self.assertFalse(proposal["executed"]) + excluded = next(item for item in proposal["rejected"] if item["id"] == "excluded") + self.assertIn("reactive-substructure", excluded["reasons"]) + self.assertIn("expected_information_gain_proxy", proposal) + + def test_result_is_deterministic_for_same_contract(self) -> None: + first = plan_next_experiment(INPUT, created_at="2026-07-21T00:00:00Z") + second = plan_next_experiment(copy.deepcopy(INPUT), created_at="2026-07-21T00:00:00Z") + self.assertEqual(first, second) + + def test_required_controls_and_required_objectives_fail_closed(self) -> None: + no_controls = copy.deepcopy(INPUT) + no_controls["candidates"] = [item for item in no_controls["candidates"] if not item.get("control")] + with self.assertRaisesRegex(ValueError, "minimum_controls"): + plan_next_experiment(no_controls) + missing = copy.deepcopy(INPUT) + del missing["candidates"][1]["properties"]["solubility"] + with self.assertRaisesRegex(ValueError, "lacks required objectives"): + plan_next_experiment(missing) + + def test_required_controls_that_exceed_budget_are_not_silently_removed(self) -> None: + constrained = copy.deepcopy(INPUT) + constrained["constraints"]["budget"] = 1 + with self.assertRaisesRegex(ValueError, "required controls"): + plan_next_experiment(constrained) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fair_export.py b/tests/test_fair_export.py new file mode 100644 index 00000000..c9366ab1 --- /dev/null +++ b/tests/test_fair_export.py @@ -0,0 +1,81 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.fair_export import export_run + + +class FairExportTests(unittest.TestCase): + def test_validated_run_exports_ro_crate_prov_bom_and_receipt(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + run = root / "run" + run.mkdir() + result = run / "result.csv" + result.write_text("metric,value\neffect,0.7\n", encoding="utf-8") + digest = hashlib.sha256(result.read_bytes()).hexdigest() + manifest = new_manifest( + "run-fair", + "Does the recorded effect satisfy the acceptance contract?", + [{"id": "compute", "description": "Compute the effect", "status": "completed"}], + ) + manifest["inputs"] = [{"id": "dataset", "identifier": "doi:10.1000/dataset", "source": "repository", "release": "1"}] + manifest["code"] = [{"path": "analysis.py", "revision": "abc123"}] + manifest["executions"] = [{"id": "exec-1", "command": "python analysis.py", "exit_code": 0}] + manifest["environment"] = {"python": "3.12", "container_digest": "sha256:" + "a" * 64, "packages": [{"name": "numpy", "version": "2"}]} + manifest["claims"] = [{"id": "C1", "text": "The recorded effect is 0.7.", "evidence": ["result.csv"]}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "result.csv", kind="result-table", sha256=digest) + manifest_path = run / "manifest.json" + write_manifest(manifest, manifest_path) + + output = root / "export" + receipt = export_run(manifest_path, output, exported_at="2026-07-21T00:00:00Z") + self.assertFalse(receipt["certified"]) + self.assertFalse(receipt["regulatory_compliance_claimed"]) + self.assertEqual(3, len(receipt["exports"])) + for filename in ("ro-crate-metadata.json", "prov.json", "scientific-bom.json", "export-receipt.json"): + self.assertTrue((output / filename).is_file()) + + crate = json.loads((output / "ro-crate-metadata.json").read_text(encoding="utf-8")) + self.assertEqual("https://w3id.org/ro/crate/1.1/context", crate["@context"]) + ids = {item["@id"] for item in crate["@graph"]} + self.assertIn("result.csv", ids) + self.assertIn("#claim-C1", ids) + + prov = json.loads((output / "prov.json").read_text(encoding="utf-8")) + self.assertIn("run:run-fair", prov["activity"]) + self.assertIn("artifact:result.csv", prov["entity"]) + + bom = json.loads((output / "scientific-bom.json").read_text(encoding="utf-8")) + component_types = {item["component_type"] for item in bom["components"]} + self.assertIn("code", component_types) + self.assertIn("dataset", component_types) + self.assertIn("package", component_types) + self.assertIn("container_digest", component_types) + + def test_changed_artifact_bytes_prevent_export(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + result = root / "result.json" + result.write_text('{"value": 1}\n', encoding="utf-8") + manifest = new_manifest( + "run-bad", + "Q", + [{"id": "x", "description": "X", "status": "completed"}], + ) + manifest["claims"] = [{"id": "C", "text": "Claim", "evidence": ["result.json"]}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "result.json", kind="result", sha256=hashlib.sha256(result.read_bytes()).hexdigest()) + path = root / "manifest.json" + write_manifest(manifest, path) + result.write_text('{"value": 2}\n', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + export_run(path, root / "export") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mcp_v3.py b/tests/test_mcp_v3.py new file mode 100644 index 00000000..62d6eef2 --- /dev/null +++ b/tests/test_mcp_v3.py @@ -0,0 +1,45 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.mcp_server import CodexScienceMCP + + +class MCPV3Tests(unittest.TestCase): + def test_tool_and_source_contract_advertise_true_pagination(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + inventory = Path(tempdir) / "inventory.json" + inventory.write_text( + json.dumps( + { + "schema_version": 1, + "source": {"commit": "fixture"}, + "summary": {"total": 0, "active": 0, "inactive": 0}, + "skills": [], + } + ), + encoding="utf-8", + ) + server = CodexScienceMCP(inventory) + listed = server.handle({"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}) + names = {item["name"] for item in listed["result"]["tools"]} + self.assertIn("science_query_source_v3", names) + + contracts = server.handle( + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "science_list_source_contracts", "arguments": {}}, + } + ) + payload = json.loads(contracts["result"]["content"][0]["text"]) + by_source = {item["source"]: item for item in payload} + for source in ("pubmed", "europepmc", "chembl", "pdb"): + self.assertEqual(["search"], by_source[source]["v3_operations"]) + self.assertEqual([], by_source["gnomad"]["v3_operations"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pipeline_compiler.py b/tests/test_pipeline_compiler.py new file mode 100644 index 00000000..0a65c810 --- /dev/null +++ b/tests/test_pipeline_compiler.py @@ -0,0 +1,100 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.pipeline_compiler import compile_pipeline_draft + + +def reviewed_run(root: Path, *, review_status: str = "passed") -> Path: + run = root / "run" + run.mkdir() + table = run / "result.json" + table.write_text('{"value": 7}\n', encoding="utf-8") + digest = hashlib.sha256(table.read_bytes()).hexdigest() + manifest = new_manifest( + "source-run", + "Does the fixture satisfy the prespecified threshold?", + [{"id": "compute", "description": "Compute the fixture result", "status": "completed"}], + ) + manifest["inputs"] = [{"id": "input", "identifier": "fixture:input"}] + manifest["code"] = [{"path": "analysis.py", "revision": "abc"}] + manifest["executions"] = [{"command": "python analysis.py", "exit_code": 0}] + manifest["environment"] = {"python": ">=3.11", "seed": 0} + manifest["claims"] = [{"id": "C1", "text": "The result is seven.", "evidence": ["result.json"]}] + manifest["review"] = {"status": review_status, "findings": []} + add_artifact(manifest, "result.json", kind="result", sha256=digest) + path = run / "manifest.json" + write_manifest(manifest, path) + return path + + +class PipelineCompilerTests(unittest.TestCase): + def test_passed_run_generates_nonactive_progressive_skill_draft(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + manifest = reviewed_run(root) + output = root / "draft" + receipt = compile_pipeline_draft( + manifest_path=manifest, + output_dir=output, + name="fixture-pipeline", + description="Run the reviewed fixture workflow on new approved inputs.", + command_contract=[["python", "analysis.py", "--input", "input.json", "--output", "result.json"]], + generated_at="2026-07-21T00:00:00Z", + ) + self.assertEqual("draft", receipt["status"]) + self.assertFalse(receipt["activated"]) + self.assertFalse(receipt["cataloged"]) + self.assertTrue((output / "SKILL.md").is_file()) + self.assertTrue((output / "input.schema.json").is_file()) + self.assertTrue((output / "output.schema.json").is_file()) + index = json.loads((output / "references" / "index.json").read_text(encoding="utf-8")) + self.assertEqual("pipeline", index["references"][0]["id"]) + skill = (output / "SKILL.md").read_text(encoding="utf-8") + self.assertIn("not active", skill.lower()) + self.assertIn("references/pipeline.md", skill) + reference = (output / "references" / "pipeline.md").read_text(encoding="utf-8") + self.assertIn("## Command contract", reference) + self.assertIn("## Validation and promotion", reference) + self.assertIn('["python", "analysis.py"', reference) + + def test_nonpassed_or_incomplete_run_cannot_be_promoted(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + manifest = reviewed_run(root, review_status="findings") + with self.assertRaisesRegex(ValueError, "passed"): + compile_pipeline_draft( + manifest_path=manifest, + output_dir=root / "draft", + name="fixture-pipeline", + description="Fixture.", + ) + + def test_existing_nonempty_output_and_invalid_name_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + manifest = reviewed_run(root) + output = root / "draft" + output.mkdir() + (output / "keep.txt").write_text("do not overwrite", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "not empty"): + compile_pipeline_draft( + manifest_path=manifest, + output_dir=output, + name="fixture-pipeline", + description="Fixture.", + ) + with self.assertRaisesRegex(ValueError, "kebab-case"): + compile_pipeline_draft( + manifest_path=manifest, + output_dir=root / "other", + name="Invalid Name", + description="Fixture.", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pipeline_compiler_security.py b/tests/test_pipeline_compiler_security.py new file mode 100644 index 00000000..9c7db637 --- /dev/null +++ b/tests/test_pipeline_compiler_security.py @@ -0,0 +1,87 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.pipeline_compiler import compile_pipeline_draft + + +def make_run(root: Path) -> Path: + run = root / "run" + run.mkdir() + output = run / "result.json" + output.write_text('{"ok": true}\n', encoding="utf-8") + manifest = new_manifest( + "secure-source-run", + "Can the reviewed workflow be represented without copying private inputs?", + [{"id": "run", "description": "Run fixture", "status": "completed"}], + ) + manifest["inputs"] = [ + { + "id": "private-input", + "path": "/private/patient/data.csv", + "sha256": "a" * 64, + "token": "must-not-propagate" + } + ] + manifest["environment"] = { + "python": "3.12", + "API_TOKEN": "must-not-propagate", + "nested": {"client_secret": "must-not-propagate", "threads": 4} + } + manifest["claims"] = [{"id": "C1", "text": "The fixture completed.", "evidence": ["result.json"]}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "result.json", kind="result", sha256=hashlib.sha256(output.read_bytes()).hexdigest()) + path = run / "manifest.json" + write_manifest(manifest, path) + return path + + +class PipelineCompilerSecurityTests(unittest.TestCase): + def test_private_input_values_and_environment_secrets_are_not_copied(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + manifest = make_run(root) + output = root / "draft" + compile_pipeline_draft( + manifest_path=manifest, + output_dir=output, + name="secure-draft", + description="Secure draft.", + command_contract=[["python", "analysis.py"]], + generated_at="2026-07-21T00:00:00Z", + ) + schema = (output / "input.schema.json").read_text(encoding="utf-8") + reference = (output / "references" / "pipeline.md").read_text(encoding="utf-8") + self.assertNotIn("/private/patient/data.csv", schema) + self.assertNotIn("must-not-propagate", schema) + self.assertNotIn("must-not-propagate", reference) + self.assertIn("[REDACTED]", reference) + payload = json.loads(schema) + self.assertEqual(1, payload["x-source-run-input-count"]) + self.assertIn("path", payload["x-source-run-input-fields"]) + + def test_credential_bearing_argv_and_urls_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + manifest = make_run(root) + cases = [ + [["tool", "--token", "secret-value"]], + [["tool", "--api-key=secret-value"]], + [["curl", "https://example.org/data?token=secret-value"]] + ] + for index, commands in enumerate(cases): + with self.subTest(commands=commands), self.assertRaisesRegex(ValueError, "credential"): + compile_pipeline_draft( + manifest_path=manifest, + output_dir=root / f"draft-{index}", + name=f"secure-draft-{index}", + description="Secure draft.", + command_contract=commands, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_project_store.py b/tests/test_project_store.py new file mode 100644 index 00000000..586d014e --- /dev/null +++ b/tests/test_project_store.py @@ -0,0 +1,138 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.project_store import ProjectStore + + +def make_run(root: Path, run_id: str, claim_text: str, metric_value: float) -> tuple[Path, str]: + run_dir = root / run_id + run_dir.mkdir(parents=True) + metrics = run_dir / "metrics.json" + metrics.write_text(json.dumps({"metric": "effect", "value": metric_value}) + "\n", encoding="utf-8") + digest = hashlib.sha256(metrics.read_bytes()).hexdigest() + manifest = new_manifest( + run_id, + "Which branch best supports the prespecified hypothesis?", + [{"id": "analyze", "description": "Compute the prespecified metric", "status": "completed"}], + ) + manifest["claims"] = [{"id": "C1", "text": claim_text, "evidence": ["metrics.json"]}] + manifest["executions"] = [{"command": "fixture", "exit_code": 0}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "metrics.json", kind="metrics", sha256=digest) + manifest_path = run_dir / "manifest.json" + write_manifest(manifest, manifest_path) + return manifest_path, digest + + +class ProjectStoreTests(unittest.TestCase): + def test_import_fork_assert_compare_and_merge_plan_preserve_run_authority(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + database = root / "project" / "evidence.sqlite" + store = ProjectStore(database) + project = store.create_project( + project_id="project-1", + title="Hypothesis comparison", + question="Which branch is supported?", + ) + self.assertEqual("project-1", project["project_id"]) + + base_manifest, _base_digest = make_run(root, "run-base", "Baseline claim", 0.5) + source_manifest, source_digest = make_run(root, "run-source", "Source branch claim", 0.8) + target_manifest, _target_digest = make_run(root, "run-target", "Target branch claim", 0.2) + + base = store.import_run(project_id="project-1", manifest_path=base_manifest, branch_name="main") + self.assertEqual("run-base", base.run_id) + store.fork_branch(project_id="project-1", source_run_id="run-base", branch_name="experiment") + source = store.import_run(project_id="project-1", manifest_path=source_manifest, branch_name="experiment") + target = store.import_run(project_id="project-1", manifest_path=target_manifest, branch_name="main") + self.assertEqual("run-base", source.parent_run_id) + self.assertEqual("run-base", target.parent_run_id) + + assertion = store.add_assertion( + project_id="project-1", + run_id="run-source", + claim_id="C1", + source_id="study-1", + polarity="supports", + locator={ + "artifact_path": "metrics.json", + "artifact_sha256": source_digest, + "json_pointer": "/value", + }, + independence_group="cohort-A", + effect_measure="difference", + estimate=0.8, + interval_low=0.6, + interval_high=1.0, + sample_size=42, + population="fixture population", + ) + self.assertTrue(assertion["assertion_id"].startswith("assertion-")) + + comparison = store.compare_runs( + project_id="project-1", + previous_run_id="run-base", + current_run_id="run-source", + ) + self.assertEqual(["metrics.json"], comparison["artifacts"]["changed"]) + self.assertEqual(["C1"], comparison["claims"]["changed"]) + self.assertTrue(comparison["review_invalidated"]) + + plan = store.prepare_merge_plan( + project_id="project-1", + source_branch="experiment", + target_branch="main", + ) + self.assertEqual("blocked-conflicts", plan["status"]) + self.assertEqual(["C1"], plan["claim_conflicts"]) + self.assertEqual(["metrics.json"], plan["artifact_conflicts"]) + self.assertFalse(plan["executed"]) + self.assertTrue(plan["requires_scientific_review"]) + + summary = store.summary(project_id="project-1") + self.assertEqual(3, len(summary["runs"])) + self.assertEqual(1, summary["assertion_count"]) + self.assertEqual(1, summary["merge_plan_count"]) + self.assertIsNotNone(summary["event_chain_head"]) + + def test_changed_manifest_bytes_cannot_be_reused_under_same_run_id(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + store = ProjectStore(root / "project.sqlite") + store.create_project(project_id="p", title="P", question="Q") + manifest_path, _digest = make_run(root, "run-1", "Claim", 1.0) + store.import_run(project_id="p", manifest_path=manifest_path, branch_name="main") + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + payload["question"] = "Changed question" + manifest_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "different manifest bytes"): + store.import_run(project_id="p", manifest_path=manifest_path, branch_name="main") + with self.assertRaisesRegex(ValueError, "changed on disk"): + store.compare_runs(project_id="p", previous_run_id="run-1", current_run_id="run-1") + + def test_assertion_requires_exact_imported_artifact_hash(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + store = ProjectStore(root / "project.sqlite") + store.create_project(project_id="p", title="P", question="Q") + manifest_path, _digest = make_run(root, "run-1", "Claim", 1.0) + store.import_run(project_id="p", manifest_path=manifest_path, branch_name="main") + with self.assertRaisesRegex(ValueError, "does not match"): + store.add_assertion( + project_id="p", + run_id="run-1", + claim_id="C1", + source_id="study", + polarity="supports", + locator={"artifact_path": "metrics.json", "artifact_sha256": "0" * 64, "json_pointer": "/value"}, + independence_group="cohort", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reference_contract.py b/tests/test_reference_contract.py new file mode 100644 index 00000000..7b713e89 --- /dev/null +++ b/tests/test_reference_contract.py @@ -0,0 +1,87 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.reference_contract import ( + audit_reference_roots, + build_reference_use_receipt, + load_reference_index, + select_references, + validate_reference_index, + validate_reference_use_ledger, +) + + +class ReferenceContractTests(unittest.TestCase): + def _skill(self, root: Path) -> Path: + skill = root / "literature-review" + references = skill / "references" + references.mkdir(parents=True) + (skill / "SKILL.md").write_text( + "---\nname: literature-review\ndescription: test\n---\n" + "## Reference usage\n" + "Read [study identity](references/study-identity.md) before deduplication.\n", + encoding="utf-8", + ) + (references / "study-identity.md").write_text( + "# Study identity\n\nUse namespaced persistent identifiers.\n", encoding="utf-8" + ) + (references / "index.json").write_text( + json.dumps( + { + "schema_version": 1, + "skill": "literature-review", + "references": [ + { + "id": "study-identity", + "path": "references/study-identity.md", + "purpose": "Resolve publication versions into study families.", + "read_when": ["deduplicating studies", "resolving corrections"], + "required_before": ["study-deduplication"], + "search_patterns": ["namespaced persistent identifiers"], + } + ], + } + ), + encoding="utf-8", + ) + return skill + + def test_index_selection_and_use_receipt(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + skill = self._skill(Path(tempdir)) + report = validate_reference_index(skill, strict=True) + self.assertEqual([], report["findings"]) + index = load_reference_index(skill) + assert index is not None + selected = select_references(index, route="study-deduplication") + self.assertEqual(["study-identity"], [item.id for item in selected]) + receipt = build_reference_use_receipt( + skill, + selected[0], + read_reason="study-deduplication", + used_for=["claim-C1"], + sections=["Study identity"], + ) + ledger = {"schema_version": 1, "skill": skill.name, "uses": [receipt.to_dict()]} + validate_reference_use_ledger(ledger) + + def test_unindexed_reference_is_reported(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + skill = self._skill(Path(tempdir)) + (skill / "references" / "extra.md").write_text("# Extra\n", encoding="utf-8") + report = validate_reference_index(skill, strict=True) + self.assertIn("unindexed-reference-file", {item["code"] for item in report["findings"]}) + + def test_root_audit_only_blocks_strict_major_findings(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) / "authored-skills" + skill = self._skill(root) + report = audit_reference_roots([root], strict_names=[skill.name]) + self.assertEqual(0, report["summary"]["major_findings"]) + self.assertEqual(1, report["summary"]["indexed_skills"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_review_action_receipts.py b/tests/test_review_action_receipts.py new file mode 100644 index 00000000..4ed21887 --- /dev/null +++ b/tests/test_review_action_receipts.py @@ -0,0 +1,121 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from codex_science.action_connectors import ( + ActionLedger, + ActionSpec, + build_action_approval, + build_preview, + execute_action, + validate_action_receipt, +) +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.reviewer_runtime import build_review_packet, finalize_review_response +from codex_science.review_receipts import validate_review_receipt + + +class Adapter: + connector_name = "fixture" + + def __init__(self) -> None: + self.state = {"id": "x", "version": 1} + + def snapshot(self, spec): + return dict(self.state) + + def preview(self, spec, before): + return {"change": spec.parameters} + + def execute(self, spec, preview): + self.state = {"id": "x", "version": 2} + return {"after_state": dict(self.state), "remote_object_ids": ["x"]} + + +def manifest_path(root: Path) -> Path: + artifact = root / "value.json" + artifact.write_text('{"value": 1}\n', encoding="utf-8") + manifest = new_manifest( + "review-unicode", + "검토 패킷의 해시가 유니코드에서도 안정적인가?", + [{"id": "record", "description": "Record value", "status": "completed"}], + ) + manifest["claims"] = [{"id": "C1", "text": "값이 기록되었다.", "evidence": ["value.json"]}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "value.json", kind="result", sha256=hashlib.sha256(artifact.read_bytes()).hexdigest()) + path = root / "manifest.json" + write_manifest(manifest, path) + return path + + +class ReviewActionReceiptTests(unittest.TestCase): + def test_unicode_review_receipt_uses_canonical_review_hash(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + packet = build_review_packet(manifest_path(root), review_modes=["record"], created_at="2026-07-21T00:00:00Z") + artifact = packet["artifacts"][0] + receipt = finalize_review_response( + packet, + { + "schema_version": 1, + "review_task_id": packet["review_task_id"], + "packet_fingerprint": packet["fingerprint"], + "reviewer": "독립-검토자", + "independent": True, + "review_modes": ["record"], + "reviewed_claim_ids": ["C1"], + "reviewed_artifacts": [{"path": artifact["path"], "sha256": artifact["sha256"]}], + "findings": [], + "limitations": ["원시 장비 파일은 검토하지 못했다."], + "status": "passed", + }, + ) + validate_review_receipt(receipt) + self.assertEqual("passed", receipt["status"]) + + def test_action_receipt_is_bound_to_exact_spec(self) -> None: + adapter = Adapter() + spec = ActionSpec.from_payload( + { + "connector": "fixture", + "operation": "update", + "mode": "write", + "target": "object/x", + "parameters": {"value": 2}, + "requested_scopes": ["object:write"], + "idempotency_key": "fixture-update-x-v1", + } + ) + preview = build_preview(adapter, spec, created_at="2026-07-21T00:00:00Z") + approval = build_action_approval(preview, approved_by="jaemin", approved_scopes=["object:write"], approved_at="2026-07-21T00:01:00Z") + with tempfile.TemporaryDirectory() as tempdir: + receipt = execute_action(adapter, spec, preview, approval=approval, ledger=ActionLedger(Path(tempdir) / "ledger.sqlite"), executed_at="2026-07-21T00:02:00Z") + validate_action_receipt(receipt, spec) + changed = ActionSpec.from_payload({**spec.to_dict(), "parameters": {"value": 3}, "idempotency_key": "fixture-update-x-v2"}) + with self.assertRaisesRegex(ValueError, "different spec"): + validate_action_receipt(receipt, changed) + + def test_mismatched_adapter_is_rejected_before_execution(self) -> None: + class Wrong(Adapter): + connector_name = "wrong" + + spec = ActionSpec.from_payload( + { + "connector": "fixture", + "operation": "read", + "mode": "read", + "target": "object/x", + "parameters": {}, + "requested_scopes": ["object:read"], + "idempotency_key": "fixture-read-x-v1", + } + ) + preview = build_preview(Adapter(), spec) + with tempfile.TemporaryDirectory() as tempdir: + with self.assertRaisesRegex(ValueError, "adapter connector"): + execute_action(Wrong(), spec, preview, approval=None, ledger=ActionLedger(Path(tempdir) / "ledger.sqlite")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reviewer_runtime.py b/tests/test_reviewer_runtime.py new file mode 100644 index 00000000..84bc3940 --- /dev/null +++ b/tests/test_reviewer_runtime.py @@ -0,0 +1,137 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.artifacts import add_artifact, new_manifest, write_manifest +from codex_science.reviewer_runtime import build_review_packet, finalize_review_response, validate_review_packet +from codex_science.review_receipts import validate_review_receipt + + +def make_reviewed_run(root: Path) -> Path: + run = root / "run" + run.mkdir() + evidence = run / "evidence.json" + evidence.write_text('{"estimate": 0.72}\n', encoding="utf-8") + digest = hashlib.sha256(evidence.read_bytes()).hexdigest() + manifest = new_manifest( + "review-source-run", + "Does the evidence support the bounded association claim?", + [{"id": "analyze", "description": "Analyze the recorded evidence", "status": "completed"}], + ) + manifest["inputs"] = [{"id": "study", "identifier": "pmid:12345678", "token": "must-redact"}] + manifest["code"] = [{"path": "analysis.py", "revision": "abc"}] + manifest["executions"] = [{"command": "python analysis.py", "exit_code": 0}] + manifest["environment"] = {"python": "3.12", "API_TOKEN": "must-redact"} + manifest["claims"] = [{"id": "C1", "text": "The recorded study supports an association.", "evidence": ["evidence.json"], "producer_rationale": "must-exclude"}] + manifest["review"] = {"status": "passed", "findings": []} + add_artifact(manifest, "evidence.json", kind="study-evidence", sha256=digest) + path = run / "manifest.json" + write_manifest(manifest, path) + return path + + +class ReviewerRuntimeTests(unittest.TestCase): + def test_packet_is_hash_bound_and_excludes_producer_only_context(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + manifest = make_reviewed_run(Path(tempdir)) + packet = build_review_packet( + manifest, + review_modes=["record", "method"], + created_at="2026-07-21T00:00:00Z", + ) + validate_review_packet(packet) + serialized = json.dumps(packet) + self.assertNotIn("must-exclude", serialized) + self.assertNotIn("must-redact", serialized) + self.assertIn("[REDACTED]", serialized) + self.assertEqual(["C1"], packet["material_claim_ids"]) + self.assertEqual(1, len(packet["artifacts"])) + self.assertTrue(packet["independent_required"]) + + def test_complete_independent_response_builds_passed_receipt(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + manifest = make_reviewed_run(Path(tempdir)) + packet = build_review_packet( + manifest, + review_modes=["record", "method"], + created_at="2026-07-21T00:00:00Z", + ) + artifact = packet["artifacts"][0] + response = { + "schema_version": 1, + "review_task_id": packet["review_task_id"], + "packet_fingerprint": packet["fingerprint"], + "reviewer": "independent-reviewer-1", + "independent": True, + "review_modes": ["record", "method"], + "reviewed_claim_ids": ["C1"], + "reviewed_artifacts": [{"path": artifact["path"], "sha256": artifact["sha256"]}], + "findings": [], + "limitations": ["The source study is not independently reproduced in this review."], + "status": "passed" + } + receipt = finalize_review_response(packet, response) + validate_review_receipt(receipt) + self.assertEqual("passed", receipt["status"]) + self.assertTrue(receipt["independent"]) + self.assertEqual(packet["review_task_id"], receipt["review_task_id"]) + + def test_nonindependent_or_incomplete_review_cannot_pass(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + manifest = make_reviewed_run(Path(tempdir)) + packet = build_review_packet( + manifest, + review_modes=["record"], + created_at="2026-07-21T00:00:00Z", + ) + artifact = packet["artifacts"][0] + response = { + "schema_version": 1, + "review_task_id": packet["review_task_id"], + "packet_fingerprint": packet["fingerprint"], + "reviewer": "producer-second-pass", + "independent": False, + "review_modes": ["record"], + "reviewed_claim_ids": [], + "reviewed_artifacts": [{"path": artifact["path"], "sha256": artifact["sha256"]}], + "findings": [], + "limitations": ["Same producer context."], + "blocked_reason": "No independent reviewer was available.", + "status": "passed" + } + receipt = finalize_review_response(packet, response) + self.assertEqual("findings", receipt["status"]) + codes = {item["code"] for item in receipt["findings"]} + self.assertIn("review-not-independent", codes) + self.assertIn("incomplete-review-coverage", codes) + + def test_response_cannot_substitute_artifact_hash_or_unknown_claim(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + manifest = make_reviewed_run(Path(tempdir)) + packet = build_review_packet(manifest, review_modes=["record"], created_at="2026-07-21T00:00:00Z") + base = { + "schema_version": 1, + "review_task_id": packet["review_task_id"], + "packet_fingerprint": packet["fingerprint"], + "reviewer": "reviewer", + "independent": True, + "review_modes": ["record"], + "reviewed_claim_ids": ["C1"], + "reviewed_artifacts": [{"path": packet["artifacts"][0]["path"], "sha256": "0" * 64}], + "findings": [], + "limitations": ["Limited."], + "status": "passed" + } + with self.assertRaisesRegex(ValueError, "does not match"): + finalize_review_response(packet, base) + changed = dict(base) + changed["reviewed_claim_ids"] = ["UNKNOWN"] + changed["reviewed_artifacts"] = [{"path": packet["artifacts"][0]["path"], "sha256": packet["artifacts"][0]["sha256"]}] + with self.assertRaisesRegex(ValueError, "unknown claims"): + finalize_review_response(packet, changed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime_hardening.py b/tests/test_runtime_hardening.py new file mode 100644 index 00000000..9e93c07f --- /dev/null +++ b/tests/test_runtime_hardening.py @@ -0,0 +1,107 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from codex_science.action_connectors import ActionSpec, build_preview +from codex_science.artifact_runtime import describe_runtime +from codex_science.compute_backends import JobSpec, LocalBackend +from codex_science.experiment_planner import plan_next_experiment +from codex_science.pipeline_compiler import _validate_command + + +class SecretAdapter: + connector_name = "secret-provider" + + def snapshot(self, spec): + return {"object": "x", "api_token": "must-not-record"} + + def preview(self, spec, before): + return {"change": "none"} + + def execute(self, spec, preview): + return {"after_state": {"object": "x"}} + + +class RuntimeHardeningTests(unittest.TestCase): + def test_artifact_runtime_has_absolute_view_limits(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "data.txt" + path.write_text("hello\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "hard runtime ceiling"): + describe_runtime( + path, + artifact_path="data.txt", + kind="text", + max_bytes=16 * 1024 * 1024 + 1, + max_records=10, + ) + with self.assertRaisesRegex(ValueError, "hard runtime ceiling"): + describe_runtime( + path, + artifact_path="data.txt", + kind="text", + max_bytes=100, + max_records=10_001, + ) + + def test_local_preflight_verifies_declared_input_hash(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + work = root / "work" + work.mkdir() + data = work / "input.json" + data.write_text('{"value": 1}\n', encoding="utf-8") + good = hashlib.sha256(data.read_bytes()).hexdigest() + spec = JobSpec.from_payload( + { + "backend": "local", + "name": "input-check", + "command": ["python", "-c", "print('ok')"], + "working_directory": str(work), + "inputs": [{"id": "input", "path": "input.json", "sha256": good}], + "outputs": [], + } + ) + checks = LocalBackend(root / "state").preflight(spec)["checks"] + input_check = next(item for item in checks if item["name"] == "input:input") + self.assertTrue(input_check["ready"]) + stale = JobSpec.from_payload({**spec.to_dict(), "inputs": [{"id": "input", "path": "input.json", "sha256": "0" * 64}]}) + result = LocalBackend(root / "other-state").preflight(stale) + input_check = next(item for item in result["checks"] if item["name"] == "input:input") + self.assertFalse(input_check["ready"]) + self.assertFalse(result["ready"]) + + def test_provider_state_with_secret_like_keys_is_rejected(self) -> None: + spec = ActionSpec.from_payload( + { + "connector": "secret-provider", + "operation": "read", + "mode": "read", + "target": "object/x", + "parameters": {}, + "requested_scopes": ["object:read"], + "idempotency_key": "secret-provider-read-x-v1", + } + ) + with self.assertRaisesRegex(ValueError, "credential-like"): + build_preview(SecretAdapter(), spec) + + def test_boolean_integer_constraints_are_rejected(self) -> None: + payload = { + "decision": "Select candidates.", + "objectives": [{"name": "score", "direction": "maximize", "weight": 1}], + "candidates": [{"id": "x", "properties": {"score": 1}, "cost": 0, "uncertainty": 0}], + "constraints": {"batch_size": True, "budget": 1, "diversity_group_cap": 1, "minimum_controls": 0}, + } + with self.assertRaisesRegex(ValueError, "must be integers"): + plan_next_experiment(payload) + + def test_any_credential_argument_flag_is_rejected(self) -> None: + for command in (["tool", "--token"], ["tool", "--password=value"], ["curl", "https://example.test/?authorization=value"]): + with self.subTest(command=command), self.assertRaisesRegex(ValueError, "credential"): + _validate_command(command, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_maturity.py b/tests/test_skill_maturity.py new file mode 100644 index 00000000..312db7a3 --- /dev/null +++ b/tests/test_skill_maturity.py @@ -0,0 +1,152 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.skill_maturity import audit_native_skills, evaluate_skill + + +SKILL_TEXT = """--- +name: demo-skill +description: Run a bounded scientific demo workflow. +--- + +# Demo skill + +## Decision contract + +Record the question, scope, and acceptance criteria. + +## Reference usage + +Read [the command reference](references/commands.md) before execution. + +## Workflow + +Use `$science-provenance`, execute the fixture, and use `$science-review`. + +## Outputs + +Write a machine-readable result. + +## Boundaries + +Do not overclaim the fixture. +""" + + +class SkillMaturityTests(unittest.TestCase): + def test_repository_policy_passes_for_declared_tier_one_skills(self) -> None: + root = Path(__file__).resolve().parents[1] + report = audit_native_skills(root) + self.assertEqual("passed", report["status"], json.dumps(report["findings"], indent=2)) + by_name = {item["name"]: item for item in report["skills"]} + self.assertEqual("L4", by_name["literature-review"]["computed_maturity"]) + self.assertEqual("L4", by_name["docking-validation"]["computed_maturity"]) + self.assertGreaterEqual(int(by_name["codex-science"]["computed_maturity"][1:]), 3) + + def test_synthetic_l4_requires_references_outputs_fixture_failures_and_tests(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + skill = root / "skills" / "demo" + references = skill / "references" + references.mkdir(parents=True) + (skill / "SKILL.md").write_text(SKILL_TEXT, encoding="utf-8") + (references / "commands.md").write_text("# Commands\n\n## Usage\n", encoding="utf-8") + (references / "index.json").write_text( + json.dumps( + { + "schema_version": 1, + "skill": "demo-skill", + "references": [ + { + "id": "commands", + "path": "references/commands.md", + "purpose": "Exact command contract.", + "read_when": ["before execution"], + "required_before": ["running the demo"], + "search_patterns": ["## Usage"], + "authority": "first-party", + "version": "1", + "evidence_boundary": "Commands do not prove scientific validity." + } + ] + } + ), + encoding="utf-8", + ) + fixture = root / "fixtures" / "case.json" + test_file = root / "tests" / "test_demo.py" + fixture.parent.mkdir() + test_file.parent.mkdir() + fixture.write_text("{}\n", encoding="utf-8") + test_file.write_text("# fixture test\n", encoding="utf-8") + (skill / "quality.json").write_text( + json.dumps( + { + "schema_version": 1, + "skill": "demo-skill", + "roles": ["executor", "reviewer"], + "declared_maturity": "L4", + "output_schemas": ["demo-result-v1"], + "acceptance_fixtures": ["fixtures/case.json"], + "seeded_failures": ["wrong-unit"], + "test_files": ["tests/test_demo.py"], + "limitations": ["Fixture only."] + } + ), + encoding="utf-8", + ) + record = evaluate_skill(root, skill) + self.assertEqual("L4", record["computed_maturity"], record["reasons"]) + self.assertEqual(["executor", "reviewer"], record["roles"]) + + def test_invalid_l4_declaration_is_not_silently_downgraded(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + root = Path(tempdir) + skill = root / "skills" / "demo" + references = skill / "references" + references.mkdir(parents=True) + (skill / "SKILL.md").write_text(SKILL_TEXT, encoding="utf-8") + (references / "commands.md").write_text("# Commands\n", encoding="utf-8") + (references / "index.json").write_text( + json.dumps( + { + "schema_version": 1, + "skill": "demo-skill", + "references": [ + { + "id": "commands", + "path": "references/commands.md", + "read_when": ["execution"], + "required_before": ["execution"], + "evidence_boundary": "Fixture only." + } + ] + } + ), + encoding="utf-8", + ) + (skill / "quality.json").write_text( + json.dumps( + { + "schema_version": 1, + "skill": "demo-skill", + "roles": ["executor"], + "declared_maturity": "L4", + "output_schemas": ["demo-v1"], + "acceptance_fixtures": [], + "seeded_failures": [], + "test_files": [], + "limitations": [] + } + ), + encoding="utf-8", + ) + record = evaluate_skill(root, skill) + self.assertLess(int(record["computed_maturity"][1:]), 4) + self.assertTrue(any("invalid quality declaration" in reason for reason in record["reasons"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transport_v3.py b/tests/test_transport_v3.py new file mode 100644 index 00000000..c3a93e29 --- /dev/null +++ b/tests/test_transport_v3.py @@ -0,0 +1,85 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from codex_science.connector_contract import QueryRequest +from codex_science.source_operations_v3 import EuropePMCSearchV3, PubMedSearchV3 +from codex_science.transport_v3 import ( + PageRequest, + TransportError, + TransportResponse, + execute_paginated, + replay_snapshot_directory, +) + + +class FakeTransport: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + def send(self, request: PageRequest) -> TransportResponse: + self.requests.append(request) + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + payload, headers = response + return TransportResponse( + 200, + request.url, + headers, + json.dumps(payload, separators=(",", ":")).encode("utf-8"), + "application/json", + "2026-07-20T00:00:00Z", + 1, + ) + + +class TransportV3Tests(unittest.TestCase): + def test_pubmed_fetches_all_pages_and_replays_raw_bodies(self) -> None: + request = QueryRequest( + "pubmed", "search", {"query": "protein folding"}, + page_size=2, max_pages=3, source_contract_version="3", + ) + transport = FakeTransport( + [ + ({"esearchresult": {"count": "3", "retstart": "0", "idlist": ["1", "2"]}}, {"etag": '"one"', "last-modified": "date"}), + ({"esearchresult": {"count": "3", "retstart": "2", "idlist": ["3"]}}, {"etag": '"two"'}), + ] + ) + with tempfile.TemporaryDirectory() as tempdir: + snapshot_dir = Path(tempdir) + result = execute_paginated(PubMedSearchV3(), request, transport=transport, snapshot_dir=snapshot_dir) + self.assertEqual("complete", result.receipt.status) + self.assertEqual(["1", "2", "3"], [item["id"] for item in result.records]) + self.assertEqual('"one"', result.receipt.pages[0].etag) + self.assertEqual(2, len(replay_snapshot_directory(snapshot_dir, request.query_id))) + + def test_user_page_bound_is_not_source_completion(self) -> None: + request = QueryRequest( + "europepmc", "search", {"query": "BRCA1"}, + page_size=1, max_pages=1, source_contract_version="3", + ) + transport = FakeTransport( + [({"nextCursorMark": "cursor-2", "resultList": {"result": [{"id": "1", "source": "MED"}]}}, {})] + ) + result = execute_paginated(EuropePMCSearchV3(), request, transport=transport) + self.assertEqual("bounded-by-user", result.receipt.status) + self.assertEqual("partial", result.receipt.missingness) + + def test_error_statuses_remain_distinct(self) -> None: + request = QueryRequest( + "pubmed", "search", {"query": "x"}, source_contract_version="3" + ) + result = execute_paginated( + PubMedSearchV3(), request, + transport=FakeTransport([TransportError("rate", status_code=429, retryable=True)]), + ) + self.assertEqual("partial-rate-limit", result.receipt.status) + self.assertEqual("partial", result.receipt.missingness) + self.assertEqual(0, result.receipt.fetched_pages) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index a5abc83b..4df1b756 100644 --- a/uv.lock +++ b/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.11" [[package]] name = "codex-science" -version = "0.3.0" +version = "0.4.0" source = { editable = "." }