From ef893c5f0883d30dc7156b6a238ec224699d3653 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:09:31 +0100 Subject: [PATCH 01/35] sync: review anticipated upstream baseline and isolate remaining fork work (#178) ## Purpose Make the fork's anticipated upstream state concrete while Joshua's reviews/merges remain pending. This is a **draft baseline proposal**, not evidence that upstream has merged anything. The proposed file tree is exactly JoshuaJewell/main at `52feb605176a4b140f4b40c2b74d2f18adec1f72` plus: - upstream #59 at `3c9d352e1f6c85353c2d78ef1ebf28e5fb64c263` - upstream #60 at `ea974a2328f7f44ad9afeb2f10e5803cd4e31e62` - upstream #61 at `d9ea982dc97c958405e1ff6f57f1b0d8d9b221d1` Resulting tree: `b27daa7f1aa5fa84f320ddae522fa5f2a15ebef5`. ## Effect on fork main This deliberately removes fork-only content from the proposed active baseline: 74 files differ from current main. The existing main commit `491a3e0702ca8ca40e411860d6166523b298b3ac` is the first parent, and all three repair heads are additional parents, preserving the work in history. No existing branch is deleted or rewritten. Do not submit this integration branch upstream. **Before merging:** decide whether main should be the upstream-aligned baseline, with unfinished work isolated onto feature branches. This PR removes real fork features from the active tree, not just stale merge metadata. Recheck upstream and repair heads after Joshua's final decisions; their eventual file tree may change. ## Branch audit There were seven total branches when inspected: main and six additional branches. | Branches | Finding | Disposition | | --- | --- | --- | | repair/permanova-proofs | Clean pending upstream #59 | Keep until merged | | repair/documentation | Clean pending upstream #60 | Keep until merged | | repair/dada2-base-r | Clean pending upstream #61 | Keep until merged | | upstream/permanova-proofs, upstream/docs-community, upstream/dada2-base-r | All three have identical tree 2361ab862d0d26d8adea7fccc24d4b3741314cb3; old contaminated submissions, not independent WIP | Retire after baseline decision; no unique feature work needs completing on these branches | | main | Contains additional work beyond the repaired upstream contributions | Preserve and isolate selected changes | Compared with the old three branches, main changes only .gitignore, the differential fixture, the restored deferred-issues specification and removal of the duplicate root code of conduct. Their substantive work is preserved. ## Genuine work to isolate and finish These are code-review findings, not claims that the features have passed a fresh full validation. | Work | Evidence / remaining work | | --- | --- | | glmGamPoi dispersion | Actual implementation and configuration exist, but glmGamPoi is absent from renv.lock. Existing tests do not establish successful reference fits through this path. Pin dependencies and validate against trusted R fits before treating it as complete. | | Significance proof model | The Agda model makes noncomputed numbers unrepresentable, while the Julia documentation explicitly permits retained pairwise rows after an omnibus failure. Reconcile model and implementation; do not claim the proof establishes the Julia representation's behavior. | | Canonical JSON/JCS | Utility and fixtures exist; no production callers found beyond module inclusion/export. Establish conformance and choose an actual integration point before claiming end-to-end provenance. | | Memory occupancy | Measurement, algebra/proof integration, tests and scripts exist. Isolate the work, verify sibling dependency pins and real runtime measurements. This is distinct from biological occupancy modelling in issue #18. | | Wildcard-IP origin handling | Separate behavioral change in server middleware. Review address/port parsing and trust assumptions, then run host/origin, DNS-rebinding and CSRF regressions before submission. | | Plot title markup stripping | Small independent helper and usage change; suitable for a focused bugfix after checking its tests and behavior. | Repository layout changes, frontend major upgrades, community policies, automation and other tooling are separate fork-maintenance decisions, not prerequisites for the three upstream repairs. Do not carry forward the obsolete claim in fork AGENTS.md that locked ggplot2 requires tibble: the retained tibble dependency is used by installation/CI tooling. Open planning issues are not evidence of implemented branch WIP; several overlap or describe functionality already upstream. ## Validation - All six repair commits applied together without conflicts. - git diff --cached --check passes. - Local combined tree and GitHub-created tree match exactly. - Latest individual PR CI runs passed: #59 run 38976014775; #60 run 38998820680; #61 run 38998823755. - Combined baseline CI is separate and must be inspected before merging. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Joshua Jewell Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .editorconfig | 42 --- .github/CODE_OF_CONDUCT.md | 83 ----- .github/SECURITY.md | 89 ------ .github/dependabot.yml | 53 ---- .github/pull_request_template.md | 48 +-- .github/workflows/ci.yml | 233 ++------------ .github/workflows/claude-code-review.yml | 46 --- .github/workflows/claude.yml | 50 --- .github/workflows/occupancy.yml | 238 -------------- .gitignore | 9 - AGENTS.md | 35 --- CHANGELOG.md | 14 - .github/CONTRIBUTING.md => CONTRIBUTING.md | 0 Justfile | 51 --- R/_renv_dependencies.R | 2 + README.md | 129 ++------ .github/codecov.yml => codecov.yml | 0 config/defaults/pipeline.yml | 8 - docs/milestones/02-deferred-issues.md | 290 ----------------- docs/occupancy-2026-10.adoc | 167 ---------- docs/occupancy/GradedStages.agda | 96 ------ docs/occupancy/TypedPeak.agda | 45 --- .../reject/ConcurrentDriftIsZero.agda | 18 -- docs/occupancy/reject/ConcurrentIsSerial.agda | 14 - docs/occupancy/reject/SerialIsCost.agda | 14 - docs/occupancy/results-2026-10-14.yml | 63 ---- docs/occupancy/schedules.yml | 21 -- docs/occupancy/vsearch-asv-then-otu.sir | 13 - docs/occupancy/vsearch-concurrent.sir | 12 - docs/occupancy/vsearch-otu-then-asv.sir | 9 - frontend/bun.lock | 288 +++++++++++------ frontend/package.json | 20 +- frontend/src/api/types.ts | 2 - frontend/src/components/AnalysisChart.tsx | 3 +- frontend/src/components/CompositionPanel.tsx | 6 - .../src/components/TaxaCompositionChart.tsx | 7 - frontend/src/figure/charts.ts | 3 +- frontend/src/utils/text.test.ts | 24 -- frontend/src/utils/text.ts | 17 - frontend/src/views/TreeView.tsx | 6 +- frontend/vite.config.ts | 6 +- scripts/install.jl => install.jl | 12 +- install.sh | 2 +- launcher.sh | 295 ------------------ .../precompile_exec.jl => precompile_exec.jl | 0 proofs/agda/MetaManifold/All.agda | 1 - proofs/agda/MetaManifold/Permanova.agda | 12 +- proofs/agda/MetaManifold/Significance.agda | 106 ------- proofs/agda/README.md | 40 +-- scripts/check-occupancy-grades.jl | 72 ----- scripts/check-occupancy.sh | 103 ------ scripts/fetch-siblings.sh | 29 -- scripts/kyaml-format.sh | 112 ------- scripts/lint-occupancy-readings.sh | 40 --- src/MetaManifold.jl | 3 - src/analysis/analysis.jl | 43 +-- src/analysis/differential.jl | 211 +++---------- src/canonical.jl | 203 ------------ src/core/memory_probe.jl | 239 +------------- src/pipeline/dada2/dada2_functions.r | 46 ++- src/server/routes/analysis.jl | 9 +- src/server/routes/config.jl | 13 +- src/server/routes/databases.jl | 7 - src/server/server.jl | 65 +--- test/fixtures/jcs_oracle.jsonl | 11 - test/integration/test_pipeline.jl | 21 +- test/occupancy_disagreement_test.jl | 19 -- test/runtests.jl | 2 - test/unit/test_canonical.jl | 74 ----- test/unit/test_dada2_tables.R | 74 +++++ test/unit/test_differential.jl | 52 --- test/unit/test_install_pins.jl | 155 ++------- test/unit/test_occupancy_grade.jl | 270 ---------------- test/unit/test_server_security.jl | 47 --- 74 files changed, 496 insertions(+), 4166 deletions(-) delete mode 100644 .editorconfig delete mode 100644 .github/CODE_OF_CONDUCT.md delete mode 100644 .github/SECURITY.md delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/claude-code-review.yml delete mode 100644 .github/workflows/claude.yml delete mode 100644 .github/workflows/occupancy.yml delete mode 100644 AGENTS.md delete mode 100644 CHANGELOG.md rename .github/CONTRIBUTING.md => CONTRIBUTING.md (100%) delete mode 100644 Justfile rename .github/codecov.yml => codecov.yml (100%) delete mode 100644 docs/milestones/02-deferred-issues.md delete mode 100644 docs/occupancy-2026-10.adoc delete mode 100644 docs/occupancy/GradedStages.agda delete mode 100644 docs/occupancy/TypedPeak.agda delete mode 100644 docs/occupancy/reject/ConcurrentDriftIsZero.agda delete mode 100644 docs/occupancy/reject/ConcurrentIsSerial.agda delete mode 100644 docs/occupancy/reject/SerialIsCost.agda delete mode 100644 docs/occupancy/results-2026-10-14.yml delete mode 100644 docs/occupancy/schedules.yml delete mode 100644 docs/occupancy/vsearch-asv-then-otu.sir delete mode 100644 docs/occupancy/vsearch-concurrent.sir delete mode 100644 docs/occupancy/vsearch-otu-then-asv.sir delete mode 100644 frontend/src/utils/text.test.ts rename scripts/install.jl => install.jl (99%) delete mode 100755 launcher.sh rename scripts/precompile_exec.jl => precompile_exec.jl (100%) delete mode 100644 proofs/agda/MetaManifold/Significance.agda delete mode 100755 scripts/check-occupancy-grades.jl delete mode 100755 scripts/check-occupancy.sh delete mode 100755 scripts/fetch-siblings.sh delete mode 100755 scripts/kyaml-format.sh delete mode 100755 scripts/lint-occupancy-readings.sh delete mode 100644 src/canonical.jl delete mode 100644 test/fixtures/jcs_oracle.jsonl delete mode 100644 test/unit/test_canonical.jl create mode 100644 test/unit/test_dada2_tables.R delete mode 100644 test/unit/test_occupancy_grade.jl diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 7bff9e56..00000000 --- a/.editorconfig +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Editor configuration for this repository. The Julia sources indent with four -# spaces, the frontend with two, YAML with two, and every text file ends with -# a newline and no trailing whitespace. - -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.jl] -indent_style = space -indent_size = 4 - -[*.{jl}] -max_line_length = off - -[{*.ts,*.tsx,*.js,*.jsx,*.json,*.css,*.html}] -indent_style = space -indent_size = 2 - -[{*.yml,*.yaml,*.kyaml}] -indent_style = space -indent_size = 2 - -[*.md] -indent_style = space -indent_size = 2 -# Markdown's hard line breaks are two trailing spaces; keep them. -trim_trailing_whitespace = false - -[Makefile] -indent_style = tab - -[Justfile] -indent_style = space -indent_size = 4 diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md deleted file mode 100644 index b02898a4..00000000 --- a/.github/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,83 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement: the maintainer, [@JoshuaJewell](https://github.com/JoshuaJewell), using the contact details on their GitHub profile. All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series of actions. - -**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. - -For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. - -[homepage]: https://www.contributor-covenant.org -[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html -[Mozilla CoC]: https://github.com/mozilla/diversity -[FAQ]: https://www.contributor-covenant.org/faq -[translations]: https://www.contributor-covenant.org/translations diff --git a/.github/SECURITY.md b/.github/SECURITY.md deleted file mode 100644 index f97f2309..00000000 --- a/.github/SECURITY.md +++ /dev/null @@ -1,89 +0,0 @@ - - -# Security policy - -## Deployment model, stated first - -MetaManifold is a **single-user, no-authentication** application: a Julia -server with a browser UI, designed to run on localhost or on a host trusted -by its one operator. The server carries real defences — a browser-origin -allowlist with a DNS-rebinding check, strict Host checking on loopback binds, -inert serving of uploaded report files, and Oxygen's docs and metrics pages -switched off — but **exposing it beyond its intended deployment is the -operator's risk, not a supported configuration.** Denial-of-service against -an internet-exposed instance is out of scope, as is anything that follows -from putting an unauthenticated API on a public network after ignoring the -deployment guidance in the README's *Deployment* section. - -## What is in scope - -- The HTTP surface: the origin allowlist and Host checks - (`src/server/server.jl`), the static-file and report-upload serving - (`_file_middleware`), SPA routing, and every route under - `src/server/routes/`. -- How configuration and primer values reach the external tools: pipeline - options and filenames become command lines; injection through them is a - finding here. -- Path handling: the file routes, report item paths, and study/run names as - path components. -- Installer and toolchain integrity: every download is SHA256-pinned - (`config/defaults/tool_versions.yml`); a tampered pin, an unchecked - substitute, or a way to make the installer fetch unverified content is a - finding. -- Provenance integrity: `attestation.yml` and the content hashes it records; - anything that can make a run attest to bytes it did not use. -- The dependency manifests: `Manifest.toml`, `renv.lock`, `frontend/bun.lock`. - -## What is out of scope - -- Vulnerabilities **in the external tools themselves** (cutadapt, FastQC, - MultiQC, DADA2, swarm, vsearch, cd-hit-est, MAFFT, trimAl, IQ-TREE, RAxML, - gappa, and the R packages): report those to their own projects. A way this - repository invokes them unsafely is in scope; their internal bugs are not. -- Julia, R, bun and DuckDB runtime CVEs: report upstream; this project pins - versions and will pick up fixes through its normal dependency updates. -- Internet-exposed DoS, as above. - -## Supported versions - -The default branch only. There are no backported releases; a fix lands on -`main` behind the usual CI gates. The version that matters for any report is -the git commit plus the pinned toolchain — and every run records exactly that -in its `attestation.yml`. - -## How to report - -**Preferred channel: private vulnerability reporting.** The repository has -it enabled; the *Report a vulnerability* button on the Security tab reaches -the maintainers privately. Use it. - -**By email:** contact **developer@joshuajewell.dev** (Jonathan Jewell, the project -owner) with `[security]` in the subject line. - -Include, where they apply: - -- what you found, and the impact you think it has; -- the endpoint, route, or pipeline stage it affects; -- the deployment mode — localhost, SSH tunnel, or split frontend/API; -- reproduction steps, or the request that triggers it; -- the versions block from the affected run's `attestation.yml` — it names the - exact commit and toolchain, which usually settles the first two replies. - -Reports are acknowledged when triaged, fixed on `main` with the normal CI -gates, and disclosed on the reporter's preferred schedule. Credit is offered -unless you prefer otherwise. - -## Mathematical foundations - -This project carries machine-checked proofs of some of its statistics, and it -treats **unsound mathematics as a reportable weakness of its own kind**: a -proof that is vacuous, a theorem that does not say what the code relies on, or -a statistical claim the arithmetic does not support. If you find one, that is -a finding — send it to **j.d.a.jewell@open.ac.uk**, where the -proof-theoretic side of the project lives. Questions and open debate about -the foundations belong in the -[mathematical foundations discussion](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/149); -private concerns about soundness belong in a report. diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 2f06b99e..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Dependabot version updates. Until this file landed, only security updates -# ran (those need no configuration), and those failed for a separate reason: -# the "Dependabot Updates" runs of 2026-09-26..27 tried to fix alerts against -# fixtures under test/doi using the npm_and_yarn updater, which refuses a -# Bun-managed project ("bun.lock but no package-lock.json"). Those fixtures -# have since left the repository and both alerts are fixed. -# -# What cannot be covered here: Julia (Project.toml / Manifest.toml) and R -# (renv.lock) are not Dependabot package-ecosystems. Their pins live in -# config/defaults/tool_versions.yml and are asserted by -# test/unit/test_install_pins.jl and the CI workflow instead. -# -# The workflows pin every action to a full commit SHA (this account's -# allowlist policy, asserted by test/unit/test_install_pins.jl). Dependabot -# advances those pins to the current SHA of the tracked version, so the -# pinning policy itself survives updates. -# -# The #124 rule: minor and patch updates are batched, because they are -# interchangeable in kind; a MAJOR (React 18->19, vite 6->8, TypeScript -# 5->7 -- which #124 landed as one grouped PR, with the build already red -# until #140 repaired it) is a migration, not an update. Every group below -# therefore names its update-types as minor and patch ONLY: an update that -# matches no group opens its own PR, carrying its own review and its own CI -# run. test/unit/test_install_pins.jl asserts this shape, so a new group -# cannot silently go back to batching majors. -version: 2 -updates: - # The frontend is Bun-managed: package.json plus bun.lock, no npm/yarn/pnpm - # lockfile, so it must be declared as the "bun" ecosystem. - - package-ecosystem: "bun" - directory: "/frontend" - schedule: - interval: "weekly" - commit-message: - prefix: "chore(deps)" - groups: - # Minor and patch in one weekly PR; majors arrive one PR each. - frontend-minor-and-patch: - patterns: ["*"] - update-types: ["minor", "patch"] - - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - commit-message: - prefix: "chore(deps)" - groups: - # Same rule as the frontend: a new major of an action (checkout - # v4.4.0 -> v7.0.1 came through #123) gets a PR of its own. - actions-minor-and-patch: - patterns: ["*"] - update-types: ["minor", "patch"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a17d0de6..d7f121ff 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,43 +1,19 @@ -## Summary +## Change and reason - + -Closes # +## Validation -## Type of change + -- [ ] 🐛 Bug fix (non-breaking change that fixes an issue) -- [ ] ✨ New feature (non-breaking change that adds functionality) -- [ ] 💥 Breaking change (would change existing behaviour, a config key or an API response) -- [ ] 📊 Statistics fix (a method returned a wrong number, or a result it could not support) -- [ ] 📖 Documentation -- [ ] 🧹 Refactor / tech debt (behaviour-preserving) -- [ ] ⚡ Performance -- [ ] 🔧 Build / CI / tooling +## Scope -## How has this been verified? + - +## Contribution checks -Tested with Julia `…`, R `…` and bun `…`. - -## What this leaves out - - - -## Checklist - -- [ ] My branch starts at the current tip of `main` (`behind_by` is `0`; see CONTRIBUTING.md) and has no merge commits from `main`. -- [ ] I ran `julia --project=. test/runtests.jl` (and `cd frontend && bun run test` for frontend changes) and they pass. -- [ ] CI ran on this PR and passes. -- [ ] No generated or vendored files, and no lockfile change the work does not need (or the reason is given above). -- [ ] New code is AGPL-3.0, or MPL-2.0 with an SPDX identifier in each file I wrote. I left existing file headers, `LICENSE`, `CITATION.cff` and the licence and acknowledgement sections of `README.md` unchanged. -- [ ] Every number shown to a user is computed from their data. When a method cannot run it raises an explicit error. -- [ ] Docs are updated, and no claim in them overstates what the code does. - -## Notes for reviewers - - +- [ ] I followed [CONTRIBUTING.md](../CONTRIBUTING.md). +- [ ] This branch starts at current upstream main and contains no merges from fork main. +- [ ] CI has run and passes. +- [ ] Existing licensing, citation and acknowledgement text is unchanged. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5f4cd1d..5bc3a6b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,10 @@ name: CI -# This fork's account allows only GitHub's own actions (actions/*) and its own, -# each pinned to a full commit SHA. Upstream's julia-actions, oven-sh/setup-bun -# and codecov/codecov-action are none of these, and with them this workflow -# ended in startup_failure on every run. So Julia and Bun are installed here -# from the archives config/defaults/tool_versions.yml pins, each refused unless -# it hashes to the pinned SHA256, and the Julia depot is cached with -# actions/cache. Coverage is kept as a workflow artifact, since the Codecov -# project is upstream's. test/unit/test_install_pins.jl asserts that every -# `uses:` in .github/workflows is allowed by that policy. - on: push: branches: [main] - # Any base branch, so a PR stacked on another one is tested too. pull_request: + branches: [main] permissions: contents: read @@ -26,95 +16,6 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'push' }} jobs: - # A pull request that touches nothing the suite exercises -- a README edit, a - # wiki page, a template, a licence record -- does not need the ~19-minute - # Julia suite or the proof checker to pass over it. The gate decides, and the - # two jobs below carry `if:` on its output; when it says not relevant, GitHub - # posts `skipped` for them, and a skipped check satisfies a required one. The - # gate itself must never be skipped -- it is the check that vouches for the - # others' absence, so Branch-Floor requires it too. - # - # The trigger stays unfiltered for a reason this repository has already - # sprang once (#116): a path-filtered `pull_request` trigger does not run at - # all on a PR that matches no path, so a required check from that workflow - # never posts and can never be satisfied. Running the cheap gate on every PR - # and letting it skip the heavy work is the pattern the occupancy workflow - # (#125) established; this applies it to the main suite. - # - # The relevant-path list is deliberately the positive form: anything not - # listed is not relevant, so a new directory joins the suite by being added - # here, not by being forgotten. These tests also read the repository's own - # shape -- the pin tests parse `.github/workflows/`, the root-shape guard - # reads the tracked tree, and the occupancy reading's Agda tree lives under - # `docs/occupancy/` -- so those paths are relevant even though they are - # documentation-shaped. In a `case` pattern, `*` crosses `/`. - gate: - name: Code relevance - runs-on: ubuntu-24.04 - timeout-minutes: 10 - outputs: - relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} - steps: - # A push only starts this workflow on main (the branch filter above), so - # the gate has nothing to decide: every push is relevant. - - name: A push is already branch-filtered - id: push - if: github.event_name == 'push' - run: echo 'relevant=true' >> "$GITHUB_OUTPUT" - - # Full history, because the three-dot diff names the merge base of the - # PR's base tip and its merge commit, and a stacked PR's merge base sits - # further back still. - - name: Checkout the change - if: github.event_name == 'pull_request' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # nothing here pushes - fetch-depth: 0 - - # --no-renames, so a file renamed out of a listed path is reported as a - # deletion under its original name and still counts. -z with a - # NUL-delimited read, so a pathname Git would quote or that contains a - # newline arrives raw and matches the pattern. The base SHA travels - # through env, not template interpolation into the script. - - name: Does the change touch anything the suite exercises? - id: pr - if: github.event_name == 'pull_request' - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - relevant=false - # Every pattern is quoted, so the glob survives array assignment and - # fires at `case` time, against the diff -- an unquoted `src/*` would - # expand against the checkout the runner happens to be standing in. - patterns=('src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*' - 'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml' - 'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' - 'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc' - '.github/workflows/*') - # The diff goes to a file first because a process substitution hides - # git diff's exit status: an unavailable comparison SHA (a force-push - # on the base branch, for instance) or a repository error would read - # as "no changes", and the gate would post relevant=false -- skipping - # the Julia suite and the proof checker, whose `skipped` conclusion - # satisfies a required check -- so a relevant PR would pass silently. - # A failed diff must fail the gate itself: that is the one failure - # the required-check design cannot afford to skip. As in the - # occupancy gate, which this mirrors. - changed="$RUNNER_TEMP/ci-changed-files" - if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then - echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 - exit 1 - fi - while IFS= read -r -d '' file; do - for pattern in "${patterns[@]}"; do - case "$file" in - $pattern) relevant=true; break 2 ;; - esac - done - done < "$changed" - echo "relevant=$relevant" >> "$GITHUB_OUTPUT" - test: # The name is a fixed string, it carries no version, and the job has NO matrix. # Both are required, and the first without the second is a trap that has already @@ -139,10 +40,6 @@ jobs: # silently. If a matrix is ever genuinely wanted here, the required context must move # to a matrix-free aggregator job first. name: Julia tests - # Runs only when the gate above found a relevant path; on a docs-only PR - # this job posts `skipped`, which satisfies the required check. - needs: gate - if: needs.gate.outputs.relevant == 'true' # Explicit, not ubuntu-latest: the R pin below names an apt package revision built # for 24.04, and a runner that silently rolled to the next LTS would take that pin # with it. `runs-on` cannot read `env`, so this is a literal. @@ -156,45 +53,22 @@ jobs: # the environment, so those commands pass --no-init-file instead. env: RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" - # Pinned rather than tracking latest stable, because a pinned Manifest.toml - # resolved by an unpinned Julia is not a reproducible build. These must equal - # runtimes.julia.archives.linux-x86_64 in the pin file, whose version must - # equal julia_version in Manifest.toml; they cannot be read from the pin - # file, since nothing can be read before Julia exists. - # test/unit/test_install_pins.jl fails if any of them disagree. - JULIA_URL: https://julialang-s3.julialang.org/bin/linux/x64/1.12/julia-1.12.5-linux-x86_64.tar.gz - JULIA_SHA256: 41b84d727e4e96fbf3ed9e92fa195d773d247b9097f73fad688f8b699758bae7 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # nothing here pushes + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - name: Install Julia - run: | - source scripts/ci/fetch_pinned.sh - fetch_pinned "$JULIA_URL" julia.tar.gz - echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - - sudo mkdir -p /opt/julia - sudo tar xzf julia.tar.gz -C /opt/julia --strip-components=1 - rm julia.tar.gz - echo /opt/julia/bin >> "$GITHUB_PATH" - - # The registry, packages, artifacts and precompiled caches, keyed on the - # Manifest (which also names the Julia version). Saved once instantiation - # has succeeded, so a failed run cannot store a half-built depot. - - name: Restore the Julia depot - id: julia-depot - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up Julia + uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 with: - path: | - ~/.julia/registries - ~/.julia/packages - ~/.julia/artifacts - ~/.julia/compiled - key: julia-depot-${{ runner.os }}-${{ hashFiles('Manifest.toml') }} - restore-keys: | - julia-depot-${{ runner.os }}- + # Pinned rather than tracking latest stable, because a pinned Manifest.toml + # resolved by an unpinned Julia is not a reproducible build. This must equal + # julia_version in Manifest.toml; it cannot be read from the pin file, since + # nothing can be read before Julia exists. test/unit/test_install_pins.jl + # fails if the three ever disagree. + version: "1.12.5" + + - name: Cache Julia packages + uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0 # Every external version CI installs is read from the committed pin file, so CI # and a developer's machine cannot drift apart. A temporary environment is used @@ -210,10 +84,7 @@ jobs: pins = YAML.load_file("config/defaults/tool_versions.yml") open(ENV["GITHUB_ENV"], "a") do io println(io, "R_APT_VERSION=", pins["runtimes"]["r"]["apt_version"]) - bun = pins["toolchain"]["bun"] - println(io, "BUN_VERSION=", bun["version"]) - println(io, "BUN_URL=", bun["archives"]["linux-x86_64"]["url"]) - println(io, "BUN_SHA256=", bun["archives"]["linux-x86_64"]["sha256"]) + println(io, "BUN_VERSION=", pins["toolchain"]["bun"]["version"]) println(io, "CUTADAPT_SPEC=cutadapt==", pins["tools"]["cutadapt"]["version"]) println(io, "MULTIQC_SPEC=multiqc==", pins["tools"]["multiqc"]["version"]) fastqc = pins["tools"]["fastqc"]["archives"]["any"] @@ -268,7 +139,7 @@ jobs: # the restore runs as root. The run id makes each key unique so every attempt # saves its progress; restore-keys picks the newest match. - name: Restore the renv cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /opt/renv-cache key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }} @@ -277,12 +148,6 @@ jobs: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}- # R_CLI_DYNAMIC and TERM make renv print progress one line at a time in the log. - # This verification list -- and the RCall visibility check below -- must - # track renv.lock: restore() installs only what the lock records, so a - # package removed from the lock fails here at verification, nowhere near - # the code that stopped using it. MEASURED in this PR: dplyr left the lock - # and this check failed until both lists were updated. Keep them in the - # same commit as any lock change. - name: Install R packages env: RENV_PATHS_CACHE: /opt/renv-cache @@ -293,6 +158,9 @@ jobs: sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)' Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' + - name: Check DADA2 CSV assembly + run: Rscript --vanilla test/unit/test_dada2_tables.R + # Saved on failure too, since the packages built before a failure are the ones # the next attempt should reuse. Root writes the cache, so it is made readable first. - name: Make the renv cache readable @@ -301,7 +169,7 @@ jobs: - name: Save the renv cache if: always() - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /opt/renv-cache key: renv-${{ runner.os }}-R${{ env.R_APT_VERSION }}-${{ hashFiles('renv.lock') }}-${{ github.run_id }} @@ -356,28 +224,10 @@ jobs: - name: Instantiate Julia project run: julia --project=. -e 'import Pkg; Pkg.instantiate()' - - name: Save the Julia depot - if: steps.julia-depot.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: - path: | - ~/.julia/registries - ~/.julia/packages - ~/.julia/artifacts - ~/.julia/compiled - key: julia-depot-${{ runner.os }}-${{ hashFiles('Manifest.toml') }} - - # Downloaded against the pinned URL and refused unless it hashes to the pinned - # SHA256, as install.jl does when no bun of this version is on PATH. - - name: Install Bun - run: | - source scripts/ci/fetch_pinned.sh - fetch_pinned "$BUN_URL" bun.zip - echo "$BUN_SHA256 bun.zip" | sha256sum -c - - unzip -q bun.zip - sudo mv bun-linux-x64/bun /usr/local/bin/bun - rm -rf bun.zip bun-linux-x64 - test "$(bun --version)" = "$BUN_VERSION" + bun-version: ${{ env.BUN_VERSION }} - name: Build frontend working-directory: frontend @@ -396,8 +246,6 @@ jobs: run: julia --project=. -e 'import Pkg; Pkg.build("RCall")' - name: Verify R packages visible from RCall - # The second of the two lists that must track renv.lock; see the - # Install R packages step above for the coupling. run: | julia --project=. -e ' using RCall @@ -444,29 +292,19 @@ jobs: fi exit "$status" - # What julia-processcoverage did: the .cov files under src/ to lcov.info, with - # CoverageTools pinned to an exact release. - name: Process coverage - run: | - julia -e ' - using Pkg - Pkg.activate(temp = true) - Pkg.add(name = "CoverageTools", version = "1.4.2") - using CoverageTools - LCOV.writefile("lcov.info", process_folder("src"))' - - # The Codecov project (codecov.yml, the README badge) is upstream's, so the fork - # keeps its coverage as an artifact of the run instead. - - name: Keep coverage - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1.2.2 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2 with: - name: lcov - path: lcov.info - retention-days: 14 + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + slug: JoshuaJewell/MetaManifold-WebUI + fail_ci_if_error: false - # Machine-checked proofs of the Benjamini-Hochberg adjustment, the TSS/RLE - # scaling algebra, PERMANOVA's permutation bookkeeping and the significance - # reporting discipline (proofs/agda, proofs/agda/README.md). + # Machine-checked proofs of the Benjamini-Hochberg adjustment and the TSS/RLE + # scaling algebra (proofs/agda, proofs/agda/README.md). # Agda publishes no 2.6.4.3 binary, and agda-stdlib 2.1 requires Agda 2.6.4.x, # so the pair comes from Debian trixie, which packages exactly that pair with # pre-built library interfaces. The image is pinned by digest and the two @@ -476,18 +314,13 @@ jobs: # reads them. agda: name: Agda proofs - # As the Julia suite above: only when the gate found a relevant path. - needs: gate - if: needs.gate.outputs.relevant == 'true' runs-on: ubuntu-24.04 container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb timeout-minutes: 30 env: DEBIAN_FRONTEND: noninteractive steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # nothing here pushes + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install Agda and the standard library run: | diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index c59ba873..00000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize, ready_for_review, reopened] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - if: github.event.pull_request.head.repo.full_name == github.repository - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - fetch-depth: 1 - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@374f254f5efaf968abfe15d399fd90972e8138b9 # v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review --comment ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - claude_args: '--allowedTools "mcp__github_inline_comment__create_inline_comment"' - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index c699b80f..00000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - fetch-depth: 1 - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@374f254f5efaf968abfe15d399fd90972e8138b9 # v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - - # This is an optional setting that allows Claude to read CI results on PRs - additional_permissions: | - actions: read - - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr *)' - diff --git a/.github/workflows/occupancy.yml b/.github/workflows/occupancy.yml deleted file mode 100644 index 0068dab5..00000000 --- a/.github/workflows/occupancy.yml +++ /dev/null @@ -1,238 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# The occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): -# * the two-thread disagreement test (test/occupancy_disagreement_test.jl) on -# the pinned vsearch, and -# * Session IR's counted bytes against the typed peak computed in Agda from -# the pinned proof siblings (scripts/check-occupancy.sh). -# -# This account allows only GitHub's own actions and its own, pinned by SHA, so -# everything here installs from pinned URLs checked against pinned SHA256s, as -# ci.yml now does too. The disagreement test also runs in ci.yml's full suite; -# it runs here as well so that a change to the probe gets its own check. -# -# #116 merged with this workflow's "Occupancy typed peak" check red (its squash -# carried a draft `TH.[_]` placeholder; #117 fixed the proof), because nothing -# required the check to pass first -- and nothing COULD have: a check posts -# only when its workflow runs, and the pull_request trigger was path-filtered, -# so requiring these checks would have deadlocked every PR that touches no -# occupancy file. A required check that never posts can never be satisfied; the -# trap ci.yml's header records for its own job names. -# -# So the pull_request path filter moved into the `Occupancy relevance` gate -# below. The workflow now runs on EVERY pull request; the gate decides whether -# the change touches the occupancy reading; the two heavy jobs run only when it -# does, and post their checks as `skipped` otherwise, which a required check -# counts as satisfied. The gate itself posts on every PR too, and it is the one -# check of the three that must be required without exception: if the gate -# failed, its dependents would be skipped and would satisfy their requirement -# silently. The push trigger keeps its native paths filter, so a push to main -# still starts this workflow only when it should. -name: Occupancy reading - -on: - push: - branches: [main] - paths: - - src/core/memory_probe.jl - - src/core/provenance.jl - - src/pipeline/tools.jl - - src/server/routes/pipeline.jl - - test/occupancy_disagreement_test.jl - - test/unit/test_occupancy_grade.jl - - scripts/check-occupancy-grades.jl - - scripts/lint-occupancy-readings.sh - - docs/occupancy/** - - docs/occupancy-2026-10.adoc - - scripts/check-occupancy.sh - - scripts/fetch-siblings.sh - - .github/workflows/occupancy.yml - # No paths filter, on purpose (see above): the gate must post its check on - # every PR, whichever files the PR touches. Any base branch, so a PR stacked - # on another one is gated too. - pull_request: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name != 'push' }} - -env: - # Julia 1.12.5, the version Manifest.toml was resolved with. The checksum is - # julialang's own (bin/checksums/julia-1.12.5.sha256), as the official Docker - # image (docker-library/julia a39df01) records it. - JULIA_URL: https://julialang-s3.julialang.org/bin/linux/x64/1.12/julia-1.12.5-linux-x86_64.tar.gz - JULIA_SHA256: 41b84d727e4e96fbf3ed9e92fa195d773d247b9097f73fad688f8b699758bae7 - -jobs: - # Decides whether the PR touches the occupancy reading at all. The file list - # is the pull_request half of the old paths filter, moved here because a - # pinned-actions workflow cannot use a third-party paths-filter action (the - # account's allow-list refuses one) -- plain `case` globs are all it takes. - # In a case pattern `*` crosses `/`, so `docs/occupancy/*` is the old - # `docs/occupancy/**`. - gate: - name: Occupancy relevance - runs-on: ubuntu-24.04 - timeout-minutes: 10 - outputs: - relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} - steps: - # A push only starts this workflow when a `paths:` file changed (the - # trigger filter above), so the gate has nothing to decide on pushes. - - name: A push is already path-filtered - id: push - if: github.event_name == 'push' - run: echo 'relevant=true' >> "$GITHUB_OUTPUT" - - # Full history, because the three-dot diff names the merge base of the - # PR's base tip and its merge commit -- a shallow checkout does not have - # both, and a stacked PR's merge base sits further back still. - - name: Checkout the change - if: github.event_name == 'pull_request' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # nothing here pushes - fetch-depth: 0 - - - name: Does the change touch the occupancy reading? - id: pr - if: github.event_name == 'pull_request' - env: - # zizmor flags template expressions interpolated into run scripts - # (template-injection), and the gate script is easier to run locally - # with the value in an environment variable in any case. The base SHA - # is GitHub-supplied, but the indirection costs nothing. - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - relevant=false - patterns=(src/core/memory_probe.jl src/core/provenance.jl src/pipeline/tools.jl - src/server/routes/pipeline.jl test/occupancy_disagreement_test.jl - test/unit/test_occupancy_grade.jl scripts/check-occupancy-grades.jl - scripts/lint-occupancy-readings.sh 'docs/occupancy/*' - docs/occupancy-2026-10.adoc scripts/check-occupancy.sh - scripts/fetch-siblings.sh .github/workflows/occupancy.yml) - # The diff goes to a file first because a process substitution hides - # git diff's exit status: an unavailable comparison SHA or a repository - # error would read as "no changes", and the gate would post - # relevant=false -- skipping the two occupancy jobs, whose `skipped` - # conclusion satisfies a required check -- so a relevant PR would - # pass silently. A failed diff must fail the gate itself: that is - # the one failure the required-check design cannot afford to skip. - changed="$RUNNER_TEMP/occupancy-changed-files" - if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then - echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 - exit 1 - fi - while IFS= read -r -d '' file; do - for pattern in "${patterns[@]}"; do - case "$file" in - $pattern) relevant=true; break 2 ;; - esac - done - done < "$changed" - echo "relevant=$relevant" >> "$GITHUB_OUTPUT" - - disagreement: - needs: gate - if: needs.gate.outputs.relevant == 'true' - name: Occupancy disagreement test - runs-on: ubuntu-24.04 - timeout-minutes: 45 - env: - # As in ci.yml: the root .Rprofile would otherwise bootstrap renv. - RENV_CONFIG_AUTOLOADER_ENABLED: "FALSE" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Julia - run: | - source scripts/ci/fetch_pinned.sh - fetch_pinned "$JULIA_URL" julia.tar.gz - echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - - sudo mkdir -p /opt/julia - sudo tar xzf julia.tar.gz -C /opt/julia --strip-components=1 - rm julia.tar.gz - echo /opt/julia/bin >> "$GITHUB_PATH" - - # RCall builds against R. The occupancy test uses no R package. - - name: Install R - run: | - sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends r-base-core - - - name: Instantiate the project - run: julia --project=. -e 'using Pkg; Pkg.instantiate()' - - # The pinned vsearch (config/defaults/tool_versions.yml), refused unless it - # hashes to the pinned SHA256, as ci.yml installs it. - - name: Install vsearch - run: | - eval "$(julia --project=. -e ' - using YAML - v = YAML.load_file("config/defaults/tool_versions.yml")["tools"]["vsearch"] - a = v["archives"]["linux-x86_64"] - println("VSEARCH_VERSION=", v["version"]); println("VSEARCH_URL=", a["url"]); println("VSEARCH_SHA256=", a["sha256"])')" - source scripts/ci/fetch_pinned.sh - fetch_pinned "$VSEARCH_URL" vsearch.tar.gz - echo "$VSEARCH_SHA256 vsearch.tar.gz" | sha256sum -c - - tar xzf vsearch.tar.gz --warning=no-unknown-keyword - sudo mv "vsearch-$VSEARCH_VERSION-linux-x86_64/bin/vsearch" /usr/local/bin/vsearch - rm -rf vsearch.tar.gz "vsearch-$VSEARCH_VERSION-linux-x86_64" - vsearch --version - - - name: Three orders of the vsearch stage - run: julia --project=. -t 2 test/occupancy_disagreement_test.jl - - # Fork issue #105: the grades the recorded run's windows imply, read back - # against its own measurement (docs/occupancy/schedules.yml). - - name: Graded reading of the recorded run - run: julia --project=. scripts/check-occupancy-grades.jl - - # And no production code reads a peak without those grades. - - name: Occupancy readings are graded - run: bash scripts/lint-occupancy-readings.sh - - typed-peak: - needs: gate - if: needs.gate.outputs.relevant == 'true' - name: Occupancy typed peak - runs-on: ubuntu-24.04 - # The Agda 2.6.4.3 Debian trixie packages, in a digest-pinned image, as the - # upstream proof job uses. The siblings need no standard library. - container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb - timeout-minutes: 20 - env: - DEBIAN_FRONTEND: noninteractive - steps: - # git before checkout, so the checkout is a repository and the siblings can be fetched. - - name: Install git, curl and Agda - run: | - apt-get update -qq - apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 - agda --version | grep -qx 'Agda version 2.6.4.3' - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false # the siblings are public; nothing here pushes - - # The Session IR stepper is Julia (standard library only). - - name: Install Julia - shell: bash - run: | - source scripts/ci/fetch_pinned.sh - fetch_pinned "$JULIA_URL" julia.tar.gz - echo "$JULIA_SHA256 julia.tar.gz" | sha256sum -c - - mkdir -p /opt/julia - tar xzf julia.tar.gz -C /opt/julia --strip-components=1 - rm julia.tar.gz - echo /opt/julia/bin >> "$GITHUB_PATH" - - - name: Counted bytes against typed peak - shell: bash - run: bash scripts/check-occupancy.sh diff --git a/.gitignore b/.gitignore index c35a1c8b..6ef7e458 100644 --- a/.gitignore +++ b/.gitignore @@ -250,11 +250,6 @@ vignettes/*.pdf ## Documentation kept local except published release notes docs/* !docs/release-notes/ -# and the dated occupancy reading (MMW-01), whose table the gate checks -!docs/occupancy-2026-10.adoc -!docs/occupancy/ -# and the deferred-features spec the backlog issues link to (#3, #4, #5) -!docs/milestones/ # translation temp files @@ -325,7 +320,6 @@ archive/ !.github !.github/** !.Rprofile -!.editorconfig # Brainstorming visual-companion scratch (mock-ups, server state) .superpowers/ @@ -341,6 +335,3 @@ dev/server.log # Agda interface files (proofs/agda) proofs/agda/_build/ -# Proof siblings fetched by scripts/fetch-siblings.sh, and Agda interface files -/_build/ -*.agdai diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index d4164769..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,35 +0,0 @@ -# AGENTS.md - -Instructions for coding agents working in this repository. Read the contribution -rules in `.github/CONTRIBUTING.md` first; this file records only the traps that -are invisible until they bite. - -## renv.lock and CI's hardcoded R-package lists are coupled - -CI installs R from `renv.lock` (`renv::restore()`), but `.github/workflows/ci.yml` -names the expected packages in TWO hardcoded verification lists: - -- the `Install R packages` step's `require()` check ("failed to install"), and -- the `Verify R packages visible from RCall` step. - -Removing a package from `renv.lock` therefore fails the run at install -verification, nowhere near the code that stopped using it. MEASURED on -JoshuaJewell/MetaManifold-WebUI#57 (2026-10-15): dplyr left the lock and both -checks failed until the lists were updated in the same PR. Whenever a change -touches `renv.lock`, grep `.github/workflows/ci.yml` for every added or removed -package name and update both lists in the same commit. - -Corollary, measured the same way: `tibble` cannot leave the lock while dada2 is -in it. `dada2 -> ggplot2 -> tibble` is dada2's own dependency chain, independent -of anything this project's code does; the lock's `Requirements` arrays are not -evidence (only a handful of records carry one). - -## Commit signing needs the SSH agent, not a passphrase prompt - -`commit.gpgsign` is on, with a passphrase-protected SSH key. A non-interactive -shell has no askpass and no terminal for a prompt, so signing fails with a -misleading "failed to write commit object" unless the agent is reachable: -`SSH_AUTH_SOCK=/tmp/lcb-ssh-agent.sock` (the fixed socket; `~/.profile` exports -it when the socket exists). Never work around this with -`commit.gpgsign=false` — check the agent first with -`SSH_AUTH_SOCK=/tmp/lcb-ssh-agent.sock ssh-add -l`. diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 9f52ed7f..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,14 +0,0 @@ - - -# Changelog - -Release notes live under `docs/release-notes/`, one file per release: - -- [v0.1.0](docs/release-notes/v0.1.0.md) - -This file is the root index the RSR scheme expects; the notes themselves are -kept where they have always been, and a new release adds a line here in the -same commit that adds its file there. diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 100% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md diff --git a/Justfile b/Justfile deleted file mode 100644 index b1cbe4da..00000000 --- a/Justfile +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# MetaManifold's task entry (RSR scheme: a thin Justfile at the root; the -# heavy work lives in the commands it names). Every recipe delegates to the -# repository's own documented entry points -- install.sh, start.sh, the test -# runner -- so there is exactly one place where each of those is defined and -# this file never re-implements them. - -# The default recipe lists what exists; `just --list` does the same. -default: - @just --list - -# The documented installer: Julia (pinned), R, the external tools, the -# frontend build. Extra args pass through to install.sh (--update, -# --modify, --sysimage). -install *ARGS: - bash install.sh {{ARGS}} - -# The documented server entry point. Serves the UI on port 8080. -serve: - bash start.sh - -# The full unit suite. Name files to run a subset: -# just test test_install_pins.jl test_server_security.jl -test *FILES: - julia --project=. test/runtests.jl {{FILES}} - -# The pin suite alone: toolchain pins, lockfile agreement, workflow shape, -# and the root-shape allowlist. -pins: - julia --project=. test/runtests.jl test_install_pins.jl - -# The Agda proof tree's audit (no postulates, no holes, everything reached -# from All.agda). The type-check itself runs in CI's Agda proofs job. -proofs: - bash proofs/tests/axiom-audit.sh - -# The frontend's own checks. -frontend-test: - cd frontend && bun run test - -frontend-typecheck: - cd frontend && bun run typecheck - -frontend-build: - cd frontend && bun run build - -# The git-signing agent's state, as the launcher sees it. -agent-status: - ./launcher.sh --agent status diff --git a/R/_renv_dependencies.R b/R/_renv_dependencies.R index 1746380f..20394ab6 100644 --- a/R/_renv_dependencies.R +++ b/R/_renv_dependencies.R @@ -3,5 +3,7 @@ if (FALSE) { library(dada2) library(vegan) + # Retain while installer and CI explicitly verify this package. + library(tibble) library(MASS) } diff --git a/README.md b/README.md index db9a5745..c4307a2d 100644 --- a/README.md +++ b/README.md @@ -74,16 +74,9 @@ Once a run completes, analysis is performed on request through the web UI, both - Pipeline stage read-count summaries - NMDS ordination (Bray-Curtis, via R/vegan) - PERMANOVA, with PERMDISP to check for differences in dispersion (via R/vegan) -- Differential abundance between two conditions: a negative-binomial model per taxon (via R/MASS) with Benjamini-Hochberg adjusted p-values (see [Differential abundance](#differential-abundance)) Counts may be normalised before analysis (none, rarefaction to a fixed or auto-resolved depth, or scaling with ranked subsampling (SRS), which reaches the depth by scaling counts and keeps more of the community structure), and contamination-flagged taxa may be included or excluded. Ordinations can also apply a Hellinger transform before the Bray-Curtis dissimilarity. Composition charts can be drawn as a grid faceted on any two of run, group and sub-group. All analysis charts are returned as Plotly JSON and rendered interactively in the browser. -## Documentation - -Longer documentation for users, maintainers, developers and theorists is kept as a wiki in -[metadatastician/MetaManifold-Evidence, under `docs/wikis/`](https://github.com/metadatastician/MetaManifold-Evidence/tree/main/docs/wikis). -Start at its [Home page](https://github.com/metadatastician/MetaManifold-Evidence/blob/main/docs/wikis/Home.md). - ## Prerequisites - **Julia** 1.12 (installed by `install.sh` if missing, and pinned to the Manifest version for this directory) @@ -101,7 +94,7 @@ bash install.sh ``` `install.sh` installs Julia (via juliaup) if missing, checks for R, then hands -off to `scripts/install.jl`, which installs the Julia dependencies, locates or downloads +off to `install.jl`, which installs the Julia dependencies, locates or downloads each external tool (cutadapt, FastQC, MultiQC, vsearch, cd-hit-est, and the phylogeny tools MAFFT, trimAl, IQ-TREE, RAxML and gappa), and reproduces the R environment from `renv.lock`. It ends with a summary listing every dependency as @@ -187,18 +180,6 @@ precompile_workload = false The server then makes those requests itself in the background as it starts, so the first page visit does not wait on compilation. -## Community - -[Discussions](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions) are open, and they are where the tool's direction is argued out: - -- [Introduce yourself](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/148) — what you run through MetaManifold, and how it went. -- [Ideas](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/150) — what the tool should do next; small conveniences and whole methods both welcome. -- [The mathematical foundations thread](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/149) — the statistics underneath the analyses: what is proved, what is questionable, what deserves a proof. -- [Show and tell](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/151) — figures, publication tables and placement trees that came out of a run. -- [Q&A](https://github.com/hyperpolymath/MetaManifold-WebUI/discussions/152) — setup, usage and interpretation. A reproducible defect is a bug report, not a Q&A. - -One house norm: runs record their exact toolchain in `attestation.yml`; paste its versions block when asking why results differ. - ## Web interface The browser interface is the primary way to drive MetaManifold. Beyond creating studies and launching runs, it offers the following: @@ -261,77 +242,32 @@ The composition view classifies each ASV/OTU into a biological category and rend ### Differential abundance -The Differential Abundance tab of a study's analysis workspace tests each taxon -for a difference in abundance between two conditions. Select exactly two runs or -pooled-run sub-groups, a results table and a rank. The first condition is the -reference, so a positive fold change means the taxon is more abundant in the -second. The rank list shows only the ranks the selected runs share. With -aggregation on, a pooled run counts as one condition rather than one per -sub-group. - -Counts are summed per taxon at the chosen rank, with rows that have no name at -that rank pooled as `Unclassified`. This happens after the read-count bounds and -any `analysis.exclude_categories` entry that applies to `differential`. Each -taxon is fitted by the configured method. The default, `nb_glm`, fits a -negative-binomial GLM, `MASS::glm.nb(y ~ group + offset(log(size factor)))`, -on the raw integer read counts; there are no pseudocounts, the group -coefficient gets a Wald test, and the effect is a log2 fold change. Size -factors come from the whole count matrix and are centred to a geometric mean -of 1. The alternative, `clr_welch`, runs Welch's t-test on centred log-ratios -over the taxa passing `min_prevalence`, after `zCompositions::cmultRepl` -replaces their zeros; its effect is a difference in CLR units, not a fold -change. Under either method, Benjamini-Hochberg adjustment runs over the taxa -that produced a p-value. The code is in -`src/analysis/differential.jl`, and the route in `src/server/routes/analysis.jl`. - -Settings live under `analysis.differential` (see -[Configuring analysis](#configuring-analysis-analysis-in-pipelineyml)): - -- `method`: `nb_glm` (default) or `clr_welch`, as above. -- `offset`: `tss` (default) uses each sample's total reads. `rle` uses the - median of ratios over the taxa with reads in every sample. Used by `nb_glm` - only; `clr_welch` ignores it, since log-ratios carry no library size. -- `min_prevalence`: a fraction from 0 to 1 (default 0). A taxon with reads in a - smaller fraction of the samples is not fitted. For `clr_welch` it is applied - before zero replacement, so on a sparse table it also keeps imputed values - for rare taxa out of every sample. -- `replacement_delta`: `clr_welch` only; `cmultRepl`'s `frac`, between 0 and 1 - (default 0.65). An imputed proportion above its taxon's smallest observed - proportion is lowered to this fraction of it. A replaced zero is not a - measurement. - -The result is a volcano plot and a table. The plot draws log2 fold change -against -log10 of the raw p-value, coloured by whether the adjusted p-value is -below 0.05. Taxa without an adjusted p-value are left off the plot but stay in -the table. Each table row has a status: - -- `ok`: the model fitted. -- `boundary`: the model fitted, but the dispersion parameter theta reached a - bound (at least 1e7, so effectively Poisson, or at most 1e-8). These taxa - keep their p-value and stay in the adjusted family. -- `failed`: no p-value, with the reason. Causes are counts that are constant - across samples, an error from `glm.nb`, a fit that did not converge, an - aliased group coefficient, or a non-finite estimate. -- `filtered`: below `min_prevalence`, so not fitted. - -A failed or filtered taxon never gets a stand-in p-value, and it is not counted -in the adjustment. The table downloads as `differential_abundance.csv` and can -be added to the report. The CSV starts with `#` lines that record the -comparison, method and settings. - -The test refuses, with an explicit error, when: - -- the selection is not exactly two conditions, or both are the same; -- the two conditions resolve to different ranks; -- a condition has no results table, no taxonomy columns or no sample columns, - or the read-count bounds remove all its samples; -- `offset` or `min_prevalence` is invalid; -- a count is not a non-negative integer, a condition has no samples, or there - are fewer than 3 samples in total; -- `tss` meets a sample with no reads, or `rle` finds no taxon with reads in - every sample (use `tss`); -- the R package MASS cannot be loaded; -- no taxon could be fitted at all. +In a study's analysis workspace, select two conditions (runs or pooled-run +subgroups), a results table and a shared taxonomic rank. The first condition +is the reference. Counts are aggregated by taxon after applicable +`analysis.exclude_categories` filters. This workspace does not pass sample +read-count bounds to the differential-abundance request, so those bounds do +not exclude samples from this analysis. + +`analysis.differential.method` chooses the method: + +- `nb_glm` (default): a negative-binomial GLM per taxon using `MASS::glm.nb`, + with a library-size offset. The effect is log2 fold change; positive values + indicate greater abundance in the second condition. `offset` selects `tss` + (default) or `rle` scaling. +- `clr_welch`: Welch's t-test on centred log-ratios after multiplicative zero + replacement with `zCompositions::cmultRepl`. The effect is a CLR difference, + not a fold change. `replacement_delta` controls the replacement fraction + (default 0.65); `offset` is not used. + +For either method, `min_prevalence` (default 0) filters taxa by the fraction of +samples with nonzero counts. Benjamini–Hochberg adjustment includes only taxa +that produced a p-value. Failed or filtered taxa have no substitute p-value; +inspect their status and note in the results table. Invalid input or an +unavailable required R package produces an explicit error. + +Implementation and reference tests: `src/analysis/differential.jl`, +`src/server/routes/analysis.jl`, and `test/unit/test_differential.jl`. ### Figures and report @@ -647,13 +583,13 @@ Each entry names a filter in the `filters:` library of `config/composition.yml`. ### Configuring analysis (`analysis:` in `pipeline.yml`) -Controls the defaults applied to the analysis charts (alpha diversity, taxa bar, NMDS, differential abundance, etc.). Per-chart choices such as the taxonomic rank and relative or absolute abundance are made in the UI. +Controls the defaults applied to the analysis charts (alpha diversity, taxa bar, NMDS, etc.). Per-chart choices such as the taxonomic rank and relative or absolute abundance are made in the UI. ```yaml analysis: exclude_categories: # composition categories to drop from figures; [] to keep all - - {set: contamination, category: Contaminant, apply_to: [diversity, taxa, venn, differential]} - # apply_to surfaces: diversity | taxa | composition | venn | differential + - {set: contamination, category: Contaminant, apply_to: [diversity, taxa, venn]} + # apply_to surfaces: diversity | taxa | composition | venn # (omit apply_to to act on every surface) alpha: # richness, Shannon and Simpson normalisation: none # none | rarefy | srs (scaling with ranked subsampling) @@ -670,9 +606,6 @@ analysis: depth: 0 # rarefy/srs reads per sample; 0 = smallest non-empty sample nmds: max_stress: 0.2 # warn if NMDS stress exceeds this value - differential: # negative-binomial differential abundance (MASS::glm.nb) - offset: tss # tss | rle (rle is refused when no taxon has reads in every sample) - min_prevalence: 0.0 # fraction of samples, 0 to 1, a taxon needs reads in to be tested ``` ### Configuring taxonomic filtering (`filters:` in `config/composition.yml`) @@ -858,7 +791,7 @@ The server exposes a REST API under `/api/v1/`. Key endpoint groups: | Results | `GET/POST/DELETE .../results/tables/...` | List, query, filter, save, export (`.xlsx`), and delete tables; OTU member drill-down | | QC | `GET .../results/qc`, `GET .../results/dada2` | MultiQC report metadata and DADA2 figures, logs, stats | | Analysis | `POST .../runs/{run}/analysis/{alpha,chart}`, `GET .../runs/{run}/analysis/{pipeline-stats,ranks}` | Per-run charts and rank discovery | -| Cross-run | `POST /studies/{study}/analysis/{alpha,chart,chart-facet,nmds,permanova,venn,publication-tables,differential}` | Comparison, faceted charts, NMDS, PERMANOVA, taxon overlap, publication tables and differential abundance across runs | +| Cross-run | `POST /studies/{study}/analysis/{alpha,chart,chart-facet,nmds,permanova,venn,publication-tables}` | Comparison, faceted charts, NMDS, PERMANOVA, taxon overlap and publication tables across runs | | Composition | `POST .../runs/{run}/composition/summary`, `POST .../composition/{source}/query`, `POST .../composition/{source}/distinct/{column}` | Organism-composition summaries and tables for a run | | Composition library | `GET /composition`, `POST/DELETE /composition/{filters,sets}/{name}`, `GET /category-sets`, `POST/DELETE /category-sets/{name}` | Edit the filters and category sets in `config/composition.yml` | | Reference trees | `GET/POST /reference-trees`, `GET/PUT/DELETE /reference-trees/{id}`, `GET/PUT .../fasta`, `POST .../run`, `GET .../log/{step}`, `GET .../qc/{align,trim}`, `GET .../alignment/{raw,trimmed}`, `GET .../treefile`, `POST .../trim-preview` | Build and inspect the shared reference trees | diff --git a/.github/codecov.yml b/codecov.yml similarity index 100% rename from .github/codecov.yml rename to codecov.yml diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 44b2c882..8181ae6d 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -358,14 +358,6 @@ analysis: # rle is refused when no taxon is present in every sample. # Used by nb_glm only; clr_welch ignores it, since log-ratios carry no library size. offset: tss - # nb_glm only: where each taxon's negative-binomial theta comes from. ml - # (the default) takes the maximum-likelihood estimate MASS::glm.nb fits per - # taxon. glmGamPoi takes the shrunken dispersion glmGamPoi::glm_gp fits - # across all taxa in one call -- sharing information across taxa, which - # steadies low-count taxa -- and fits each taxon through - # MASS::negative.binomial(theta = 1/dispersion), so the Wald test is - # unchanged. Requires the glmGamPoi R package. - dispersion: ml # Fraction of samples, 0 to 1, in which a taxon must have reads to be # tested. Taxa below it are listed as filtered, without a p-value. For # clr_welch it is applied before zero replacement, so on a sparse table it also diff --git a/docs/milestones/02-deferred-issues.md b/docs/milestones/02-deferred-issues.md deleted file mode 100644 index 67e6f59c..00000000 --- a/docs/milestones/02-deferred-issues.md +++ /dev/null @@ -1,290 +0,0 @@ - - -# Deferred Features — Ready-to-Paste GitHub Issue Bodies - -These are for features that must NOT be implemented in the current task (exact statistics layer and symbolic engine), -but have clear scientific value and should be tracked. - -Each issue includes scientific value, difficulty, risks. - ---- - -## Issue 1: Exact Statistics Layer — Exact Tests for Small Samples - -**Title:** `feat(analysis): Exact statistics layer — Fisher's exact, exact NB, permutation tests for small-n microbiome data` - -**Labels:** `enhancement`, `analysis`, `deferred`, `scientific-value:high`, `difficulty:hard` - -**Body:** - -### Scientific Value -Parametric approximations (NB GLM, Gaussian LM) break down with small sample sizes (n<10 per group) or sparse features. Exact tests (Fisher's exact for presence/absence, exact NB test via `exactTest` in edgeR, permutation-based PERMANOVA with exact p-values) provide valid inference when asymptotics fail. Critical for rare biosphere, low-biomass samples, and clinical cohorts with limited n. - -- **Use case:** Pathogen detection where presence in 2/3 cases vs 0/20 controls should be significant even with tiny n. -- **Impact:** Reduces false negatives in small studies, improves reproducibility for low-abundance taxa. - -### Scope (Deferred — DO NOT IMPLEMENT HERE) -- Exact Fisher's test for 2x2 tables (presence/absence vs group) -- Exact negative binomial test (edgeR `exactTest` or `exactTestDoubleTail`) -- Permutation-based exact p-values for NB GLM (parametric bootstrap) -- Exact CLR/ILR via permutation of Aitchison distances -- Integration with AnalysisConfig: new method `exact_fisher`, `exact_nb`, `permutation` - -### Difficulty -**Hard** — requires: -- R integration (exact tests via `stats::fisher.test`, `edgeR`, `permute`) -- Combinatorial explosion for large tables (need network algorithm for Fisher) -- Performance: exact tests O(n!) naive, need optimized implementations -- Memory: permutation stores B=10k resamples per taxon -- Validation: compare against known exact p-values from R - -### Risks -- **Performance regression:** Exact tests 10-100x slower than asymptotic; must be behind Advanced Analysis expander and Evidence Mode, with warning about runtime. -- **Memory:** Permutation matrices for 10k taxa x 10k permutations = 100M entries ~ 800MB -- **Scientific misuse:** Exact does not mean assumption-free; still assumes exchangeability under null. Need context help explaining when exact is appropriate vs when it is overly conservative. -- **Dependency:** Adds `edgeR`, `BiocParallel` to renv.lock — increases install time and fragility - -### Acceptance Criteria -- [ ] New methods in AnalysisConfig v2: `exact_fisher`, `exact_nb`, `permutation_nb` -- [ ] BH still mandatory, DANGER banner if disabled -- [ ] Benchmark: <10% regression for existing methods, new methods benchmarked separately with 100 taxa, n=5 per group -- [ ] Tests: exact p-values match R's `fisher.test` for 10 known tables -- [ ] Docs: context help explains exact vs asymptotic, when to use -- [ ] UI: behind Advanced Analysis expander, shows estimated runtime - -### Related -- Blocked by: AnalysisConfig v1 (this PR) -- Blocks: Symbolic engine (needs exact p-value expressions) - ---- - -## Issue 2: Symbolic Engine — Formula Manipulation and Provenance - -**Title:** `feat(analysis): Symbolic engine for formula manipulation, contrast derivation, and provenance` - -**Labels:** `enhancement`, `analysis`, `deferred`, `scientific-value:high`, `difficulty:very-hard` - -**Body:** - -### Scientific Value -Current AnalysisConfig stores formula as string (`~ group + batch`). Symbolic engine would: -- Parse formula into AST, validate variables exist, derive contrasts (e.g., `groupB - groupA`) -- Automatically generate all pairwise contrasts for multi-level factors -- Prove that BH correction is applied to the correct family (e.g., all taxa, or per-contrast?) -- Generate human-readable report: "Testing 1500 taxa for effect of group, controlling for batch, with BH FDR 0.05" -- Enable DOI bundle to include symbolic derivation of analysis intent - -- **Use case:** Study with 4 groups (control, diseaseA, diseaseB, diseaseC) — should automatically test all 6 pairwise, with BH across 1500*6=9000 tests, not 1500. -- **Impact:** Prevents p-hacking by making analysis intent explicit and auditable. - -### Scope (Deferred — DO NOT IMPLEMENT HERE) -- Formula parser: R-style `~` and `+`, `*`, `:`, `I()`, `(1|batch)` for random effects (future) -- Contrast derivation: for factor with k levels, generate k-1 or k*(k-1)/2 contrasts -- Provenance: symbolic proof that result hash chains to formula AST hash -- Integration with DEED: `(formula (ast ...) (contrasts ...))` -- Nickel contract: formula as structured data, not string - -### Difficulty -**Very Hard** — requires: -- Parser combinators or RCall to `terms.formula` -- Symbolic algebra (like `Symbolics.jl` or custom) -- Handling of R's non-standard evaluation (NSE) for formulas -- Provenance: need to hash AST, not just string (string `~ group + batch` vs `~ batch + group` are same model but different strings — should hash to same? Or not? Decision needed) -- UI: visual formula editor with drag-and-drop of metadata columns, live validation - -### Risks -- **Complexity:** Symbolic engine is a research project itself; may introduce bugs in contrast derivation leading to wrong scientific conclusions (worst risk) -- **Scope creep:** Random effects `(1|batch)` opens mixed models, which is a whole new layer (lme4, glmmTMB) — must be explicitly out of scope for v1 -- **Performance:** Parsing 1500 formulas (one per taxon) x 6 contrasts = 9000 parses — need caching -- **R dependency:** If using R's `terms`, need R runtime lock, which may deadlock with pipeline stages -- **Security:** Formula injection if AST allows arbitrary R code (e.g., `~ group + system('rm -rf /')`) — must whitelist allowed symbols - -### Acceptance Criteria -- [ ] Formula AST type in Julia, with `from_string` and `to_string` roundtrip -- [ ] Contrast derivation for factors, with BH family size correctly computed -- [ ] Provenance: config hash includes AST hash, not just string hash -- [ ] Tests: 20 formulas parsed, contrasts derived, compared to R's `model.matrix` and `contrasts` -- [ ] Security: refuses formulas containing `system`, `eval`, `parse`, etc. -- [ ] Docs: explains symbolic vs string, why AST matters for provenance -- [ ] UI: formula editor with autocomplete of metadata columns, shows derived contrasts - -### Related -- Blocked by: AnalysisConfig v1, exact stats layer (needs exact p-value symbols) -- Blocks: Automated report generation, DOI bundle v2 - ---- - -## Issue 3: Compositional Data Analysis — Advanced Methods (ANCOM-BC, ALDEx2, Songbird) - -**Title:** `feat(analysis): Advanced compositional methods — ANCOM-BC, ALDEx2, Songbird` - -**Labels:** `enhancement`, `analysis`, `deferred`, `scientific-value:high`, `difficulty:hard` - -**Body:** - -### Scientific Value -CLR/ILR+LM in v1 are basic compositional methods. Advanced methods address specific biases: -- **ANCOM-BC:** Bias correction for sampling fraction, handles zero inflation better than pseudocount -- **ALDEx2:** Bayesian Dirichlet-multinomial, accounts for sampling uncertainty, provides effect size + p-value -- **Songbird:** Multinomial regression for compositional data, ranks taxa by association with covariates - -- **Use case:** Gut microbiome with highly variable library sizes and many zeros — ANCOM-BC reduces false positives from pseudocount. -- **Impact:** More accurate differential abundance for compositional data, which is inherently relative. - -### Scope -- ANCOM-BC via R `ANCOMBC` package -- ALDEx2 via `ALDEx2` package (CLR + Welch's t + BH, with Dirichlet sampling) -- Songbird via QIIME2 or Python `songbird` (multinomial regression) -- Integration with AnalysisConfig: new methods `ancom_bc`, `aldex2`, `songbird` - -### Difficulty -**Hard** — requires: -- R packages `ANCOMBC`, `ALDEx2` (Bioconductor, heavy dependencies) -- Python interop for Songbird (via `PythonCall.jl` or subprocess) -- Zero handling: ANCOM-BC has own zero handling, not pseudocount -- Benchmarking against existing CLR/ILR -- Memory: ALDEx2 Dirichlet sampling 128 samples per taxon x 10k taxa = 1.28M CLR values - -### Risks -- **Dependency hell:** ANCOMBC depends on `lme4`, `pbapply`, etc.; may conflict with existing renv.lock -- **Performance:** Songbird multinomial regression is iterative, may take hours for 10k taxa -- **Scientific controversy:** Compositional methods are debated (Gloor et al. vs Morton et al.); need balanced context help, not taking sides -- **Reproducibility:** Songbird uses TensorFlow, non-deterministic unless seed fixed — need to record seed in provenance - -### Acceptance Criteria -- [ ] New methods in AnalysisConfig v2 -- [ ] BH mandatory, DANGER banner preserved -- [ ] Tests: compare against R reference for 3 datasets (mock, gut, soil) -- [ ] Benchmark: runtime and memory vs CLR_LM, with 10% regression gate for existing methods -- [ ] Docs: explains when to use ANCOM-BC vs CLR vs Songbird, with citations - ---- - -## Issue 4: CladeCumulus — Phylogenetic Tree Integration - -**Title:** `feat(cladistics): CladeCumulus phylogenetic integration — tree from taxonomy + phylogeny, not just taxonomy ranks` - -**Labels:** `enhancement`, `cladistics`, `deferred`, `scientific-value:medium`, `difficulty:hard` - -**Body:** - -### Scientific Value -Current CladeCumulus builds tree from taxonomic ranks (Domain, Phylum, ...). True phylogenetic tree (from 16S sequences via FastTree or IQ-TREE) would enable: -- Cumulative frequencies along phylogenetic branches (not just taxonomic) -- Phylogenetic diversity metrics (Faith's PD, UniFrac) in CladeCumulus -- Detection of clades that are phylogenetically clustered but taxonomically dispersed (e.g., convergent evolution) - -### Scope -- Build phylogeny from ASV sequences (via `DECIPHER` + `phangorn` or external FastTree) -- Integrate with CladeTree: nodes have both taxonomic and phylogenetic parents -- Cumulative frequencies along phylogeny: sum of counts in clade -- Epistemic colour coding still applies, but residual_count now includes phylogenetic uncertainty - -### Difficulty -**Hard** — requires: -- Sequence alignment (MAFFT or DECIPHER) -- Tree building (FastTree, IQ-TREE) — external binary, need provenance like other tools -- Tree parsing (Newick) and integration with existing taxonomy -- Performance: alignment of 10k ASVs O(n^2) memory, tree building O(n^3) worst case - -### Risks -- **Performance:** 10k ASVs alignment may take hours and 10GB RAM — need to limit to top N or provide subsampling -- **Provenance:** New tool (FastTree) needs probing and hashing like vsearch/swarm -- **Scientific:** Phylogeny from short 16S V4 amplicons is noisy; need to warn that tree is approximate -- **UI:** Phylogenetic tree + taxonomic tree = two hierarchies, need toggle or combined view — may clutter UI if not behind Evidence Mode - ---- - -## Issue 5: Evidence Mode — Full Epistemic UI - -**Title:** `feat(ui): Full Evidence Mode — epistemic status editor, fiber visualizer, residual explorer` - -**Labels:** `enhancement`, `ui`, `epistemic`, `deferred`, `scientific-value:medium`, `difficulty:medium` - -**Body:** - -### Scientific Value -Evidence Mode toggle currently only gates Advanced Analysis and CladeCumulus. Full Evidence Mode would include: -- Editor for `avec_fibre` column (manual curation of which rows carry semantic fibre) -- Fiber visualizer: show Echo fiber for a selected taxon (all candidate worlds consistent with observation) -- Residual explorer: finite model of residual decomposition (like residual-evidence-types explorer.html) -- Warrant editor: what evidence tokens support a claim? - -### Scope -- UI for editing avec_fibre boolean per row (in DataTable) -- Fiber visualizer component: shows witnesses (possible true worlds) over observed -- Residual explorer: slider for noise bound, shows candidate count and presence status -- Integration with provenance: fiber edits logged - -### Difficulty -**Medium** — mostly frontend, but needs backend API for fiber computation - -### Risks -- **UI clutter:** If not carefully designed, Evidence Mode could overwhelm non-expert users. Must keep clean, with progressive disclosure. -- **Performance:** Fiber for 10k taxa x 100 candidate worlds = 1M candidates, need virtualized list -- **Scientific misuse:** Manual editing of avec_fibre could be used to cherry-pick results — need to log edits in provenance and DOI bundle, with DANGER banner if many rows changed - ---- - -## Issue 6: DOI Bundle — Zenodo Integration and Automated DOI Minting - -**Title:** `feat(doi): Zenodo integration — automated DOI minting from DOI-ready bundles` - -**Labels:** `enhancement`, `doi`, `infra`, `deferred`, `scientific-value:high`, `difficulty:medium` - -**Body:** - -### Scientific Value -DOI-ready bundles currently create local directory with DataCite JSON. Zenodo integration would: -- Upload bundle to Zenodo via API, mint DOI automatically -- Link DOI to GitHub release, make analysis citable -- Enable reproducibility: anyone with DOI can download exact config + results + provenance - -### Scope -- Zenodo API client (via HTTP.jl) -- Upload bundle zip, create deposition, publish, get DOI -- Store DOI in provenance and link to GitHub Project board -- UI: "Mint DOI" button in AnalysisConfigEditor, shows DOI badge - -### Difficulty -**Medium** — requires Zenodo token, HTTP client, error handling - -### Risks -- **Token security:** Zenodo token must be stored as secret, not in code -- **Cost:** Zenodo is free but has rate limits; need to handle 429 -- **Irreversibility:** Publishing to Zenodo is irreversible (DOI minted) — need confirmation dialog with DANGER banner -- **Dependency:** Zenodo API may change; need to pin API version - ---- - -## Issue Template Footer (for all) - -**For every issue:** -- [ ] Tests and benchmarks (fail CI on >10% regression) -- [ ] JSON + Nickel + DEED schemas updated -- [ ] Docs and context-sensitive help -- [ ] UI clean, behind Evidence Mode if advanced -- [ ] Provenance-rich, immutable derived objects -- [ ] Linked to Project board "Analysis Layer & Cladistics Development" -- [ ] Milestone report after completion - ---- - - diff --git a/docs/occupancy-2026-10.adoc b/docs/occupancy-2026-10.adoc deleted file mode 100644 index 6d291b05..00000000 --- a/docs/occupancy-2026-10.adoc +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only -// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -= Occupancy of the vsearch stage -:revdate: 2026-10-14 - -The vsearch stage runs two independent searches, ASV and OTU. The full -pipeline spawns them together; a single-stage re-run runs them one after the -other. The high-water-mark (HWM) algebra of occupancy-types predicts that the -stage's cost is the same in every order but its memory peak is not: serially -the peak is the larger search's, concurrently it can be their sum -(choreographic-types A-2 and A-3). This page records one measurement of that, -on 2026-10-14, and checks it against the typed reading. - -== What was measured - -`MemoryProbe` (`src/core/memory_probe.jl`), off unless a pipeline.yml sets -`occupancy_probe.enabled: true`, records for each attested stage: - -* `julia_allocated_bytes`: the `Base.gc_num()` delta over the stage (cost); -* per task (one spawned search): the largest sampled VmRSS of the child - process it started (`peak_rss_kib`), the kernel's own VmHWM for it - (`hwm_kib`), and the sample window its children were live in — `first_sample`, - the first sample taken after its first child started, and `last_sample`, the - last taken before its last was waited for, 0 for never — which its grade is - read from; -* `child_peak_rss_kib`: the largest sum of the stage's children's VmRSS at - one sampling instant (the stage's occupancy). - -The run: `test/occupancy_disagreement_test.jl`, which runs the same two searches -in three orders under the probe, on a fixture it generates: 1000 reference -sequences of 400 bp, 300 ASV and 150 OTU queries. Linux 6.18 with 4 CPUs, -Julia 1.12.5 with 2 threads, vsearch 2.30.5 with `--threads 1`. Sampling -interval 10 ms. - -== Results - -[cols="2,1,1,1,1,1,1",options="header"] -|=== -| Order | ASV peak (KiB) | OTU peak (KiB) | Stage peak, measured (KiB) | Counted bytes, Session IR | Typed peak, HWM | Julia allocation (bytes) -| ASV then OTU | 9204 | 9180 | 9204 | 9204 | 9204 | 978432 -| OTU then ASV | 9152 | 9192 | 9192 | 9192 | 9192 | 978720 -| Concurrent | 9176 | 9204 | 18380 | 18380 | 18380 | 978816 -|=== - -* *Cost commutes.* The allocation totals agree to 0.04%. The test asserts - agreement within 1%; a planted extra search in one order breaks it (spread - 25%). -* *Each search's peak is the same in every order*, to within 0.6% - (9152-9204 KiB). Each search's VmHWM equals its sampled peak, so the - sampling caught the peaks. -* *The stage peak is not the same.* Serially it is the larger search's - (9192-9204 KiB); concurrently it is 18380 KiB, twice that. This is the A-3 - order dependence, and nothing in the stage's outputs shows it: the hits are - byte-identical in all three orders. The prediction would have been falsified - had the concurrent peak matched the serial one; it did not. -* *Counted bytes equal the typed peak*, in all three orders. The counted bytes - come from occupancy-types' Session IR stepper running - `docs/occupancy/vsearch-*.sir`, one buffer per search. The typed peak is - computed, by `refl`, from occupancy-types' `Timeline` over resource-algebra's - HWM grade (`docs/occupancy/TypedPeak.agda`). A search alone exits holding - nothing (drift 0); in the concurrent order the first is still resident - when the second starts (drift = peak). The sequence law - `max (peak a) (drift a + peak b)` then gives the larger peak serially and the - sum concurrently. -* *The measured stage peak also equals the typed peak* here. For the concurrent - order the typed peak is the interleaving in which both are resident at once; - a measurement could come in lower if the two searches' peaks did not - coincide. Here they did: each search's memory plateaus once its database is - loaded. - -== Reading the numbers - -A recorded `peak_rss_kib` is a reading, not an occupancy: what a stage occupied -depends on what the next stage inherits from it. So the probe grades a stage's -whole window (`MemoryProbe`'s `task_grades`, `stage_grade`, -`predicted_peak_rss_kib`, `cost_bound_rss_kib`, `reclaimed_kib`) and attaches the -result to a recorded stage as `grades`; a record with no sample windows is left -ungraded rather than graded on an assumed order. Tasks are read in sampled order -(first sample, ties by record order). Each handover keeps only the tasks still -live at it, so a task's `drift_kib` is the change in the live peak sum across its -boundary — negative when the boundary also releases an earlier task — and -`release_kib` is everything freed between its start and the next boundary, which -can exceed that task's own peak. The folded drift is the sum of the peaks still -live when the stage ends; the pinned instance's own `drift` is that non-negative -live sum, the signed per-task delta being the extension its header records as -H-2. - -For the three orders above, the algebra's predicted peak reproduces the measured -stage peak exactly — 9204, 9192 and 18380 KiB — and the slack against the cost -bound is what the cuts reclaimed: 9180, 9152 and 0 KiB. -`scripts/check-occupancy-grades.jl` checks that equality, occupancy≤cost and -non-negative reclamation against this page's raw record, rebuilding each order's -windows from `docs/occupancy/schedules.yml` (the file records no windows of its -own; the Session IR texts in this directory state the same interleavings). The -reclamation readings are theorems over the pinned siblings at these numbers: -`docs/occupancy/GradedStages.agda` — `no-reclaim` for the concurrent stage, -`reclaimed` for the serial one, where 9204 KiB are released at the frontier and -the cost reading counts them — and the two protocol-level statements -(`occupancy-at-cut`, `protocol-occupancy≤cost`) as the siblings' own two-step -over the stage's timeline. `docs/occupancy/reject` gains a fourth rejection, for -reading the concurrent stage's drift as 0. - -`scripts/lint-occupancy-readings.sh` keeps production code from reading the key -directly: outside `src/core/memory_probe.jl`, an occupancy figure must go through -those helpers. - -== Status - -[cols="1,3,1,2",options="header"] -|=== -| ID | Claim | Status | Artefact -| MMW-01.1 | With the flag off, a stage record has exactly its earlier keys; with it on, it gains `occupancy`; old records still parse and merge | TESTED | `test/occupancy_disagreement_test.jl` ("flag off", "round trip", "through the pipeline's _attest") -| MMW-01.2 | For this workload, the vsearch stage's cost (Julia allocation) is the same in all three orders | TESTED | same file, "two-thread disagreement" (asserted, within 1%) -| MMW-01.3 | For this workload, the stage peak depends on the order (A-3), and each search's own peak does not | TESTED | same file, recorded in the test log; table above -| MMW-01.4 | For the three Session IR texts, the stepper's counted bytes equal the HWM typed peak | TESTED | `scripts/check-occupancy.sh` (stepper output against `TypedPeak.agda`; three rejections in `docs/occupancy/reject`) -| MMW-01.5 | The recorded run's grades reproduce its measured stage peaks, and the reclamation the pinned siblings' laws read off them is what the cuts freed | TESTED | `scripts/check-occupancy-grades.jl` (the recorded record), `docs/occupancy/GradedStages.agda` (`no-reclaim`, `reclaimed`, `occupancy-at-cut` at the measured numbers), `test/unit/test_occupancy_grade.jl` -| bridge | Counted bytes are bounded by the typed peak in general (occupancy-types' v0 → typed bridge) | OPEN | unchanged; this page gives three instances, not a proof -|=== - -== Limits - -* *Sampled.* A per-task peak is the largest VmRSS seen at 10 ms intervals. VmHWM - is exact up to the last sample, but growth after it is not seen. A child that - lives for less than one interval may not be sampled at all (`samples: 0`). -* *Child memory only.* The children are counted; the Julia process is not. Its - `julia_live_bytes_delta` is recorded, but without a forced collection it - reflects when the GC last ran, not what the stage kept. -* *Allocation is process-wide.* While the study loop runs several runs' - pipelines at once, a stage's `julia_allocated_bytes` includes the others' - allocations too. The child figures are per stage, since each stage samples - only the children it started. -* *Linux only*, since it reads /proc. For a stage offloaded with `remote:`, - the local child is an ssh client, so its memory says nothing about the work. -* *One thread for vsearch.* With several threads, vsearch writes its hits in - whatever order the threads finish. `_sort_hits_by_query!` then does a - different amount of work, because its sort key splits each line on every - comparison. That moved the stage's allocation total by up to 12% between - repeats of the same order. So in production the cost is order-independent - only up to that noise; precomputing the sort key would remove it. This PR - does not change it. -* *One host, one day.* CI runs the same test on every push and logs its own - numbers; they are not compared with these. - -== Reproducing - ----- -julia --project=. -t 2 test/occupancy_disagreement_test.jl # set OCCUPANCY_RESULTS= for YAML -bash scripts/check-occupancy.sh # needs julia and agda 2.6.4.x -julia --project=. scripts/check-occupancy-grades.jl # the recorded run's grades -bash scripts/lint-occupancy-readings.sh # no ungraded readings in src/ ----- - -`scripts/fetch-siblings.sh` fetches the siblings at their pins into -`_build/dependencies`: occupancy-types `af95728` and resource-algebra -`cf9235c`. - -== As the proofs progress - -* *Per-buffer (n-ary) occupancy* in occupancy-types: each search is already one - buffer in the Session IR texts, so the typed peak could then be read per - search. -* *Signed drift*: frees could be graded as negative drift, so the Julia side's - collections (the pipeline's `GC.gc(false)` hints) become modelled - reclamation rather than unmodelled noise. -* *choreographic-types' operational `step` as a function*: the three orders - become three traces of one typed protocol. The frontier certificate its - two-thread application note describes could then be displayed in the WebUI. diff --git a/docs/occupancy/GradedStages.agda b/docs/occupancy/GradedStages.agda deleted file mode 100644 index 3c45730d..00000000 --- a/docs/occupancy/GradedStages.agda +++ /dev/null @@ -1,96 +0,0 @@ -{-# OPTIONS --safe --without-K #-} --- SPDX-License-Identifier: AGPL-3.0-only --- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --- --- The vsearch stage's grades, and occupancy-types' reclamation theorems at the --- measured numbers. Fork issue #105 asks for exactly this: grade the stages --- (alloc/hold/release) and compare the predicted peak with the measured one. --- TypedPeak.agda computes the typed peak by the algebra's sequence law; this --- module adds the grades the measurement itself gives, and the laws that say --- what the difference between a peak and a cost reading is: occupancy≤cost, and --- the strict reclamation at a cut where something is released. --- --- Consumed, never re-declared, from the siblings scripts/fetch-siblings.sh pins --- and scripts/check-occupancy.sh checks: --- occupancy-types af957280075699c659de0abd9a3b0defade77309 --- resource-algebra cf9235c532712a783a404c359a1055f40134794a --- Everything is stated in this file's own Timeline instance (`TH`, as --- TypedPeak.agda does): the Grade-level laws are Reclaim's as they stand, and --- the two protocol-level ones are Reclaim's own two-step — its `cut-factor` over --- its `reclaim-law`/`occupancy≤cost` — applied to `xs` and `ys` of that instance. --- --- A grade is `hwm peak drift _`: what an interaction reached, and what it still --- holds when the next one begins. The peaks below are the 2026-10-14 run's --- (docs/occupancy-2026-10.adoc, docs/occupancy/results-2026-10-14.yml), in KiB. -module GradedStages where - -open import ResourceAlgebra.Prelude -open import ResourceAlgebra.Instances.HWM -open import OccupancyTypes.Order -open import OccupancyTypes.Reclaim - -import OccupancyTypes.Timeline -module TH = OccupancyTypes.Timeline HWM - --- A search that has not been waited for when the next one starts: it still holds --- everything it reached, so the cut reclaims none of it. -still : Nat → Grade -still p = hwm p p (≤-refl p) - --- A search that has exited before the next one starts: it holds nothing, so the --- cut reclaims all of it. -gone : Nat → Grade -gone p = hwm p 0 z≤n - --- The full pipeline runs the two searches together: ASV is still resident when --- OTU starts, so the stage peak is their sum and nothing is reclaimed — --- Reclaim.no-reclaim's equality, whose `refl` argument discharges the theorem's --- own hypothesis (drift = peak) at the measured grade. -run-together : peak (still 9176 ⊗h gone 9204) ≡ 18380 -run-together = no-reclaim (still 9176) refl (gone 9204) - --- A single-stage re-run runs them one after the other: ASV has exited, so the --- cut reclaims its 9204 KiB and the stage peak is the larger search's --- (Reclaim.reclaim-law, by `refl`). -run-in-turn : peak (gone 9204 ⊗h gone 9180) ≡ 9204 -run-in-turn = refl - --- The same claim as Reclaim.reclaimed states it: with drift < peak at the cut --- and a suffix that does anything, occupancy is strictly below cost. 9204 KiB --- are released at the frontier, so the cost reading (18384 KiB) overstates the --- stage. -run-in-turn --- --- The vsearch stage's typed peak, computed (TESTED, by refl) with --- occupancy-types' Timeline over resource-algebra's HWM grade, both consumed --- by pin through scripts/fetch-siblings.sh: --- occupancy-types af957280075699c659de0abd9a3b0defade77309 --- resource-algebra cf9235c532712a783a404c359a1055f40134794a --- Nothing here re-declares their definitions. --- --- Each search is one interaction graded by its measured peak, in KiB, from the --- 2026-10-14 run in docs/occupancy-2026-10.adoc. A search's process exits --- holding nothing, so run alone its drift is 0. Run concurrently, the first is --- still resident when the second starts, so that is graded with drift = peak. --- That is the interleaving the measurement observed (both resident at once). --- The algebra's sequence law, max (peak a) (drift a + peak b), then gives: --- serial = the larger peak; concurrent = the sum. -module TypedPeak where - -open import ResourceAlgebra.Prelude -open import ResourceAlgebra.Instances.HWM - -import OccupancyTypes.Timeline -module TH = OccupancyTypes.Timeline HWM - --- A search whose process has exited before the next one starts. -exits : Nat → Grade -exits p = hwm p 0 z≤n - --- A search still resident when the next one starts. -resident : Nat → Grade -resident p = hwm p p (≤-refl p) - --- ASV 9204 then OTU 9180. -asv-then-otu : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 9204 -asv-then-otu = refl - --- OTU 9192 then ASV 9152. -otu-then-asv : peak (TH.grade (exits 9192 TH.∷ exits 9152 TH.∷ TH.[])) ≡ 9192 -otu-then-asv = refl - --- ASV 9176 and OTU 9204 together. -concurrent : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 18380 -concurrent = refl diff --git a/docs/occupancy/reject/ConcurrentDriftIsZero.agda b/docs/occupancy/reject/ConcurrentDriftIsZero.agda deleted file mode 100644 index 749c213f..00000000 --- a/docs/occupancy/reject/ConcurrentDriftIsZero.agda +++ /dev/null @@ -1,18 +0,0 @@ -{-# OPTIONS --safe --without-K #-} --- SPDX-License-Identifier: AGPL-3.0-only --- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --- --- Must be REFUSED: reading the concurrent stage as one that hands on nothing. --- Running the two searches together leaves the ASV search resident when the OTU --- search starts, so the stage's drift is that search's peak — 9176 KiB, what a --- suffix would see — and not 0. Grading the drift as 0 would read the stage's --- COST as its occupancy, which is the very reading the measured overlap exists --- to refute. -module ConcurrentDriftIsZero where - -open import ResourceAlgebra.Prelude -open import ResourceAlgebra.Instances.HWM -open import GradedStages - -wrong : drift (still 9176 ⊗h gone 9204) ≡ 0 -wrong = refl diff --git a/docs/occupancy/reject/ConcurrentIsSerial.agda b/docs/occupancy/reject/ConcurrentIsSerial.agda deleted file mode 100644 index 592de6ee..00000000 --- a/docs/occupancy/reject/ConcurrentIsSerial.agda +++ /dev/null @@ -1,14 +0,0 @@ -{-# OPTIONS --safe --without-K #-} --- SPDX-License-Identifier: AGPL-3.0-only --- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --- --- Must be REFUSED: the claim that running the two searches concurrently leaves --- the peak where a serial run puts it (the larger of the two). -module ConcurrentIsSerial where - -open import ResourceAlgebra.Prelude -open import ResourceAlgebra.Instances.HWM -open import TypedPeak - -wrong : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 9204 -wrong = refl diff --git a/docs/occupancy/reject/SerialIsCost.agda b/docs/occupancy/reject/SerialIsCost.agda deleted file mode 100644 index dbb6f4ba..00000000 --- a/docs/occupancy/reject/SerialIsCost.agda +++ /dev/null @@ -1,14 +0,0 @@ -{-# OPTIONS --safe --without-K #-} --- SPDX-License-Identifier: AGPL-3.0-only --- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --- --- Must be REFUSED: reading a serial run's peak as the sum of the two peaks, --- i.e. occupancy as cost (occupancy-types R-6). -module SerialIsCost where - -open import ResourceAlgebra.Prelude -open import ResourceAlgebra.Instances.HWM -open import TypedPeak - -wrong : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 18384 -wrong = refl diff --git a/docs/occupancy/results-2026-10-14.yml b/docs/occupancy/results-2026-10-14.yml deleted file mode 100644 index 394236d6..00000000 --- a/docs/occupancy/results-2026-10-14.yml +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# The raw MemoryProbe records behind docs/occupancy-2026-10.adoc, written by -# test/occupancy_disagreement_test.jl with OCCUPANCY_RESULTS set (2026-10-14). -vsearch: "vsearch v2.30.5_linux_x86_64, 15.7GB RAM, 4 cores" -julia: "1.12.5" -threads: 2 -cpus: 4 -cost_spread: 0.00039231071008238524 -orders: - asv_then_otu: - probe: "MemoryProbe 1" - interval_ms: 10 - samples: 56 - julia_allocated_bytes: 978432 - julia_live_bytes_delta: 978544 - child_peak_rss_kib: 9204 - tasks: - - label: "vsearch:asv" - processes: 1 - samples: 33 - peak_rss_kib: 9204 - hwm_kib: 9204 - - label: "vsearch:otu" - processes: 1 - samples: 20 - peak_rss_kib: 9180 - hwm_kib: 9180 - otu_then_asv: - probe: "MemoryProbe 1" - interval_ms: 10 - samples: 57 - julia_allocated_bytes: 978720 - julia_live_bytes_delta: 978832 - child_peak_rss_kib: 9192 - tasks: - - label: "vsearch:otu" - processes: 1 - samples: 22 - peak_rss_kib: 9192 - hwm_kib: 9192 - - label: "vsearch:asv" - processes: 1 - samples: 33 - peak_rss_kib: 9152 - hwm_kib: 9152 - concurrent: - probe: "MemoryProbe 1" - interval_ms: 10 - samples: 33 - julia_allocated_bytes: 978816 - julia_live_bytes_delta: 978928 - child_peak_rss_kib: 18380 - tasks: - - label: "vsearch:asv" - processes: 1 - samples: 30 - peak_rss_kib: 9176 - hwm_kib: 9176 - - label: "vsearch:otu" - processes: 1 - samples: 24 - peak_rss_kib: 9204 - hwm_kib: 9204 diff --git a/docs/occupancy/schedules.yml b/docs/occupancy/schedules.yml deleted file mode 100644 index dfdedb81..00000000 --- a/docs/occupancy/schedules.yml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# The interleaving each order in results-2026-10-14.yml ran with, as the sample -# window [first, last] of each of its tasks, in the task order the record keeps. -# These are the same schedules the Session IR texts in this directory state -# (vsearch-concurrent.sir runs the two searches together; the other two run them -# one after the other), expressed in the probe's own sample indices: the windows -# of a serial order do not overlap, and the concurrent order's do. -# -# Read by test/unit/test_occupancy_grade.jl and scripts/check-occupancy-grades.jl, -# so the two cannot disagree about what the recorded run did. -asv_then_otu: - - [1, 2] - - [3, 4] -otu_then_asv: - - [1, 2] - - [3, 4] -concurrent: - - [1, 4] - - [2, 5] diff --git a/docs/occupancy/vsearch-asv-then-otu.sir b/docs/occupancy/vsearch-asv-then-otu.sir deleted file mode 100644 index e462fb5e..00000000 --- a/docs/occupancy/vsearch-asv-then-otu.sir +++ /dev/null @@ -1,13 +0,0 @@ -(* SPDX-License-Identifier: AGPL-3.0-only *) -(* vsearch-asv-then-otu.sir - the vsearch stage with the two searches run one - after the other, ASV first: the single-stage re-run's order. Each search is - one buffer, sized by its measured peak in KiB (docs/occupancy-2026-10.adoc, - 2026-10-14 run); each process exits holding nothing, so its buffer is dropped - before the next search starts. - - Expected: steps = 0 (no communication), peak_live = max 9204 9180 = 9204. *) -let asv = alloc 9204 in -let _ = drop asv in -let otu = alloc 9180 in -let _ = drop otu in -unit diff --git a/docs/occupancy/vsearch-concurrent.sir b/docs/occupancy/vsearch-concurrent.sir deleted file mode 100644 index 93be40bf..00000000 --- a/docs/occupancy/vsearch-concurrent.sir +++ /dev/null @@ -1,12 +0,0 @@ -(* SPDX-License-Identifier: AGPL-3.0-only *) -(* vsearch-concurrent.sir - the two searches spawned together, as the full - pipeline runs them. Session IR has no parallel composition, so this is the - interleaving the measurement observed: both buffers live at once, then both - dropped. - - Expected: steps = 0, peak_live = 9176 + 9204 = 18380. *) -let asv = alloc 9176 in -let otu = alloc 9204 in -let _ = drop asv in -let _ = drop otu in -unit diff --git a/docs/occupancy/vsearch-otu-then-asv.sir b/docs/occupancy/vsearch-otu-then-asv.sir deleted file mode 100644 index ba575591..00000000 --- a/docs/occupancy/vsearch-otu-then-asv.sir +++ /dev/null @@ -1,9 +0,0 @@ -(* SPDX-License-Identifier: AGPL-3.0-only *) -(* vsearch-otu-then-asv.sir - the same two searches, OTU first. - - Expected: steps = 0, peak_live = max 9192 9152 = 9192. *) -let otu = alloc 9192 in -let _ = drop otu in -let asv = alloc 9152 in -let _ = drop asv in -unit diff --git a/frontend/bun.lock b/frontend/bun.lock index 11ec980d..742f42a5 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -7,128 +7,226 @@ "dependencies": { "@upsetjs/react": "^1.11.0", "jspdf": "^4.2.1", - "plotly.js-dist-min": "^4.1.2", - "react": "^19.3.0", - "react-dom": "^19.3.0", - "react-router-dom": "^7.18.4", + "plotly.js-dist-min": "^2.35.2", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0", "svg2pdf.js": "^2.8.1", }, "devDependencies": { - "@types/plotly.js": "^3.0.15", - "@types/react": "^19.3.0", - "@types/react-dom": "^19.3.0", - "@vitejs/plugin-react": "^6.1.2", - "typescript": "^7.0.2", - "vite": "^8.3.4", + "@types/plotly.js": "^2.33.4", + "@types/react": "^18.3.18", + "@types/react-dom": "^18.3.5", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.2", + "vite": "^6.1.0", }, }, }, "overrides": { - "@types/react": "^19.3.0", - "@types/react-dom": "^19.3.0", + "@types/react": "^18.3.18", + "@types/react-dom": "^18.3.5", }, "packages": { + "@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], + + "@babel/compat-data": ["@babel/compat-data@7.29.0", "", {}, "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg=="], + + "@babel/core": ["@babel/core@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-compilation-targets": "^7.28.6", "@babel/helper-module-transforms": "^7.28.6", "@babel/helpers": "^7.28.6", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/traverse": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA=="], + + "@babel/generator": ["@babel/generator@7.29.1", "", { "dependencies": { "@babel/parser": "^7.29.0", "@babel/types": "^7.29.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw=="], + + "@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.28.6", "", { "dependencies": { "@babel/compat-data": "^7.28.6", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA=="], + + "@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="], + + "@babel/helper-module-imports": ["@babel/helper-module-imports@7.28.6", "", { "dependencies": { "@babel/traverse": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw=="], + + "@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.28.6", "", { "dependencies": { "@babel/helper-module-imports": "^7.28.6", "@babel/helper-validator-identifier": "^7.28.5", "@babel/traverse": "^7.28.6" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA=="], + + "@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.28.6", "", {}, "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug=="], + + "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], + + "@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="], + + "@babel/helpers": ["@babel/helpers@7.28.6", "", { "dependencies": { "@babel/template": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw=="], + + "@babel/parser": ["@babel/parser@7.29.0", "", { "dependencies": { "@babel/types": "^7.29.0" }, "bin": "./bin/babel-parser.js" }, "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww=="], + + "@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="], + + "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="], + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], - "@oxc-project/types": ["@oxc-project/types@0.154.0", "", {}, "sha512-KYOUGQFlLWR+s4QMV2Q9ko38TZm7EdO87Qm7rvZLqk/webse/cDjtyxKhg/47l0byCbs/k46LT8qc4PKjAikzA=="], + "@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="], - "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.14", "", { "os": "android", "cpu": "arm" }, "sha512-aWR+XKH9+rW7m5fWq/5360J3mHycrx/o6xZM2DQFdJnTMdyP7/WzxPjhtLbVWdKo2y612/4U1bWqb3HQhiF7vg=="], + "@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="], - "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.2.14", "", { "os": "android", "cpu": "arm64" }, "sha512-QiVxHofl6A8lQU7WAtOUY4QMVZXohxwz7WJJoeh8NBEUyKfIkMRKhYyq8az/1RypNiIXzNTtiS6KwHz0erxYRQ=="], + "@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], - "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.2.14", "", { "os": "darwin", "cpu": "arm64" }, "sha512-lTQOzHuBIooh3jIr5XINQkjPGEuGy+Dsy0meXDr6Sjty0b99kYgGh4uBC/8mJX1T0kC8d6S0aPP1Y+bHkh6aBw=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], - "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.2.14", "", { "os": "darwin", "cpu": "x64" }, "sha512-hyMKEGatjmBYCs/Q4sbIcL/GqpFEqsX3wc+HYVQ/85A4rxc1o7rPm4DUI4GV4k/h0ITddwFSGpqQunI14GXjTA=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], - "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.2.14", "", { "os": "freebsd", "cpu": "x64" }, "sha512-u371GbXoVXrEgP20PwzoGNOdAOqAMhzo19JzzY+V4YmmnC+DTOoqdbMu+O+54emM59Mik/UdrtWgcuEDKWRPKA=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], - "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.2.14", "", { "os": "linux", "cpu": "arm" }, "sha512-BqwKsdZQf6EqVWr8fv2MrDYzAvOZ7+NYFSJESRGs87fkseEhs0TU9TXgRJTkZoRRkxTizyEg1LKUXfU3/78uqg=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], - "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.2.14", "", { "os": "linux", "cpu": "arm64" }, "sha512-SD9kY8M009x/7s57Z1dgUVlP5+uYBgKQK0Tkr1WIWngrtUJmRCW1RJFMY7QDla2khXhmg2i5Io7MUjcmZNZ/KQ=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], - "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.2.14", "", { "os": "linux", "cpu": "arm64" }, "sha512-BJ2CL19M6uuWesMgnqJzgxoI+N+eV7b0A9O9vLV42y+gitC41fNa8KElau1tHVzuLWX5GzEuxMPhAO0tQXQMkg=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], - "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.2.14", "", { "os": "linux", "cpu": "ppc64" }, "sha512-4+Kl8daPV8NDHaD7mfCcc1pRM+Qg7ab8DyMm5FG5WoZgv4h2i3KFN1CybYBNbQqmuYm7ftJLvmMI1eHYJ22S/Q=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], - "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.2.14", "", { "os": "linux", "cpu": "s390x" }, "sha512-V/h8ePq1fKDrmtFMz6xcCN3DKvuvMRrwwD1q0PkBeeo4DpK64biaKvqWGVxqCeYKqeRk7gE7/iKPxLCn08O97A=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], - "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.2.14", "", { "os": "linux", "cpu": "x64" }, "sha512-WxalXzgrLVpKC93xfpaJ3gCXUWzeCr+FRCbDVD2mUAy4L6Vo3lNVz0epUEUpxPWJkLNYUwM4OSIikv36OJ9mLg=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], - "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.2.14", "", { "os": "linux", "cpu": "x64" }, "sha512-3G3tnAmTpdk9jO3GQtyHXqeFD2SlecfTws/Y3M2mmj9nMBwwuLT4kur8sZKeu7ENUHSpVO3cIsm5A52LEWWwlg=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], - "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.2.14", "", { "os": "none", "cpu": "arm64" }, "sha512-qyr5XiNGxm2ytx17vvtcJ9nXn6v1gENETi503bJx3OGwZ1tSonj7BosxbMa/xeAfSl5iS9X6KjEWYXLu6v1L7Q=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], - "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.2.14", "", { "os": "win32", "cpu": "arm64" }, "sha512-oA4o5o0S2yB199YyYXDzKUv1Z56Rc+mgOjwm7Z/vJ9Z8igiDPYvoOuDyo6g7AK/ZaF09nPhDKVnGBcYHO11GXA=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], - "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.2.14", "", { "os": "win32", "cpu": "x64" }, "sha512-9mE94bf2GZUaR/fUSXCMpBoWkctu421Mi0ORpm4zqfbkjKep7RxzPFa5etcGs5h+2KApoWBOA/hAJa/7saGgEQ=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], - "@types/lz-string": ["@types/lz-string@1.5.0", "", { "dependencies": { "lz-string": "*" } }, "sha512-s84fKOrzqqNCAPljhVyC5TjAo6BH4jKHw9NRNFNiRUY5QSgZCmVm5XILlWbisiKl+0OcS7eWihmKGS5akc2iQw=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], - "@types/pako": ["@types/pako@2.0.4", "", {}, "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], - "@types/plotly.js": ["@types/plotly.js@3.0.15", "", {}, "sha512-ftBxk2fB7BxxeMpT0IVEb/3HKpYTXO5nCdR5mGEz6gHpLd6GGRFxvDVoG1gpD/gBAgoN36KIyRflxCnOfWlskQ=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], - "@types/raf": ["@types/raf@3.4.3", "", {}, "sha512-c4YAvMedbPZ5tEyxzQdMoOhhJ4RD3rngZIdwC2/qDN3d7JpEhB6fiBRKVY1lg5B7Wk+uPBjn5f39j1/2MY1oOw=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], - "@types/react": ["@types/react@19.3.0", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], - "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], - "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], + + "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + + "@remix-run/router": ["@remix-run/router@1.23.2", "", {}, "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w=="], + + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="], + + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.59.0", "", { "os": "android", "cpu": "arm" }, "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg=="], + + "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.59.0", "", { "os": "android", "cpu": "arm64" }, "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q=="], + + "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.59.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg=="], + + "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.59.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w=="], + + "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.59.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA=="], - "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.59.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg=="], - "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw=="], - "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.59.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA=="], - "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA=="], - "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.59.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA=="], - "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg=="], - "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + "@rollup/rollup-linux-loong64-musl": ["@rollup/rollup-linux-loong64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q=="], - "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA=="], - "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + "@rollup/rollup-linux-ppc64-musl": ["@rollup/rollup-linux-ppc64-musl@4.59.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA=="], - "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg=="], - "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.59.0", "", { "os": "linux", "cpu": "none" }, "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg=="], - "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.59.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w=="], - "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg=="], - "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.59.0", "", { "os": "linux", "cpu": "x64" }, "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg=="], - "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + "@rollup/rollup-openbsd-x64": ["@rollup/rollup-openbsd-x64@4.59.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ=="], - "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.59.0", "", { "os": "none", "cpu": "arm64" }, "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA=="], - "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.59.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A=="], - "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.59.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA=="], - "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA=="], - "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.59.0", "", { "os": "win32", "cpu": "x64" }, "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA=="], + + "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], + + "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], + + "@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="], + + "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], + + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + + "@types/lz-string": ["@types/lz-string@1.5.0", "", { "dependencies": { "lz-string": "*" } }, "sha512-s84fKOrzqqNCAPljhVyC5TjAo6BH4jKHw9NRNFNiRUY5QSgZCmVm5XILlWbisiKl+0OcS7eWihmKGS5akc2iQw=="], + + "@types/pako": ["@types/pako@2.0.4", "", {}, "sha512-VWDCbrLeVXJM9fihYodcLiIv0ku+AlOa/TQ1SvYOaBuyrSKgEcro95LJyIsJ4vSo6BXIxOKxiJAat04CmST9Fw=="], + + "@types/plotly.js": ["@types/plotly.js@2.35.14", "", {}, "sha512-CcD/32JcK19+xWH4FFpmYez/5X9kOjUcBr8Hxh7gQ/3Z32gIoLLy/L9xvC7DG5YikPvJjq6QN05B9+MCRu/Ncw=="], + + "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], + + "@types/raf": ["@types/raf@3.4.3", "", {}, "sha512-c4YAvMedbPZ5tEyxzQdMoOhhJ4RD3rngZIdwC2/qDN3d7JpEhB6fiBRKVY1lg5B7Wk+uPBjn5f39j1/2MY1oOw=="], + + "@types/react": ["@types/react@18.3.28", "", { "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" } }, "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw=="], + + "@types/react-dom": ["@types/react-dom@18.3.7", "", { "peerDependencies": { "@types/react": "^18.0.0" } }, "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ=="], + + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], "@upsetjs/model": ["@upsetjs/model@1.11.0", "", {}, "sha512-pQEdunei2WC9mdhFR6umzE45j8VJfei/cG0TyVaaC0YE5zh1WI/TJRJJQC5jCiGhylISy99ANJs6XmQI33lqCw=="], "@upsetjs/react": ["@upsetjs/react@1.11.0", "", { "dependencies": { "@types/lz-string": "^1.3.34", "@types/react": "^17.0.38", "@upsetjs/model": "~1.11.0", "lz-string": "^1.4.4" }, "peerDependencies": { "react": ">=17" } }, "sha512-APDFkXowPk1U94Iz5dtzpjcSiR5GcJkhLdMdeTnyLDYxIBf1mA+ElbBrEdAzaSGCiZo76ZKfgmuCMSMEfLtN8g=="], - "@vitejs/plugin-react": ["@vitejs/plugin-react@6.1.2", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "oxc-transform-react": "^0.152.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler", "oxc-transform-react"] }, "sha512-fhJNGUdscQa4HkI0xFxwonWyhnVReCPDbXI7FH7jmzpnVCrFQI7PRrhWecO263610vOrs9tr2Y4xcf8uAdFxbw=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="], "base64-arraybuffer": ["base64-arraybuffer@1.0.2", "", {}, "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ=="], + "baseline-browser-mapping": ["baseline-browser-mapping@2.10.8", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-PCLz/LXGBsNTErbtB6i5u4eLpHeMfi93aUv5duMmj6caNu6IphS4q6UevDnL36sZQv9lrP11dbPKGMaXPwMKfQ=="], + + "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], + + "caniuse-lite": ["caniuse-lite@1.0.30001778", "", {}, "sha512-PN7uxFL+ExFJO61aVmP1aIEG4i9whQd4eoSCebav62UwDyp5OHh06zN4jqKSMePVgxHifCw1QJxdRkA1Pisekg=="], + "canvg": ["canvg@3.0.11", "", { "dependencies": { "@babel/runtime": "^7.12.5", "@types/raf": "^3.4.0", "core-js": "^3.8.3", "raf": "^3.4.1", "regenerator-runtime": "^0.13.7", "rgbcolor": "^1.0.1", "stackblur-canvas": "^2.0.0", "svg-pathdata": "^6.0.3" } }, "sha512-5ON+q7jCTgMp9cjpu4Jo6XbvfYwSB2Ow3kzHKfIyJfaCAOHLbdKPQqGKgfED/R5B+3TFFfe8pegYA+b423SRyA=="], - "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], + "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], "core-js": ["core-js@3.49.0", "", {}, "sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg=="], @@ -138,10 +236,16 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], - "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "dompurify": ["dompurify@3.4.16", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ=="], + "electron-to-chromium": ["electron-to-chromium@1.5.313", "", {}, "sha512-QBMrTWEf00GXZmJyx2lbYD45jpI3TUFnNIzJ5BBc8piGUDwMPa1GV6HJWTZVvY/eiN3fSopl7NRbgGp9sZ9LTA=="], + + "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + + "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + "fast-png": ["fast-png@6.4.0", "", { "dependencies": { "@types/pako": "^2.0.3", "iobuffer": "^5.3.2", "pako": "^2.1.0" } }, "sha512-kAqZq1TlgBjZcLr5mcN6NP5Rv4V2f22z00c3g8vRrwkcqjerx7BEhPbOnWCPqaHUl2XWQBJQvOT/FQhdMT7X/Q=="], "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], @@ -152,39 +256,31 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + "html2canvas": ["html2canvas@1.4.1", "", { "dependencies": { "css-line-break": "^2.1.0", "text-segmentation": "^1.0.3" } }, "sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA=="], "iobuffer": ["iobuffer@5.4.0", "", {}, "sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA=="], - "jspdf": ["jspdf@4.2.1", "", { "dependencies": { "@babel/runtime": "^7.28.6", "fast-png": "^6.2.0", "fflate": "^0.8.1" }, "optionalDependencies": { "canvg": "^3.0.11", "core-js": "^3.6.0", "dompurify": "^3.3.1", "html2canvas": "^1.0.0-rc.5" } }, "sha512-YyAXyvnmjTbR4bHQRLzex3CuINCDlQnBqoSYyjJwTP2x9jDLuKDzy7aKUl0hgx3uhcl7xzg32agn5vlie6HIlQ=="], - - "lightningcss": ["lightningcss@1.33.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.33.0", "lightningcss-darwin-arm64": "1.33.0", "lightningcss-darwin-x64": "1.33.0", "lightningcss-freebsd-x64": "1.33.0", "lightningcss-linux-arm-gnueabihf": "1.33.0", "lightningcss-linux-arm64-gnu": "1.33.0", "lightningcss-linux-arm64-musl": "1.33.0", "lightningcss-linux-x64-gnu": "1.33.0", "lightningcss-linux-x64-musl": "1.33.0", "lightningcss-win32-arm64-msvc": "1.33.0", "lightningcss-win32-x64-msvc": "1.33.0" } }, "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA=="], - - "lightningcss-android-arm64": ["lightningcss-android-arm64@1.33.0", "", { "os": "android", "cpu": "arm64" }, "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg=="], + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], - "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.33.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], - "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.33.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ=="], + "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], - "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.33.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg=="], - - "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.33.0", "", { "os": "linux", "cpu": "arm" }, "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ=="], - - "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg=="], + "jspdf": ["jspdf@4.2.1", "", { "dependencies": { "@babel/runtime": "^7.28.6", "fast-png": "^6.2.0", "fflate": "^0.8.1" }, "optionalDependencies": { "canvg": "^3.0.11", "core-js": "^3.6.0", "dompurify": "^3.3.1", "html2canvas": "^1.0.0-rc.5" } }, "sha512-YyAXyvnmjTbR4bHQRLzex3CuINCDlQnBqoSYyjJwTP2x9jDLuKDzy7aKUl0hgx3uhcl7xzg32agn5vlie6HIlQ=="], - "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.33.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ=="], + "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], - "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg=="], + "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], - "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.33.0", "", { "os": "linux", "cpu": "x64" }, "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw=="], + "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], - "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.33.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], - "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], - "lz-string": ["lz-string@1.5.0", "", { "bin": { "lz-string": "bin/bin.js" } }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], - - "nanoid": ["nanoid@3.3.20", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ=="], + "node-releases": ["node-releases@2.0.36", "", {}, "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA=="], "pako": ["pako@2.2.0", "", {}, "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w=="], @@ -192,33 +288,35 @@ "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - "picomatch": ["picomatch@4.0.7", "", {}, "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA=="], + "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], - "plotly.js-dist-min": ["plotly.js-dist-min@4.1.2", "", {}, "sha512-VSmS9KVNun16S8qFYYp9LlL/XBby4ueeGqB+K9AqRCVsup8dSpGULap4UyR1f2mz4pV3qjWp1sR5chwLgIKPoQ=="], + "plotly.js-dist-min": ["plotly.js-dist-min@2.35.3", "", {}, "sha512-sz2HLP8gkysLx/BanM2PtJTtZ1PLPwdHwMWNri2YxLBy3IOeuDsVQtlmWa4hoK3j/fi4naaD3uZJqH5ozM3zGg=="], - "postcss": ["postcss@8.5.30", "", { "dependencies": { "nanoid": "^3.3.20", "picocolors": "^1.1.1", "source-map-js": "^1.2.2" } }, "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA=="], + "postcss": ["postcss@8.5.8", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg=="], "raf": ["raf@3.4.1", "", { "dependencies": { "performance-now": "^2.1.0" } }, "sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA=="], - "react": ["react@19.3.0", "", {}, "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog=="], + "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], + + "react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="], - "react-dom": ["react-dom@19.3.0", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.0" } }, "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q=="], + "react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="], - "react-router": ["react-router@7.18.4", "", { "dependencies": { "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, "optionalPeers": ["react-dom"] }, "sha512-PUPQcMhMGRAslLcvtlPz/kmzBEWPhLdgLFrL7pLNepBL6dX0lWj4WD2cUYVgYCuT3jxvghYFg81cDTj44DhetQ=="], + "react-router": ["react-router@6.30.3", "", { "dependencies": { "@remix-run/router": "1.23.2" }, "peerDependencies": { "react": ">=16.8" } }, "sha512-XRnlbKMTmktBkjCLE8/XcZFlnHvr2Ltdr1eJX4idL55/9BbORzyZEaIkBFDhFGCEWBBItsVrDxwx3gnisMitdw=="], - "react-router-dom": ["react-router-dom@7.18.4", "", { "dependencies": { "react-router": "7.18.4" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" } }, "sha512-yrfmJHIpDG7taCpqKjT1G5B6q3O2K+RN8/fgNf0lTjCwiPbQ0ei6vXX9ZjQR+7ld8Tr7Z5xmyMnZ8YJrphWQUw=="], + "react-router-dom": ["react-router-dom@6.30.3", "", { "dependencies": { "@remix-run/router": "1.23.2", "react-router": "6.30.3" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-pxPcv1AczD4vso7G4Z3TKcvlxK7g7TNt3/FNGMhfqyntocvYKj+GCatfigGDjbLozC4baguJ0ReCigoDJXb0ag=="], "regenerator-runtime": ["regenerator-runtime@0.13.11", "", {}, "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="], "rgbcolor": ["rgbcolor@1.0.1", "", {}, "sha512-9aZLIrhRaD97sgVhtJOW6ckOEh6/GnvQtdVNfdZ6s67+3/XwLS9lBcQYzEEhYVeUowN7pRzMLsyGhK2i/xvWbw=="], - "rolldown": ["rolldown@1.2.14", "", { "dependencies": { "@oxc-project/types": "=0.154.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm-eabi": "1.2.14", "@rolldown/binding-android-arm64": "1.2.14", "@rolldown/binding-darwin-arm64": "1.2.14", "@rolldown/binding-darwin-x64": "1.2.14", "@rolldown/binding-freebsd-x64": "1.2.14", "@rolldown/binding-linux-arm-gnueabihf": "1.2.14", "@rolldown/binding-linux-arm64-gnu": "1.2.14", "@rolldown/binding-linux-arm64-musl": "1.2.14", "@rolldown/binding-linux-ppc64-gnu": "1.2.14", "@rolldown/binding-linux-s390x-gnu": "1.2.14", "@rolldown/binding-linux-x64-gnu": "1.2.14", "@rolldown/binding-linux-x64-musl": "1.2.14", "@rolldown/binding-openharmony-arm64": "1.2.14", "@rolldown/binding-win32-arm64-msvc": "1.2.14", "@rolldown/binding-win32-x64-msvc": "1.2.14" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-eTBTTqMR6uHKLgcUqpJP8ntzDI6V4LYA8negAMBEI7gkgVvOdrx6VsvS2Ym+83KhOPdLCHWJu0w7UQFJZhHKNw=="], + "rollup": ["rollup@4.59.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.59.0", "@rollup/rollup-android-arm64": "4.59.0", "@rollup/rollup-darwin-arm64": "4.59.0", "@rollup/rollup-darwin-x64": "4.59.0", "@rollup/rollup-freebsd-arm64": "4.59.0", "@rollup/rollup-freebsd-x64": "4.59.0", "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", "@rollup/rollup-linux-arm-musleabihf": "4.59.0", "@rollup/rollup-linux-arm64-gnu": "4.59.0", "@rollup/rollup-linux-arm64-musl": "4.59.0", "@rollup/rollup-linux-loong64-gnu": "4.59.0", "@rollup/rollup-linux-loong64-musl": "4.59.0", "@rollup/rollup-linux-ppc64-gnu": "4.59.0", "@rollup/rollup-linux-ppc64-musl": "4.59.0", "@rollup/rollup-linux-riscv64-gnu": "4.59.0", "@rollup/rollup-linux-riscv64-musl": "4.59.0", "@rollup/rollup-linux-s390x-gnu": "4.59.0", "@rollup/rollup-linux-x64-gnu": "4.59.0", "@rollup/rollup-linux-x64-musl": "4.59.0", "@rollup/rollup-openbsd-x64": "4.59.0", "@rollup/rollup-openharmony-arm64": "4.59.0", "@rollup/rollup-win32-arm64-msvc": "4.59.0", "@rollup/rollup-win32-ia32-msvc": "4.59.0", "@rollup/rollup-win32-x64-gnu": "4.59.0", "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg=="], - "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], + "scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="], - "set-cookie-parser": ["set-cookie-parser@2.7.2", "", {}, "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw=="], + "semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], - "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], "specificity": ["specificity@0.4.1", "", { "bin": { "specificity": "./bin/specificity" } }, "sha512-1klA3Gi5PD1Wv9Q0wUoOQN1IWAuPu0D1U03ThXTr0cJ20+/iq2tHSDnK7Kk/0LXJ1ztUB2/1Os0wKmfyNgUQfg=="], @@ -232,12 +330,16 @@ "text-segmentation": ["text-segmentation@1.0.3", "", { "dependencies": { "utrie": "^1.0.2" } }, "sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw=="], - "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], + "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], "utrie": ["utrie@1.0.2", "", { "dependencies": { "base64-arraybuffer": "^1.0.2" } }, "sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw=="], - "vite": ["vite@8.3.4", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", "postcss": "^8.5.29", "rolldown": "~1.2.12", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ=="], + "vite": ["vite@6.4.1", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g=="], + + "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], } } diff --git a/frontend/package.json b/frontend/package.json index ab8e600e..8793ef05 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,19 +13,19 @@ "dependencies": { "@upsetjs/react": "^1.11.0", "jspdf": "^4.2.1", - "plotly.js-dist-min": "^4.1.2", - "react": "^19.3.0", - "react-dom": "^19.3.0", - "react-router-dom": "^7.18.4", + "plotly.js-dist-min": "^2.35.2", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^6.28.0", "svg2pdf.js": "^2.8.1" }, "devDependencies": { - "@types/plotly.js": "^3.0.15", - "@types/react": "^19.3.0", - "@types/react-dom": "^19.3.0", - "@vitejs/plugin-react": "^6.1.2", - "typescript": "^7.0.2", - "vite": "^8.3.4" + "@types/plotly.js": "^2.33.4", + "@types/react": "^18.3.18", + "@types/react-dom": "^18.3.5", + "@vitejs/plugin-react": "^4.3.4", + "typescript": "^5.7.2", + "vite": "^6.1.0" }, "overrides": { "@types/react": "$@types/react", diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 4ea20add..87457dad 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -439,8 +439,6 @@ export interface DifferentialResult { min_prevalence: number /** nb_glm only. */ offset?: 'tss' | 'rle' - /** nb_glm only: where each taxon's theta comes from. */ - dispersion?: 'ml' | 'glmGamPoi' /** clr_welch only. */ replacement_delta?: number } diff --git a/frontend/src/components/AnalysisChart.tsx b/frontend/src/components/AnalysisChart.tsx index 17e8a437..3e40f157 100644 --- a/frontend/src/components/AnalysisChart.tsx +++ b/frontend/src/components/AnalysisChart.tsx @@ -1,14 +1,13 @@ // © 2026 Joshua Benjamin Jewell. All rights reserved. // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import Plotly from 'plotly.js-dist-min' -import { stripHtmlTags } from '../utils/text' import { AddToReport } from './AddToReport' import { PlotlyChart } from './PlotlyChart' const figureTitle = (fig: unknown): string | null => { const t = (fig as { layout?: { title?: unknown } })?.layout?.title const text = typeof t === 'string' ? t : (t as { text?: string } | undefined)?.text - return text ? stripHtmlTags(text) : null + return text ? text.replace(/<[^>]+>/g, '') : null } // The chart as standalone SVG, at a fixed print width. diff --git a/frontend/src/components/CompositionPanel.tsx b/frontend/src/components/CompositionPanel.tsx index 5e677683..08863fc3 100644 --- a/frontend/src/components/CompositionPanel.tsx +++ b/frontend/src/components/CompositionPanel.tsx @@ -16,12 +16,6 @@ import type { // Fixed colour of the catch-all "Unassigned" bucket, mirroring the backend. const UNASSIGNED_COLOUR = '#95a5a6' -/** - * Browse a run's composition by taxonomy rank or category, with sub-group and - * results-table selectors and controls to save category colours. The analysis - * context's annotation source takes precedence over `source`; absent both, - * VSEARCH is used. A failed table lookup clears the options and shows a toast. - */ export function CompositionPanel({ study, run, group, subgroups, source: configuredSource, }: { diff --git a/frontend/src/components/TaxaCompositionChart.tsx b/frontend/src/components/TaxaCompositionChart.tsx index 9601aebd..f14657a3 100644 --- a/frontend/src/components/TaxaCompositionChart.tsx +++ b/frontend/src/components/TaxaCompositionChart.tsx @@ -24,13 +24,6 @@ const LABEL_STYLE: React.CSSProperties = { gap: 4, } -/** - * Select and compute a taxon or category bar chart for one run, or for `runs` - * when that list is non-empty, with optional facets across runs. `table` - * defaults to merged. Providing `subgroup`, including null for all sub-groups, - * controls that selection and hides the internal selector; facets set their - * own scope. Changed inputs clear the figure; chart failures show a toast. - */ export function TaxaCompositionChart({ study, run, diff --git a/frontend/src/figure/charts.ts b/frontend/src/figure/charts.ts index 6ee30a69..29845147 100644 --- a/frontend/src/figure/charts.ts +++ b/frontend/src/figure/charts.ts @@ -2,7 +2,6 @@ // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import { api } from '../api/client' import { extractAlphaPanel } from '../components/alphaMetrics' -import { stripHtmlTags } from '../utils/text' import { textWidth } from './legend' import { PX_PER_PT, tickPt, titlePt, type AlphaMetricKey, type ChartSpec, type FigureDoc, type FigurePane } from './types' @@ -120,7 +119,7 @@ export function paneFigure(raw: unknown, pane: FigurePane, doc: FigureDoc, opts: // A title wider than its pane shrinks to fit, clear of the group letter. const titleRoom = opts.widthPx - 2 * doc.letters.sizePt * PX_PER_PT const titleSize = Math.min(titlePt(doc) * PX_PER_PT, - titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(stripHtmlTags(title), titlePt(doc) * PX_PER_PT, family))) + titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(title.replace(/<[^>]+>/g, ''), titlePt(doc) * PX_PER_PT, family))) for (const k of axisKeys(L, 'x').concat(axisKeys(L, 'y'))) { const ax = { ...(L[k] as Record) } diff --git a/frontend/src/utils/text.test.ts b/frontend/src/utils/text.test.ts deleted file mode 100644 index f5ed71b9..00000000 --- a/frontend/src/utils/text.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -// © 2026 Joshua Benjamin Jewell. All rights reserved. -// Licensed under the GNU Affero General Public License version 3 (AGPLv3). -import { describe, expect, test } from 'bun:test' -import { splitLines, stripHtmlTags } from './text' - -describe('stripHtmlTags', () => { - test('removes the markup Plotly allows in a title', () => { - expect(stripHtmlTags('CO2 by run')).toBe('CO2 by run') - }) - - test('a tag hiding inside another does not survive', () => { - expect(stripHtmlTags('ipt>alert(1)')).not.toMatch(/<[^>]+>/) - }) - - test('text without markup, including a bare >, is unchanged', () => { - expect(stripHtmlTags('p > 0.05 for all')).toBe('p > 0.05 for all') - }) -}) - -describe('splitLines', () => { - test('one trimmed value per line, blanks dropped', () => { - expect(splitLines(' a\n\n b \nc')).toEqual(['a', 'b', 'c']) - }) -}) diff --git a/frontend/src/utils/text.ts b/frontend/src/utils/text.ts index ffdecc9c..dc076655 100644 --- a/frontend/src/utils/text.ts +++ b/frontend/src/utils/text.ts @@ -9,20 +9,3 @@ export function splitLines(text: string): string[] { return text.split('\n').map(v => v.trim()).filter(v => v.length > 0) } - -/** - * Strip the pseudo-HTML markup Plotly accepts in titles and axis labels (, - *
, and friends), leaving the bare text for measuring or for a - * filename. A single pass is not enough: replacing "ipt>" once - * removes the inner tag and leaves the halves joined into another, so the - * strip repeats until the text stops changing. - */ -export function stripHtmlTags(text: string): string { - let stripped = text - let previous = '' - do { - previous = stripped - stripped = stripped.replace(/<[^>]+>/g, '') - } while (stripped !== previous) - return stripped -} diff --git a/frontend/src/views/TreeView.tsx b/frontend/src/views/TreeView.tsx index 8147ac29..c0778282 100644 --- a/frontend/src/views/TreeView.tsx +++ b/frontend/src/views/TreeView.tsx @@ -183,9 +183,7 @@ function TreeCanvas({ source, tree, src, toast }: { // Autosave the view document. const firstSave = useRef(true) - // React 19's types dropped the zero-argument useRef overload, so the timer - // id starts as an explicit undefined instead. - const saveTimer = useRef(undefined) + const saveTimer = useRef() const saveNow = useCallback(() => { const doc: TreeViewDoc = { version: 1, @@ -230,7 +228,7 @@ function TreeCanvas({ source, tree, src, toast }: { el.addEventListener('wheel', onWheel, { passive: false }) return () => el.removeEventListener('wheel', onWheel) }, []) - const scrollTimer = useRef(undefined) + const scrollTimer = useRef() const onScroll = () => { const el = scrollRef.current if (!el) return diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 5aea76c5..e25db006 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -9,9 +9,9 @@ export default defineConfig({ sourcemap: false, rollupOptions: { output: { - // vite 8 (rolldown) took only the function form; the old object form - // died at "manualChunks is not a function". - manualChunks: (id: string) => (id.includes('plotly.js-dist-min') ? 'plotly' : undefined), + manualChunks: { + plotly: ['plotly.js-dist-min'], + }, }, }, }, diff --git a/scripts/install.jl b/install.jl similarity index 99% rename from scripts/install.jl rename to install.jl index f3465b3b..eb8e9e47 100644 --- a/scripts/install.jl +++ b/install.jl @@ -6,7 +6,7 @@ # required R packages. Writes resolved tool paths to config/tools.yml. # # Usage via install.sh, or directly by: -# julia --project=. scripts/install.jl [--update] [--modify] [--sysimage] +# julia --project=. install.jl [--update] [--modify] [--sysimage] # # Options: # --update Re-assert the pinned tool versions and refresh managed binaries in bin/ @@ -41,7 +41,7 @@ import Downloads const UPDATE_MODE = "--update" in ARGS const MODIFY_MODE = "--modify" in ARGS const SYSIMAGE_MODE = "--sysimage" in ARGS -const PROJECT_ROOT = normpath(dirname(@__DIR__)) +const PROJECT_ROOT = @__DIR__ const BIN_DIR = joinpath(PROJECT_ROOT, "bin") const CONFIG_DIR = joinpath(PROJECT_ROOT, "config") const TOOLS_CONFIG = joinpath(CONFIG_DIR, "tools.yml") @@ -869,14 +869,14 @@ function setup_r_packages(; rebuild::Bool=false)::Tuple{Symbol,String} # renv's exit code: restore can fail on one leaf package and still leave a # working library, or "succeed" against a stale cache. probe = raw""" - pkgs <- c("dada2", "vegan", "dplyr", "tibble") + pkgs <- c("dada2", "vegan", "tibble") cat(paste(pkgs[!vapply(pkgs, requireNamespace, logical(1), quietly = TRUE)], collapse = ",")) """ bad = try strip(read(Cmd(`Rscript -e $probe`; dir = PROJECT_ROOT), String)) catch - "dada2,vegan,dplyr,tibble" + "dada2,vegan,tibble" end isempty(bad) ? (:ok, "") : (:incomplete, String(bad)) @@ -1025,7 +1025,7 @@ end ## Sysimage creation const SYSIMAGE_EXT = Sys.isapple() ? ".dylib" : ".so" const SYSIMAGE_PATH = joinpath(PROJECT_ROOT, "MetaManifold$(SYSIMAGE_EXT)") -const PRECOMPILE_EXEC_PATH = joinpath(@__DIR__, "precompile_exec.jl") +const PRECOMPILE_EXEC_PATH = joinpath(PROJECT_ROOT, "precompile_exec.jl") function build_sysimage() # PackageCompiler is a project dependency, installed by Pkg.instantiate(). @@ -1247,7 +1247,7 @@ function main() record!("Julia sysimage", STEP_OK, SYSIMAGE_PATH) catch e @error "Sysimage build failed: $e" - record!("Julia sysimage", STEP_FAILED, "rerun: julia --project=. scripts/install.jl --sysimage") + record!("Julia sysimage", STEP_FAILED, "rerun: julia --project=. install.jl --sysimage") end else record!("Julia sysimage", STEP_OK, "not requested (pass --sysimage to build one)") diff --git a/install.sh b/install.sh index a8e0b718..c5e0a18e 100755 --- a/install.sh +++ b/install.sh @@ -283,4 +283,4 @@ fi echo "" echo "Running install.jl..." echo "" -julia --project=. scripts/install.jl "$@" +julia --project=. install.jl "$@" diff --git a/launcher.sh b/launcher.sh deleted file mode 100755 index d37ba967..00000000 --- a/launcher.sh +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# @launcher-deed begin -# ;; SPDX-License-Identifier: MPL-2.0 -# (praxis-deed -# :schema-version "1.0.0" -# :canonical-name "metamanifold-webui-launcher" -# (artefact :type "launcher" :version "0.1.0" -# :generator "hand-adapted from launcher-standard 0.6.0's archetype") -# (app :name "MetaManifold" :display "MetaManifold" -# :url "https://github.com/hyperpolymath/MetaManifold-WebUI" -# :archetype "application") -# (compliance :standard-version "0.6.0" -# :standards ("launcher-standard.adoc")) -# (modes :accepted ("--setup" "--start" "--doctor" "--status" "--agent" "--help" "--version")) -# (platforms :supported ("linux" "macos")) -# (lifecycle-phases :covered ("provision" "diagnose" "run") -# :deferred ("heal"))) -# @launcher-deed end -# -# The MetaManifold launcher, in the launcher-standard's mode vocabulary but -# self-contained: this is an application repository, not an RSR-minted one, -# so the modes delegate directly to the repository's own entry points -# (install.sh, start.sh) rather than through provision-set machinery. -# -# The one thing that is this launcher's own is --agent: the git-signing -# agent's persistence is the operator's choice, and every mode of it is -# reversible. The signing key is ~/.ssh/git-signing2; the socket is -# /tmp/git-signing-agent.sock. -# -# --agent systemd a systemd user service owns the agent, and the shell -# profile exports the socket (a service cannot export -# into login shells). Survives reboots; one instance. -# --agent profile the shell profile starts the agent if the socket is -# dead and exports it. No systemd; works over plain ssh. -# --agent both the service plus the spawn-if-dead fallback line. -# --agent none removes whatever of the above was installed. -# --agent status reports socket, key, and which mechanisms are present. -# -# Each mechanism is written between marked blocks in the profile and a unit -# file, idempotently, with a dated backup of anything edited. - -set -uo pipefail - -REPO_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" -SOCKET="/tmp/git-signing-agent.sock" -KEY="${HOME}/.ssh/git-signing2" -UNIT_NAME="git-signing-agent" -UNIT_FILE="${HOME}/.config/systemd/user/${UNIT_NAME}.service" -PROFILE="${HOME}/.profile" -BLOCK_BEGIN="# >>> metamanifold git-signing agent >>>" -BLOCK_END="# <<< metamanifold git-signing agent <<<" - -# Print all arguments as one launcher-prefixed message to standard output. -say() { printf 'launcher: %s\n' "$*"; } - -# Print all arguments as one launcher-prefixed message to standard error. -warn() { printf 'launcher: %s\n' "$*" >&2; } - -# Report all arguments through warn and terminate the launcher with status 1. -die() { warn "$*"; exit 1; } - -# Return 0 only when SOCKET is a Unix socket and ssh-add can list identities. -# An agent with no loaded identities also returns non-zero. -agent_alive() { - [ -S "$SOCKET" ] && SSH_AUTH_SOCK="$SOCKET" ssh-add -l >/dev/null 2>&1 -} - -# Remove marked agent blocks from PROFILE, following symlinks and preserving -# file permissions. A missing profile succeeds. Return 1 for nested or -# unterminated blocks, or file-operation failures; no backup is made here. -remove_profile_block() { - [ -f "$PROFILE" ] || return 0 - local target tmp mode - target="$(readlink -f "$PROFILE")" || return 1 - tmp="$(mktemp "$(dirname "$target")/.profile.tmp.XXXXXX")" || return 1 - if ! awk -v b="$BLOCK_BEGIN" -v e="$BLOCK_END" ' - $0 == b { - if (skip) exit 1 - skip = 1 - next - } - $0 == e { - if (skip) { skip = 0; next } - print - next - } - !skip { print } - END { if (skip) exit 1 } - ' "$target" > "$tmp"; then - rm -f "$tmp" - return 1 - fi - mode="$(stat -c '%a' "$target" 2>/dev/null || stat -f '%Lp' "$target")" || { - rm -f "$tmp" - return 1 - } - chmod "$mode" "$tmp" || { - rm -f "$tmp" - return 1 - } - mv "$tmp" "$target" || { - rm -f "$tmp" - return 1 - } -} - -# Back up PROFILE and replace its marked agent block, creating the file if absent. -# "export" sets the socket for future shells; "spawn" also tries to start an -# agent when ssh-add cannot list identities. Neither mode loads the signing key. -# Exit 1 if profile creation, backup or block removal fails; append failures -# return a non-zero status and do not restore the backup. -write_profile_block() { # $1 = "export" | "spawn" - [ -f "$PROFILE" ] || touch "$PROFILE" || die "could not create $PROFILE" - local backup - backup="$(mktemp "$PROFILE.launcher-backup.$(date +%Y%m%d%H%M%S).XXXXXX")" || - die "could not create backup of $PROFILE" - cp -p "$PROFILE" "$backup" || { - rm -f "$backup" - die "backup of $PROFILE failed" - } - remove_profile_block || die "could not update $PROFILE" - { - printf '%s\n' "$BLOCK_BEGIN" - if [ "$1" = "spawn" ]; then - printf 'if [ ! -S "%s" ] || ! SSH_AUTH_SOCK="%s" ssh-add -l >/dev/null 2>&1; then\n' "$SOCKET" "$SOCKET" - printf ' ssh-agent -a "%s" >/dev/null 2>&1\n' "$SOCKET" - printf 'fi\n' - fi - printf 'export SSH_AUTH_SOCK="%s"\n' "$SOCKET" - printf '%s\n' "$BLOCK_END" - } >> "$PROFILE" -} - -# Create or overwrite UNIT_FILE for an ssh-agent on SOCKET, without a backup. -# Return the write status; this does not enable or start the service. -write_unit() { - mkdir -p "$(dirname "$UNIT_FILE")" - cat > "$UNIT_FILE" </dev/null || die "systemctl not found; use --agent profile" - write_unit - systemctl --user daemon-reload - systemctl --user enable --now "$UNIT_NAME" || die "could not enable ${UNIT_NAME} (try: loginctl enable-linger)" - write_profile_block export - say "systemd user service enabled; profile exports SSH_AUTH_SOCK=$SOCKET" - say "note: over plain ssh, or without a user session, also run: loginctl enable-linger \$USER" - ;; - profile) - write_profile_block spawn - say "profile now starts the agent if the socket is dead and exports SSH_AUTH_SOCK=$SOCKET" - ;; - both) - command -v systemctl >/dev/null || die "systemctl not found; use --agent profile" - write_unit - systemctl --user daemon-reload - systemctl --user enable --now "$UNIT_NAME" || die "could not enable ${UNIT_NAME}" - write_profile_block spawn - say "systemd service plus spawn-if-dead profile fallback installed" - ;; - none) - if command -v systemctl >/dev/null 2>&1; then - systemctl --user disable --now "$UNIT_NAME" >/dev/null 2>&1 || - warn "could not disable $UNIT_NAME" - else - warn "systemctl not found; user unit was not disabled" - fi - rm -f "$UNIT_FILE" - if command -v systemctl >/dev/null 2>&1; then - systemctl --user daemon-reload || - warn "could not reload the user manager" - fi - if [ -f "$PROFILE" ]; then - backup="$(mktemp "$PROFILE.launcher-backup.$(date +%Y%m%d%H%M%S).XXXXXX")" || { - warn "could not create backup of $PROFILE; profile block was not removed" - return 1 - } - if ! cp -p "$PROFILE" "$backup"; then - rm -f "$backup" - warn "could not back up $PROFILE; profile block was not removed" - return 1 - fi - fi - if ! remove_profile_block; then - warn "could not remove the profile block" - return 1 - fi - say "removed unit and profile block; any agent already running is left alone" - ;; - status) - agent_alive && say "socket alive: $SOCKET" || say "socket dead or absent: $SOCKET" - [ -f "$KEY" ] && say "key present: $KEY" || say "key MISSING: $KEY" - [ -f "$UNIT_FILE" ] && say "systemd unit: installed" || say "systemd unit: not installed" - grep -q "$BLOCK_BEGIN" "$PROFILE" 2>/dev/null && say "profile block: present" || say "profile block: absent" - ;; - *) - die "usage: launcher.sh --agent systemd|profile|both|none|status" - ;; - esac -} - -# Report toolchain and signing readiness, then exit 1 if julia, bun, R or git -# is missing from PATH, or agent_alive fails; otherwise exit 0. Missing tool -# configuration or a signing-key check failure only produces a warning. -doctor() { - fail=0 - for tool in julia bun R git; do - if command -v "$tool" >/dev/null; then - say "ok: $tool on PATH" - else - warn "MISSING: $tool not on PATH" - fail=1 - fi - done - [ -f "$REPO_DIR/config/tools.yml" ] && say "ok: config/tools.yml exists" \ - || warn "config/tools.yml missing (run: ./launcher.sh --setup)" - if agent_alive; then - say "ok: signing agent alive at $SOCKET" - SSH_AUTH_SOCK="$SOCKET" ssh-add -l 2>/dev/null | grep -q "$(ssh-keygen -lf "${KEY}.pub" 2>/dev/null | awk '{print $2}')" \ - && say "ok: signing key loaded" || warn "signing key not loaded (run: ssh-add $KEY)" - else - warn "signing agent not reachable at $SOCKET (see: ./launcher.sh --agent status)" - fail=1 - fi - exit "$fail" -} - -# Print the launcher header and available modes to standard output. -help() { - sed -n '2,40p' "${BASH_SOURCE[0]}" | grep '^#' | sed 's/^# \{0,1\}//' - cat <<'MODES' -Modes: - --setup install everything (delegates to install.sh; args pass through) - --start serve the web UI (delegates to start.sh) - --doctor check the toolchain and the signing agent - --status whether the server looks reachable on its port - --agent MODE systemd | profile | both | none | status (the signing agent) - --help this text - --version the launcher's version -MODES -} - -# Print the launcher and launcher-standard versions to standard output. -version() { echo "metamanifold-webui-launcher 0.1.0 (launcher-standard 0.6.0, hand-adapted)"; } - -# Dispatch the first argument as a launcher mode. Setup replaces this process -# with install.sh and forwards remaining arguments; start executes start.sh. -# Status checks the configured local port (default 8080) with a two-second -# timeout; HTTP error responses count as reachable and connection failure does -# not set a failure exit status. Missing or unknown modes print help and exit 1. -main() { - case "${1:-}" in - --setup) shift; exec bash "$REPO_DIR/install.sh" "$@" ;; - --start) exec bash "$REPO_DIR/start.sh" ;; - --doctor) doctor ;; - --status) - status_url="http://127.0.0.1:${JULIA_METAMANIFOLD_PORT:-8080}/" - if curl -s -o /dev/null --max-time 2 "$status_url"; then - say "something is answering on $status_url" - else - say "nothing answering on $status_url (start with: ./launcher.sh --start)" - fi - ;; - --agent) shift; agent_mode "${1:-}" ;; - --help|-h) help ;; - --version) version ;; - *) help; exit 1 ;; - esac -} - -main "$@" diff --git a/scripts/precompile_exec.jl b/precompile_exec.jl similarity index 100% rename from scripts/precompile_exec.jl rename to precompile_exec.jl diff --git a/proofs/agda/MetaManifold/All.agda b/proofs/agda/MetaManifold/All.agda index 0770f154..ff5c18c0 100644 --- a/proofs/agda/MetaManifold/All.agda +++ b/proofs/agda/MetaManifold/All.agda @@ -13,4 +13,3 @@ import MetaManifold.Prelude import MetaManifold.BenjaminiHochberg import MetaManifold.Scaling import MetaManifold.Permanova -import MetaManifold.Significance diff --git a/proofs/agda/MetaManifold/Permanova.agda b/proofs/agda/MetaManifold/Permanova.agda index fbf0220a..97ed741d 100644 --- a/proofs/agda/MetaManifold/Permanova.agda +++ b/proofs/agda/MetaManifold/Permanova.agda @@ -17,8 +17,8 @@ -- that this happens exactly when every within-block permutation leaves the -- term's labels where they were, so the permutations never exchange them. -- They also say that when it is true, some within-block permutation does --- exchange them. So no testable term is withheld, and no untestable term --- is reported. +-- exchange them. This characterises the block-constancy filter only; +-- it does not establish statistical testability or a reported p-value. -- -- What is NOT modelled, and why: -- @@ -58,7 +58,7 @@ -- withheld⇔never-exchanged ... so its false answer, which withholds the -- p-value, holds exactly when no within-block -- permutation exchanges the term's labels --- tested⇔exchanged ... and its true answer, which reports the p-value, +-- tested⇔exchanged ... and its true answer, which passes this filter, -- holds exactly when some within-block permutation -- does -- constant-in-blocks, varies-in-a-block @@ -411,8 +411,8 @@ module Varies {A B : Set} (_≟ᴬ_ : DecidableEquality A) (_≟ᴮ_ : Decidable (Equivalence.from Constant⇔BlockConstant (Equivalence.from untestable⇔fixed fixed))) - -- Julia reports a term's p-value (`_varies_within_any_block` is true) - -- exactly when some within-block permutation exchanges its labels. + -- The block-constancy filter admits a term exactly when some within-block + -- permutation exchanges its labels. Other checks can still withhold it. tested⇔exchanged : variesWithinAnyBlock xs ≡ true ⇔ Σ (Permutation′ (length xs)) λ π → WithinBlock π × ¬ Fixes π tested⇔exchanged = mk⇔ to from @@ -465,7 +465,7 @@ module Fixtures where Labels.Fixes (blockOf group) (labelOf group) π fixture-group-withheld = Equivalence.to (withheld⇔never-exchanged group) refl - -- run's p-value is reported, and rightly: some within-block permutation + -- run passes the block-constancy filter: some within-block permutation -- exchanges run labels. fixture-run-tested : Σ (Permutation′ 12) λ π → Labels.WithinBlock (blockOf run) (labelOf run) π × diff --git a/proofs/agda/MetaManifold/Significance.agda b/proofs/agda/MetaManifold/Significance.agda deleted file mode 100644 index fcf0f57c..00000000 --- a/proofs/agda/MetaManifold/Significance.agda +++ /dev/null @@ -1,106 +0,0 @@ --- SPDX-License-Identifier: AGPL-3.0-only --- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) --- --- The significance reporting discipline: what a test that did not run may be --- read as, proved. `run_alpha_significance` in `src/analysis/analysis.jl` --- returns an `AlphaSignificance` whose status distinguishes a computed --- omnibus result from an unavailable or failed test (`:r_unavailable`, --- `:no_paired_samples`, `:test_failed`), and whose docstring carries the --- rule: "False means no test was performed, so neither `omnibus` nor --- `pairwise` may be presented as a finding." --- --- The theorems here apply the Observational Null Disclosure discipline --- (hyperpolymath/absolute-zero, the OND pillar) to that rule: --- --- 1. `publish` reads the omnibus p-value only from a result whose test --- ran, and requires evidence that it ran. On a result that did not run, --- the evidence would be an inhabitant of `false ≡ true`, which does not --- exist: the branch cannot produce a number, refused by the typechecker --- -- not asserted by a test. --- 2. `pairwise-published` is the same for the pairwise table. --- 3. `reason` is the dual: the reader-facing explanation can be read only --- from a result that did not run. --- 4. `not-computed⇒unrepresentable`: a result whose status is not --- `computed` carries no numbers by construction -- there is no field to --- read, so an unrun test can never be published as a null result. --- --- Mapped to the Julia (test/unit/test_r_runtime.jl): "a test that did not --- run is not a null result", "a chart states that pairwise tests were not --- run", "the caption distinguishes the reasons a test did not run", and "an --- omnibus statistic R cannot compute is a failed test". --- --- What is modelled, and what is not: the statuses are Julia's; the omnibus --- p-value is an exact rational (the Julia holds a Float64); the pairwise --- table is modelled as the list of its p-values, not the DataFrame; the --- reason strings are opaque. The statistics themselves are NOT modelled -- --- this module is about what may be read from a result, not how the result --- was computed. - -{-# OPTIONS --safe --without-K #-} - -module MetaManifold.Significance where - -open import Data.Bool using (Bool; true; false) -open import Data.List using (List; map) -open import Data.Rational using (ℚ) -open import Data.String using (String) -open import Relation.Nullary using (¬_) -open import Relation.Binary.PropositionalEquality using (_≡_; refl; cong) - --- Julia's statuses: :computed, :r_unavailable, :no_paired_samples, :test_failed. -data Status : Set where - computed r-unavailable no-paired-samples test-failed : Status - --- The discipline: the omnibus p-value and the pairwise p-values exist only --- in the constructor that claims the test ran. A result that reveals nothing --- names its status, carries a proof that the status is not `computed`, and a --- reader-facing reason -- and no numbers, because there are none to read. -data AlphaSignificance : Set where - computed-result : (omnibus : ℚ) (pairwise : List ℚ) → AlphaSignificance - reveals-nothing : (status : Status) → ¬ (status ≡ computed) → String → AlphaSignificance - --- Julia's was_computed, mirrored. -was-computed : AlphaSignificance → Bool -was-computed (computed-result _ _) = true -was-computed (reveals-nothing _ _ _) = false - --- 1. Publishing the omnibus p-value requires evidence the test ran. On the --- reveals-nothing branch the evidence is an inhabitant of `false ≡ true`, --- which does not exist, so the branch cannot produce a number at all. -publish : (r : AlphaSignificance) → was-computed r ≡ true → ℚ -publish (computed-result omnibus _) refl = omnibus -publish (reveals-nothing _ _ _) () - --- 2. The pairwise table, same discipline. -pairwise-published : (r : AlphaSignificance) → was-computed r ≡ true → List ℚ -pairwise-published (computed-result _ pairwise) refl = pairwise -pairwise-published (reveals-nothing _ _ _) () - --- 3. The dual: the reader-facing reason can be read only from a result that --- did not run. A computed result has no reason to show, and the evidence --- it would need (`true ≡ false`) does not exist. -reason : (r : AlphaSignificance) → was-computed r ≡ false → String -reason (computed-result _ _) () -reason (reveals-nothing _ _ why) refl = why - --- 4. The status of a result that did not run is never `computed`, by --- construction: the proof obligation lives in the constructor, so a --- `reveals-nothing` value cannot even be built with the `computed` --- status. -unrun-status-not-computed : - (s : Status) (h : ¬ (s ≡ computed)) (why : String) → - let r = reveals-nothing s h why in was-computed r ≡ false -unrun-status-not-computed s h why = refl - --- The status of a computed result is `computed`, and its evidence is the --- constructor's own shape -- so the two constructors never meet: a result --- is either readable as numbers or readable as a reason, never both. -computed-status : (o : ℚ) (p : List ℚ) → - was-computed (computed-result o p) ≡ true -computed-status o p = refl - --- Map every pairwise p-value through a function without changing the --- publish discipline: reading remains possible only on the computed branch. -map-pairwise : (f : ℚ → ℚ) (r : AlphaSignificance) → was-computed r ≡ true → List ℚ -map-pairwise f (computed-result _ pairwise) refl = map f pairwise -map-pairwise f (reveals-nothing _ _ _) () diff --git a/proofs/agda/README.md b/proofs/agda/README.md index 2590a5ad..a7d25cab 100644 --- a/proofs/agda/README.md +++ b/proofs/agda/README.md @@ -122,29 +122,10 @@ returns. | `untestable⇔fixed` | Constant within every block ⇔ fixed by every within-block permutation. | – | – | | `varies-false⇔Constant` | The model of `_varies_within_any_block` returns `false` exactly when no block holds two different values. | `_varies_within_any_block` | lines 47–51 | | `withheld⇔never-exchanged` | So `untestable` (the function returned `false`) holds exactly when no within-block permutation exchanges the term's labels. | `untestable`, `withheld` | lines 92–93 | -| `tested⇔exchanged` | And a p-value is reported (the function returned `true`) exactly when some within-block permutation does. No testable term is withheld. | `untestable`, `withheld` | line 98 | +| `tested⇔exchanged` | The block-constancy filter admits a term exactly when some within-block permutation changes its labels; this does not guarantee a reported p-value. | `untestable`, `withheld` | line 98 | | `constant-in-blocks`, `varies-in-a-block` | The two direct cases in the test, by evaluation. | `_varies_within_any_block` | lines 49–50 | | `fixture-group-withheld`, `fixture-run-tested` | On the test's twelve samples (six individuals, two runs each, three individuals per group), `group` is never exchanged and `run` is. | `run_permanova` | lines 83–98 | -### Significance reporting (`Significance`) - -The Observational Null Disclosure discipline (hyperpolymath/absolute-zero, OND -pillar) applied to `run_alpha_significance`'s reporting rule: a test that did -not run reveals nothing, and must never be published as a null result. The -numbers exist only in the constructor whose status is `computed`; every other -status carries a proof it is not `computed` and a reader-facing reason. - -| Agda theorem | Plain statement | Julia | Test (`test/unit/test_r_runtime.jl`) | -|---|---|---|---| -| `publish` | The omnibus p-value is readable only from a result whose test ran, with evidence it ran; the unrun branch cannot produce a number (the typechecker refuses it). | `was_computed` guards | lines 100–124 | -| `pairwise-published` | The pairwise table, same discipline. | `was_computed` guards | lines 125–152 | -| `reason` | The dual: the reader-facing explanation is readable only from a result that did not run. | the `reason` field | lines 153–175 | -| `unrun-status-not-computed`, `computed-status` | A result is readable as numbers or as a reason, never both: the two constructors cannot meet. | the `AlphaSignificance` struct | lines 176+ | - -What is modelled: the four statuses exactly; the omnibus p-value as an exact -rational; the pairwise table as its list of p-values. What is not: the -statistics themselves, and the DataFrame shape. - ## Corrections to the requirement wording - **R-BH-2a is true for the procedure.** An earlier archived note recorded @@ -160,6 +141,11 @@ statistics themselves, and the DataFrame shape. ## What is not proved, and why +- **Statistical validity and successful reporting.** Label exchangeability here + characterises only the block-constancy filter. It does not prove model + identifiability, assumptions for inference, or that vegan returns a p-value. + Julia also withholds missing p-values independently of this filter. + - **Geometric means, `exp` and `log`.** A geometric mean of rationals is in general irrational, so `_geomean` cannot be computed over `ℚ`. Wherever Julia divides by a geometric mean, the theorems divide by an arbitrary @@ -203,17 +189,3 @@ statistics themselves, and the DataFrame shape. A concrete counterexample is `p = [0.01, 0.011]`, `n = 2`. Without the minimum the scaled values are `0.02` and `0.011`, which are not monotone. - -3. **The PERMANOVA proofs refuse a wrong model.** Each of these, planted in - a scratch copy of `Permanova.agda`, made Agda reject it: - - `variesWithinAnyBlock` never stores a block's first value; - - it returns `true` on a match instead of a mismatch; - - it never returns `true`; - - the p-value drops the `+ 1` from `k + 1`; - - the floor is `1/(B + 2)`; - - any permutation counts as within-block; - - the fixture's `group` differs inside one individual (then - `fixture-group-withheld` fails); - - the fixture's `run` is constant inside every individual (then - `fixture-run-tested` fails); - - the 1% claim is made at 2% (`1/50 > 1/64`). diff --git a/scripts/check-occupancy-grades.jl b/scripts/check-occupancy-grades.jl deleted file mode 100755 index ce160d09..00000000 --- a/scripts/check-occupancy-grades.jl +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Gate for the graded reading of the recorded run (fork issue #105): rebuild each -# order's timeline from its sample window (docs/occupancy/schedules.yml), grade it -# with the module the probe itself uses (src/core/memory_probe.jl — included -# rather than the whole package, so this needs OrderedCollections and YAML only), -# and compare the algebra's predicted peak with the measured one. -# -# The recorded run is fixed evidence, so the comparison is an equality: this is -# the reading docs/occupancy-2026-10.adoc publishes, and a change that moves it -# must be a deliberate one. Two further claims hold of every order and are -# checked: the prediction is never above the cost bound (occupancy≤cost) and the -# slack between them is never negative (what the cuts reclaim). -# -# Run from the repository root: julia --project=. scripts/check-occupancy-grades.jl -include(joinpath(@__DIR__, "..", "src", "core", "memory_probe.jl")) -using .MemoryProbe -using YAML - -const ROOT = normpath(joinpath(@__DIR__, "..")) -const RESULTS = joinpath(ROOT, "docs", "occupancy", "results-2026-10-14.yml") -const SCHEDULES = joinpath(ROOT, "docs", "occupancy", "schedules.yml") - -orders = YAML.load_file(RESULTS)["orders"] -schedules = YAML.load_file(SCHEDULES) - -failures = 0 -function fail(message) - global failures += 1 - println("FAIL: ", message) -end - -println(rpad("order", 16), rpad("predicted", 11), rpad("measured", 11), - rpad("cost", 8), "reclaimed") -# An order's timeline, from the peaks the probe recorded and the window each task -# ran in. Nothing else is taken from the schedule: the grades are the -# measurement's. -function timeline(rec, windows) - tasks = Any[Dict{String,Any}("label" => t["label"], - "peak_rss_kib" => t["peak_rss_kib"], - "first_sample" => windows[i][1], - "last_sample" => windows[i][2]) - for (i, t) in enumerate(rec["tasks"])] - Dict{String,Any}("child_peak_rss_kib" => rec["child_peak_rss_kib"], "tasks" => tasks) -end - -for name in sort(collect(keys(orders))) - rec = orders[name] - occ = timeline(rec, schedules[name]) - predicted = MemoryProbe.predicted_peak_rss_kib(occ) - cost = MemoryProbe.cost_bound_rss_kib(occ) - reclaimed = MemoryProbe.reclaimed_kib(occ) - measured = rec["child_peak_rss_kib"] - println(rpad(name, 16), rpad(string(predicted), 11), rpad(string(measured), 11), - rpad(string(cost), 8), string(reclaimed)) - - predicted == measured || - fail("$name: predicted $predicted KiB != measured $measured KiB") - predicted <= cost || - fail("$name: predicted $predicted KiB above the cost bound $cost KiB (occupancy≤cost)") - reclaimed >= 0 || - fail("$name: reclaimed $reclaimed KiB is negative") - d = MemoryProbe.disagreement(occ) - d === nothing || fail("$name: $d") -end - -if failures > 0 - println("check-occupancy-grades: $failures failure(s)") - exit(1) -end -println("check-occupancy-grades: OK — the prediction reproduces the recorded measurement") diff --git a/scripts/check-occupancy.sh b/scripts/check-occupancy.sh deleted file mode 100755 index c43f28e4..00000000 --- a/scripts/check-occupancy.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Gate for the occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): -# 1. each Session IR text checks, and occupancy-types' stepper counts its bytes; -# 2. TypedPeak.agda and GradedStages.agda type-check against the pinned -# siblings, and each module in docs/occupancy/reject is refused at its own -# declaration; -# 3. the counted bytes equal the typed peak, order by order. -# Needs julia and agda (2.6.4.x) on PATH. Exit 0 only if all three hold. -set -euo pipefail -cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." -export LC_ALL="${LC_ALL:-C.UTF-8}" - -bash scripts/fetch-siblings.sh -deps=_build/dependencies -sir="$deps/occupancy-types/src/julia" -agda --version | head -1 -flags=(--safe --without-K --no-libraries --ignore-interfaces --double-check - -i docs/occupancy -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") -logs=_build/check-logs -mkdir -p "$logs" -failures=0 - -# GitHub renders an `::error` line as a check annotation, and annotations -- unlike -# job logs -- can be read back through the API. A failure that only happens in CI -# must be readable from a machine that cannot download the log, so every failure -# below says what it was and repeats the prover's own last lines: % escapes to -# %25 and newlines to %0A, as the workflow-command syntax requires. -annotate() { # title file log - [ -n "${GITHUB_ACTIONS:-}" ] || return 0 - printf '::error file=%s,title=%s::%s\n' "$2" "$1" \ - "$(awk 'NR > 1 { printf "%%0A" } { gsub(/%/, "%25"); printf "%s", $0 }' "$3" | - tail -c 6000)" -} - -# One Agda module that must type-check, with its own log. -check_agda() { # file label - if ! agda "${flags[@]}" "$1" >"$logs/$2.log" 2>&1; then - cat "$logs/$2.log" - annotate "Agda refused $2" "$1" "$logs/$2.log" - echo "ERROR: $1 does not type-check" >&2 - exit 1 - fi - echo "PASS: $2" -} - -# Order -> Session IR file and TypedPeak lemma. -orders=(asv-then-otu otu-then-asv concurrent) - -echo "== Session IR (occupancy-types v0 stepper)" -declare -A counted -for o in "${orders[@]}"; do - file="docs/occupancy/vsearch-$o.sir" - julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" check "$file" - out="$(julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" run "$file" --trace)" - echo "$out" - counted[$o]="$(sed -n 's/.*peak_live=\([0-9]*\).*/\1/p' <<<"$out" | head -1)" -done - -echo "== Typed peak (Agda, HWM Timeline)" -check_agda docs/occupancy/TypedPeak.agda TypedPeak.agda - -# The graded reading (fork issue #105): occupancy-types' Reclaim theorems at the -# measured grades, from the same pinned siblings. The recorded run's grades -# against its measurement are checked by scripts/check-occupancy-grades.jl, which -# needs the project's Julia packages and so runs in the disagreement job. -check_agda docs/occupancy/GradedStages.agda GradedStages.agda -for file in docs/occupancy/reject/*.agda; do - name="$(basename -- "$file" .agda)" - log="$logs/$name.log" - if agda "${flags[@]}" -i docs/occupancy/reject "$file" >"$log" 2>&1; then - cat "$log" - annotate "Invalid claim accepted: $name" "$file" "$log" - echo "ERROR: invalid claim accepted: $name" >&2; failures=$((failures + 1)); continue - fi - if ! grep -Eq "/$name\.agda:[0-9]+,[0-9]+" "$log" || ! grep -q 'when checking that' "$log" || - grep -Eq 'Not in scope|Failed to find|Unsolved|Parse error' "$log"; then - cat "$log" - annotate "$name failed for the wrong reason" "$file" "$log" - echo "ERROR: $name failed for the wrong reason" >&2; failures=$((failures + 1)); continue - fi - echo "PASS: rejected $name ($(grep -m1 -E '[0-9]+ != [0-9]+' "$log"))" -done - -echo "== Counted bytes against typed peak (KiB)" -printf '%-14s %10s %10s\n' order counted typed -for o in "${orders[@]}"; do - typed="$(grep -A1 -E "^$o :" docs/occupancy/TypedPeak.agda | sed -n 's/.*≡ \([0-9]*\)$/\1/p' | head -1)" - printf '%-14s %10s %10s\n' "$o" "${counted[$o]}" "$typed" - if [ -z "${counted[$o]}" ] || [ -z "$typed" ] || [ "${counted[$o]}" != "$typed" ]; then - echo "ERROR: $o: counted ${counted[$o]:-?} != typed ${typed:-?}" >&2; failures=$((failures + 1)) - echo "counted ${counted[$o]:-?} KiB != typed ${typed:-?} KiB" >"$logs/counted-$o.log" - annotate "Counted bytes != typed peak ($o)" docs/occupancy/TypedPeak.agda "$logs/counted-$o.log" - fi -done - -if (( failures > 0 )); then - echo "check-occupancy: $failures failure(s)" >&2 - exit 1 -fi -echo "check-occupancy: OK" diff --git a/scripts/fetch-siblings.sh b/scripts/fetch-siblings.sh deleted file mode 100755 index 31a98bc1..00000000 --- a/scripts/fetch-siblings.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Fetch the proof siblings the occupancy reading (docs/occupancy-2026-10.adoc) -# consumes, each at its pinned commit, into _build/dependencies. Their -# definitions are used, never copied. Re-pin one sibling per commit. -set -euo pipefail -cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." - -# occupancy-types: the HWM Timeline (Agda) and the SessionIR stepper (Julia). -OCCUPANCY_TYPES_COMMIT="${OCCUPANCY_TYPES_COMMIT:-af957280075699c659de0abd9a3b0defade77309}" -# resource-algebra: the HWM grade algebra occupancy-types builds on. -RESOURCE_ALGEBRA_COMMIT="${RESOURCE_ALGEBRA_COMMIT:-cf9235c532712a783a404c359a1055f40134794a}" - -fetch() { - local name="$1" commit="$2" dir="_build/dependencies/$1" - if [ ! -d "$dir/.git" ]; then - git init --quiet "$dir" - git -C "$dir" remote add origin "https://github.com/hyperpolymath/$name.git" - fi - git -C "$dir" fetch --quiet --depth 1 origin "$commit" - git -C "$dir" checkout --quiet --detach "$commit" - echo "$name at $(git -C "$dir" rev-parse --short HEAD)" -} - -mkdir -p _build/dependencies -fetch occupancy-types "$OCCUPANCY_TYPES_COMMIT" -fetch resource-algebra "$RESOURCE_ALGEBRA_COMMIT" diff --git a/scripts/kyaml-format.sh b/scripts/kyaml-format.sh deleted file mode 100755 index 145b157d..00000000 --- a/scripts/kyaml-format.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: MPL-2.0 -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# kyaml-format.sh — format arbitrary YAML as KYAML, or check that it already is -# (YAML-POLICY Y-3; KYAML adoption step 3, standards#1022). -# -# The rewriter is `yq -p yaml -o kyaml '.'`, verbatim — the one rewriter the -# step-2 comment-preservation proof (tools/yaml-comment-proof, standards#1021) -# covers. That proof does not generalise to any other rewriter, so this script -# deliberately adds none: no post-processing, no re-indenting, no sed. -# -# Usage: -# kyaml-format.sh [--check] FILE... -# -# (default) rewrite each FILE in place as KYAML when it is not already. -# --check rewrite nothing; exit 1 if any FILE is not already KYAML. -# -# Exit contract: -# 0 = every FILE is KYAML (after the rewrite, in format mode) -# 1 = --check: at least one FILE is not KYAML (each one is named) -# 2 = refused: no FILE given, a FILE is missing or unparseable, a FILE has no -# final newline, or the rewrite would change the parsed data. A refusal -# never rewrites that FILE and never counts as clean. -# -# Why the final-newline refusal: a `|` block scalar that ends at EOF with no -# line break is read differently by go-yaml (yq) and eemeli/yaml — measured on -# k9-contractile.yml, the trailing "\n" of its last run: body exists for one -# parser and not the other. Formatting such a file would silently pick one -# reading, so the script refuses it and names the fix instead. -set -uo pipefail - -MODE=format -if [ "${1:-}" = "--check" ]; then - MODE=check - shift -fi - -if [ "$#" -eq 0 ]; then - echo "kyaml-format: no files given (usage: kyaml-format.sh [--check] FILE...)" >&2 - exit 2 -fi - -command -v yq >/dev/null 2>&1 || { echo "kyaml-format: yq is not on PATH" >&2; exit 2; } - -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT - -# canonical_data — prints the file's parsed value as key-sorted JSON; -# fails when the file does not parse. -canonical_data() { - yq -p yaml -o json -I 0 'sort_keys(..)' "$1" -} - -# kyaml_of — writes the KYAML rendering of to ; -# fails when does not parse. -kyaml_of() { - yq -p yaml -o kyaml '.' "$1" > "$2" -} - -# refuse — reports a file this script will not judge. -refuse() { - echo "::error file=$1::kyaml-format: REFUSED $1 — $2" >&2 - refused=$((refused + 1)) -} - -refused=0 drift=0 clean=0 rewritten=0 -i=0 -for f in "$@"; do - i=$((i + 1)) - out="$TMP/$i.kyaml" - if [ ! -f "$f" ]; then - refuse "$f" "no such file"; continue - fi - if [ -s "$f" ] && [ -n "$(tail -c1 "$f")" ]; then - refuse "$f" "no final newline; add one first (a trailing block scalar is ambiguous without it)"; continue - fi - if ! kyaml_of "$f" "$out" 2>"$TMP/err"; then - refuse "$f" "does not parse: $(head -1 "$TMP/err")"; continue - fi - if cmp -s "$f" "$out"; then - clean=$((clean + 1)); continue - fi - # The rewrite is only acceptable when it preserves the data exactly. - if ! before=$(canonical_data "$f") || ! after=$(canonical_data "$out") || [ "$before" != "$after" ]; then - refuse "$f" "the KYAML rendering does not parse to the same data"; continue - fi - if [ "$MODE" = check ]; then - echo "::error file=$f::kyaml-format: $f is not KYAML — run scripts/kyaml-format.sh $f" >&2 - drift=$((drift + 1)) - else - cat "$out" > "$f" - echo "kyaml-format: rewrote $f" - rewritten=$((rewritten + 1)) - fi -done - -total=$# -if [ "$refused" -gt 0 ]; then - echo "kyaml-format: $refused of $total file(s) REFUSED — not judged, not clean" >&2 - exit 2 -fi -if [ "$drift" -gt 0 ]; then - echo "kyaml-format: $drift of $total file(s) are not KYAML" >&2 - exit 1 -fi -if [ "$MODE" = check ]; then - echo "kyaml-format: all $total file(s) are KYAML" -else - echo "kyaml-format: $total file(s) KYAML ($rewritten rewritten, $clean already clean)" -fi -exit 0 diff --git a/scripts/lint-occupancy-readings.sh b/scripts/lint-occupancy-readings.sh deleted file mode 100755 index 114779a0..00000000 --- a/scripts/lint-occupancy-readings.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Fork issue #105: a task's recorded `peak_rss_kib` is a reading, not an -# occupancy. What a stage occupied is determined only by what the next stage -# inherits from it — the drift — which is why the probe grades a whole window and -# why src/core/memory_probe.jl's helpers (`graded`, `predicted_peak_rss_kib`, -# `cost_bound_rss_kib`, `reclaimed_kib`, `task_grades`) exist. Code outside the -# probe that reaches for the key itself is comparing measured figures without -# that context; on the recorded run that reading is off by 9180 KiB for -# `asv_then_otu` and 9204 KiB for `otu_then_asv` (the reclaimed suffix). -# -# So: production code may name the key only in src/core/memory_probe.jl, where -# the record is built and graded. Tests may read it (they assert the recorded -# numbers directly, which is fine — they are evidence, not presentation). -# -# Run from the repository root: bash scripts/lint-occupancy-readings.sh -set -u -o pipefail - -if [[ ! -f src/core/memory_probe.jl ]]; then - echo "lint-occupancy-readings: run from the repository root" >&2 - exit 2 -fi - -failures=0 -while IFS= read -r hit; do - [[ -z "$hit" ]] && continue - echo "ERROR: ungraded occupancy reading: $hit" >&2 - failures=$((failures + 1)) -done < <(grep -rn --include='*.jl' '"peak_rss_kib"' src/ | - grep -v '^src/core/memory_probe\.jl:' || true) - -if (( failures > 0 )); then - echo "lint-occupancy-readings: $failures ungraded reading(s); use the grading helpers in" >&2 - echo "MemoryProbe (see docs/occupancy-2026-10.adoc, 'Reading the numbers') instead." >&2 - exit 1 -fi - -echo "lint-occupancy-readings: OK — every peak_rss_kib reading in src/ goes through the probe's grades" diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index b5f13a59..31db2a5c 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -33,9 +33,6 @@ include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/differential.jl") -# Canonical JSON: deterministic bytes for hashing and content-addressing. -include("canonical.jl") - # Web server include("server/server.jl") diff --git a/src/analysis/analysis.jl b/src/analysis/analysis.jl index 2b0105d2..f40bcc2e 100644 --- a/src/analysis/analysis.jl +++ b/src/analysis/analysis.jl @@ -950,25 +950,14 @@ _not_computed(status::Symbol, reason::AbstractString; """ was_computed(r::AlphaSignificance) -> Bool -Whether `r` has status `:computed`. False means no omnibus result is available, so +Whether `r` holds an actual test result. False means no test was performed, so neither `omnibus` nor `pairwise` may be presented as a finding. """ was_computed(r::AlphaSignificance) = r.status === :computed """ Run the omnibus (and optionally pairwise) alpha-diversity test, returning an -`AlphaSignificance` whose status distinguishes a computed omnibus result from -an unavailable or failed test. - -`values`, group `labels` and `sample_ids` describe the same samples. Unpaired -data use Kruskal-Wallis; `paired_samples` uses common sample IDs across all -groups, averaging repeated IDs within a group, with paired Wilcoxon for two -groups or Friedman for more. `pairwise` requests Wilcoxon comparisons with -Benjamini-Hochberg adjusted p-values. - -Unavailable R/vegan, no common IDs for pairing, or an omnibus R error converted -to a missing p-value return statuses `:r_unavailable`, `:no_paired_samples` or -`:test_failed`, respectively. A failed omnibus result retains any pairwise rows. +`AlphaSignificance` whose status says whether a test was actually performed. Waits for the R runtime like the other R analyses. While a pipeline run holds it, `RBusyError` reaches the server, which reports the chart as unavailable @@ -1174,22 +1163,6 @@ function _add_pairwise_annotations!(layout::Dict{String,Any}, layout[yaxis_layout_key] = axis end -""" - alpha_boxplot(groups; show_points=true, annotate_significance=false, - pairwise_brackets=false, paired_samples=false, - paired_lines=false, significance_test="kruskal_wallis") -> Dict - -Build a Plotly figure with richness, Shannon and Gini-Simpson boxplot panels. -Each group is `(label, sample_ids, richness, shannon, simpson)`, with metric -values aligned to its sample IDs. `paired_lines` joins matching IDs across -groups and disables point jitter. `paired_samples` selects paired tests when -an ID is shared by every group; otherwise tests are unpaired. - -With at least two group labels and `significance_test == "kruskal_wallis"`, -tests run even when neither annotation option is enabled. The options display -omnibus captions or pairwise brackets, including notices for unavailable or -failed tests. `RBusyError` propagates when the R runtime stays busy. -""" function alpha_boxplot(groups::AbstractVector{<:Tuple{String, Vector{String}, AbstractVector{<:Real}, Vector{Float64}, Vector{Float64}}}; show_points::Bool=true, annotate_significance::Bool=false, @@ -1469,13 +1442,11 @@ end PERMANOVA via vegan::adonis2 with up to 999 permutations. `metadata` is a DataFrame with one row per sample and covariate columns. -Columns named `sample` (case-insensitive) and constant columns are excluded. -Each remaining covariate is tested as its own term (`by = "terms"`), sequentially in +Each covariate is tested as its own term (`by = "terms"`), sequentially in column order, so each row is conditioned on the terms before it. The top-level `term`, `r2`, `f_statistic` and `p_value` are those of the first term. `blocks`, one label per sample (e.g. the individual a sample came from), -restricts permutations to within blocks. It is ignored when its length does -not match the sample count or when no label repeats, +restricts permutations to within blocks. It is ignored when no label repeats, since singleton blocks admit no permutation at all. A term that is constant within every block cannot be tested by within-block permutation (each sample never leaves its block, so the permutations never exchange that term's labels @@ -1485,12 +1456,6 @@ and the p-value does not test it), so its p-value is withheld and than 999 when the blocks admit fewer, and `min_p_value` the smallest p-value that number can produce. `transform` is applied before `vegdist`; see [`run_nmds`](@ref). - -Returns `nothing` when R/vegan is unavailable or no covariates remain. An -`adonis2` error becomes a named tuple containing `message`; on success, -`dispersion` contains PERMDISP results for the combined covariate groups or -an `error` message. `RBusyError`, transformation errors and R errors outside -these caught tests propagate. """ function run_permanova(mat::Matrix{Float64}, metadata::DataFrame; seed::Integer=123, transform::String="none", diff --git a/src/analysis/differential.jl b/src/analysis/differential.jl index 864bd2b7..d2c78de0 100644 --- a/src/analysis/differential.jl +++ b/src/analysis/differential.jl @@ -22,18 +22,13 @@ using DataFrames, RCall using ..RRuntime: with_r_lock using ..Analysis: _palette_hex, R_WAIT_SECONDS export DifferentialConfig, ScalingRefusal, MASSUnavailable, ZCompositionsUnavailable, - GlmGamPoiUnavailable, OFFSET_METHODS, METHODS, tss_factors, rle_factors, - size_factors, bh_adjust, validate_counts, replace_zeros, clr_transform, - fit_nb, fit_welch, differential_abundance, volcano_chart + OFFSET_METHODS, METHODS, tss_factors, rle_factors, size_factors, bh_adjust, + validate_counts, replace_zeros, clr_transform, fit_nb, fit_welch, + differential_abundance, volcano_chart # Methods accepted under `analysis.differential.method`. const METHODS = ("nb_glm", "clr_welch") -# The per-taxon dispersion `nb_glm` uses: maximum likelihood (MASS::glm.nb's -# own theta) or glmGamPoi's shrinkage estimator, which shares information -# across taxa. `"ml"` is the default and keeps #24's results unchanged. -const DISPERSIONS = ("ml", "glmGamPoi") - # Offset (size factor) methods accepted under `analysis.differential.offset`. const OFFSET_METHODS = ("tss", "rle") @@ -48,7 +43,7 @@ const DEFAULT_DELTA = 0.65 """ DifferentialConfig(offset="tss", min_prevalence=0.0; method="nb_glm", - replacement_delta=0.65, dispersion="ml") + replacement_delta=0.65) Settings for one differential abundance analysis, read from `analysis.differential` in the pipeline config. `method` is `"nb_glm"` or @@ -59,45 +54,26 @@ log-ratios, which are already free of library size. have at least one read to be tested. `replacement_delta`, in (0, 1), is the `frac` passed to `zCompositions::cmultRepl` by `clr_welch`: an imputed proportion above its taxon's smallest observed proportion is lowered to this fraction of it. -`dispersion` names the per-taxon dispersion `nb_glm` uses (`"ml"` or -`"glmGamPoi"`): `"ml"` takes the maximum-likelihood theta from `MASS::glm.nb`, -`"glmGamPoi"` takes the shrunken dispersion from `glmGamPoi::glm_gp` -- fitted -across all taxa in one call, which is where its shared information helps low-count -taxa -- and fits each taxon through `MASS::negative.binomial(theta = ...)` so the -Wald test and its checks are unchanged. It is `nb_glm` only. -Method and offset names are stripped and lowercased. Throws `ArgumentError` -for unsupported names or out-of-range fractions, including settings unused by -the selected method. """ struct DifferentialConfig method :: String offset :: String min_prevalence :: Float64 replacement_delta :: Float64 - dispersion :: String function DifferentialConfig(offset::AbstractString="tss", min_prevalence::Real=0.0; method::AbstractString="nb_glm", - replacement_delta::Real=DEFAULT_DELTA, - dispersion::AbstractString="ml") + replacement_delta::Real=DEFAULT_DELTA) m = lowercase(strip(String(method))) m in METHODS || throw(ArgumentError( "analysis.differential.method must be one of $(join(METHODS, ", ")), not '$method'")) o = lowercase(strip(String(offset))) o in OFFSET_METHODS || throw(ArgumentError( "analysis.differential.offset must be one of $(join(OFFSET_METHODS, ", ")), not '$offset'")) - d = strip(String(dispersion)) - di = findfirst(s -> lowercase(s) == lowercase(d), DISPERSIONS) - di === nothing && throw(ArgumentError( - "analysis.differential.dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) - d = DISPERSIONS[di] - (d == "ml" || m == "nb_glm") || throw(ArgumentError( - "analysis.differential.dispersion is a negative-binomial setting, but method is " * - "'$m'; only nb_glm has a dispersion to choose")) (isfinite(min_prevalence) && 0 <= min_prevalence <= 1) || throw(ArgumentError( "analysis.differential.min_prevalence must be a fraction in [0, 1], not $min_prevalence")) (isfinite(replacement_delta) && 0 < replacement_delta < 1) || throw(ArgumentError( "analysis.differential.replacement_delta must be in (0, 1), not $replacement_delta")) - new(m, o, Float64(min_prevalence), Float64(replacement_delta), d) + new(m, o, Float64(min_prevalence), Float64(replacement_delta)) end end @@ -139,18 +115,6 @@ Base.showerror(io::IO, ::ZCompositionsUnavailable) = print(io, "the R package zCompositions is not available, so zeros cannot be replaced for " * "clr_welch; restore the R library from renv.lock (Rscript -e 'renv::restore()')") -""" - GlmGamPoiUnavailable() - -Raised when the R package glmGamPoi, which provides the shrunken dispersion -`analysis.differential.dispersion: glmGamPoi` asks for, cannot be loaded. -""" -struct GlmGamPoiUnavailable <: Exception end - -Base.showerror(io::IO, ::GlmGamPoiUnavailable) = print(io, - "the R package glmGamPoi is not available, so the shrunken dispersion cannot be " * - "fitted; restore the R library from renv.lock (Rscript -e 'renv::restore()')") - """ _geomean(x) -> Float64 @@ -279,36 +243,7 @@ da_result <- data.frame(status = rep("ok", da_n), note = rep("", da_n), estimate = rep(NA_real_, da_n), se = rep(NA_real_, da_n), statistic = rep(NA_real_, da_n), pvalue = rep(NA_real_, da_n), theta = rep(NA_real_, da_n), stringsAsFactors = FALSE) -# glmGamPoi's shrunken dispersion, fitted across all taxa in one call -- its -# genes x samples convention, so the counts are transposed, and size_factors -# is the linear size factor (the per-taxon fits below take its log as the -# offset). A fit error or a dispersion that is not one finite positive value -# per taxon fails every taxon with the reason: with theta given, not -# estimated, there is nothing left to fall back to. -da_phi <- rep(NA_real_, da_n) -da_gp_failed <- "" -if (da_use_gp) { - da_gp <- tryCatch( - glmGamPoi::glm_gp(t(da_counts), design = model.matrix(~ da_g), - size_factors = exp(da_offset)), - error = function(e) e) - if (inherits(da_gp, "error")) { - da_gp_failed <- paste("glm_gp stopped with an error:", conditionMessage(da_gp)) - } else { - da_phi <- as.numeric(da_gp[["overdispersions"]]) - if (length(da_phi) != da_n || any(!is.finite(da_phi)) || any(da_phi <= 0)) { - da_gp_failed <- paste("glm_gp returned a dispersion that is not one finite", - "positive value per taxon") - da_phi <- rep(NA_real_, da_n) - } - } -} for (da_j in seq_len(da_n)) { - if (nzchar(da_gp_failed)) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- da_gp_failed - next - } da_y <- da_counts[, da_j] if (length(unique(da_y)) < 2L) { da_result[da_j, "status"] <- "failed" @@ -328,48 +263,18 @@ for (da_j in seq_len(da_n)) { next } da_warns <- character(0) - # "ml": theta is estimated per taxon by glm.nb's own alternating iterations. - # "glmGamPoi": theta is TAKEN from the shrunken dispersion fitted above - # (theta = 1 / phi) and the per-taxon fit goes through glm() with MASS's - # negative.binomial family, so the Wald test and every check below are the - # same machinery on a given dispersion instead of an estimated one. - if (da_use_gp) { - da_theta <- 1 / da_phi[da_j] - if (!is.finite(da_theta) || da_theta <= 0) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- "the shrunken dispersion for this taxon is not usable" - next - } - da_fit <- tryCatch( - withCallingHandlers( - glm(da_y ~ da_g + offset(da_offset), - family = MASS::negative.binomial(theta = da_theta), - control = glm.control(maxit = 100)), - warning = function(w) { - da_warns <<- c(da_warns, conditionMessage(w)) - invokeRestart("muffleWarning") - }), - error = function(e) e) - if (inherits(da_fit, "error")) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- paste("the negative-binomial fit stopped with an error:", - conditionMessage(da_fit)) - next - } - } else { - da_fit <- tryCatch( - withCallingHandlers( - MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), - warning = function(w) { - da_warns <<- c(da_warns, conditionMessage(w)) - invokeRestart("muffleWarning") - }), - error = function(e) e) - if (inherits(da_fit, "error")) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) - next - } + da_fit <- tryCatch( + withCallingHandlers( + MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), + warning = function(w) { + da_warns <<- c(da_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + }), + error = function(e) e) + if (inherits(da_fit, "error")) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) + next } da_warns <- unique(da_warns) if (!isTRUE(da_fit[["converged"]]) || !is.null(da_fit[["th.warn"]]) || @@ -391,23 +296,15 @@ for (da_j in seq_len(da_n)) { da_result[da_j, "note"] <- "the fit returned a non-finite estimate, standard error or p-value" next } - # "glmGamPoi" set da_theta before the fit; "ml" takes it from the fit. - if (!da_use_gp) da_theta <- da_fit[["theta"]] + da_theta <- da_fit[["theta"]] da_result[da_j, c("estimate", "se", "statistic", "pvalue")] <- unname(da_row[1:4]) da_result[da_j, "theta"] <- da_theta if (!is.finite(da_theta) || da_theta >= da_theta_upper) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- if (da_use_gp) - paste("the shrunken dispersion puts these counts at the Poisson end:", - "no overdispersion is left, so the fit is effectively Poisson") - else - "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" + da_result[da_j, "note"] <- "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" } else if (da_theta <= da_theta_lower) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- if (da_use_gp) - "the shrunken dispersion puts these counts at the extreme-variance end" - else - "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" + da_result[da_j, "note"] <- "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" } else if (length(da_warns) > 0L) { da_result[da_j, "note"] <- paste("R warned:", paste(da_warns, collapse = "; ")) } @@ -422,50 +319,36 @@ A finite number as `Float64`, or `nothing` for R's NA, NaN or an infinity. _num(x) = (ismissing(x) || isnothing(x) || !isfinite(x)) ? nothing : Float64(x) """ - fit_nb(counts, groups, offset; levels, dispersion="ml") -> DataFrame + fit_nb(counts, groups, offset; levels) -> DataFrame Fit `MASS::glm.nb(y ~ group + offset(offset))` to each column of the samples x taxa integer matrix `counts`. `levels` is `(reference, contrast)`, so the estimate is the natural-log fold change of `contrast` over `reference`. -`dispersion` names where each taxon's theta comes from: `"ml"` (the default) -takes `glm.nb`'s own maximum-likelihood estimate, `"glmGamPoi"` takes the -shrunken dispersion `glmGamPoi::glm_gp` fits across all taxa in one call and -runs each per-taxon fit through `MASS::negative.binomial(theta = 1/phi)`, so -the Wald test and its checks are unchanged. Returns one row per taxon with `status` (`ok`, `boundary` or `failed`), `note`, `estimate`, `se`, `statistic`, `pvalue` and `theta`; a failed taxon has `nothing` for every statistic. Throws `MASSUnavailable` when MASS cannot be -loaded, `GlmGamPoiUnavailable` when `dispersion` is `"glmGamPoi"` and glmGamPoi -cannot be loaded, and `RBusyError` when the R runtime stays busy. +loaded and `RBusyError` when the R runtime stays busy. """ function fit_nb(counts::AbstractMatrix{<:Integer}, groups::AbstractVector{<:AbstractString}, - offset::AbstractVector{<:Real}; levels::NTuple{2,String}, - dispersion::AbstractString="ml") - d = strip(String(dispersion)) - lowercase(d) in lowercase.(DISPERSIONS) || throw(ArgumentError( - "dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) - use_gp = lowercase(d) == "glmgampoi" + offset::AbstractVector{<:Real}; levels::NTuple{2,String}) size(counts, 1) == length(groups) == length(offset) || throw(ArgumentError( "counts have $(size(counts, 1)) samples, groups $(length(groups)), offset $(length(offset))")) raw = with_r_lock(; timeout=R_WAIT_SECONDS[]) do RCall.rcopy(RCall.reval("requireNamespace('MASS', quietly = TRUE)")) || throw(MASSUnavailable()) - use_gp && !RCall.rcopy(RCall.reval("requireNamespace('glmGamPoi', quietly = TRUE)")) && - throw(GlmGamPoiUnavailable()) RCall.globalEnv[:da_counts] = Matrix{Int}(counts) RCall.globalEnv[:da_group] = String.(groups) RCall.globalEnv[:da_levels] = collect(levels) RCall.globalEnv[:da_offset] = Float64.(offset) RCall.globalEnv[:da_theta_upper] = THETA_UPPER RCall.globalEnv[:da_theta_lower] = THETA_LOWER - RCall.globalEnv[:da_use_gp] = use_gp try RCall.reval(_FIT_R) DataFrame(RCall.rcopy(RCall.reval("da_result"))) finally RCall.reval("rm(list = intersect(ls(), c('da_counts', 'da_group', 'da_levels', " * - "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_use_gp', 'da_g', 'da_term', " * - "'da_n', 'da_result', 'da_phi', 'da_gp_failed', 'da_gp', 'da_j', 'da_y', " * - "'da_warns', 'da_fit', 'da_co', 'da_row', 'da_theta', 'da_absent'))); invisible(gc())") + "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_g', 'da_term', " * + "'da_n', 'da_result', 'da_j', 'da_y', 'da_warns', 'da_fit', 'da_co', " * + "'da_row', 'da_theta', 'da_absent'))); invisible(gc())") end end DataFrame(status = String.(raw.status), @@ -482,8 +365,7 @@ end Centred log-ratio of each row (sample) of the samples x taxa matrix `x`: `log(x_ij)` minus the mean of `log(x_i·)`, so every row sums to zero. Every -value must be finite and positive or an `ArgumentError` is thrown; replace -zeros first. +value must be positive; replace zeros first. """ function clr_transform(x::AbstractMatrix{<:Real}) all(v -> isfinite(v) && v > 0, x) || throw(ArgumentError( @@ -516,9 +398,8 @@ da_out <- tryCatch( Check that cmultRepl's output `out` is one valid composition per sample: the same shape as the input whose zeros `zero_mask` marks, and every row finite, -positive and summing to 1 within an absolute tolerance of 1e-8. Throws -`ErrorException` if the shapes differ, or `ArgumentError` naming the first -invalid sample and pointing at `min_prevalence`. Returns `out` unchanged. +positive and summing to 1. Throws `ArgumentError` naming the first sample that +fails, pointing at `min_prevalence`. """ function _check_compositions(out::AbstractMatrix{<:Real}, zero_mask::AbstractMatrix{Bool}, samples::AbstractVector{<:AbstractString}) @@ -546,9 +427,6 @@ data. Returns `(proportions, zeros_replaced, imputed_mass, warnings)`: `proportions` has the same shape as `counts`, each row summing to 1 with no zero left; `imputed_mass` is the share of each sample's total held by replaced values; `warnings` is whatever R warned. -`frac` controls the fraction of a taxon's smallest observed proportion used -to lower an excessive imputed proportion (see [`DifferentialConfig`](@ref)). -Without zeros, returns row-normalised counts, but still requires zCompositions. The caller chooses the taxa: everything passed in is replaced and kept, none is dropped. Throws `ArgumentError` when there are fewer than 2 taxa, when a taxon @@ -556,8 +434,7 @@ has reads in fewer than 2 samples (the GBM prior is then undefined), when a sample has no reads, or when cmultRepl refuses or returns a value that is not a positive proportion; the messages name the taxon or sample and point at `min_prevalence`. Throws `ZCompositionsUnavailable` when zCompositions cannot -be loaded and `RBusyError` when the R runtime stays busy. Mismatched sample or -taxon labels raise `ArgumentError`; a changed output shape raises `ErrorException`. +be loaded and `RBusyError` when the R runtime stays busy. """ function replace_zeros(counts::AbstractMatrix{<:Real}; frac::Real=DEFAULT_DELTA, @@ -663,10 +540,7 @@ the mean of `contrast` minus the mean of `reference`; the variances of the two groups are not assumed equal, since the groups often differ in size. Returns one row per taxon with `status` (`ok` or `failed`), `note`, `estimate`, `se`, `statistic` (Welch's t) and `pvalue`; a failed taxon has `nothing` for every -statistic. Errors from an individual R test become failed rows with a reason -in `note`. Throws `ArgumentError` if the group labels do not match the number -of samples and `RBusyError` when the R runtime stays busy; other R evaluation -or conversion errors propagate. +statistic. Throws `RBusyError` when the R runtime stays busy. """ function fit_welch(values::AbstractMatrix{<:Real}, groups::AbstractVector{<:AbstractString}; levels::NTuple{2,String}) @@ -727,23 +601,19 @@ every observed taxon first would let imputed values for rare taxa take over the samples of a sparse table. The estimate is the difference in mean CLR, contrast minus reference: a difference in centred-log-ratio units, not a fold change, relative to the geometric mean of the tested taxa. Replaced zeros are -not measurements; `diagnostics.zero_replacement` reports the median and maximum -share of sample mass held by replaced zeros. Each group needs at least 2 samples. +not measurements; `diagnostics.zero_replacement` reports their share of each +sample. Each group needs at least 2 samples. For both methods, taxa present in fewer than `config.min_prevalence` of the samples are reported as `filtered` and not fitted, and the Benjamini-Hochberg family is the fitted taxa that produced a p-value; failed and filtered taxa carry `padj = nothing`. The result's `effect` names the row key that holds the effect size and its label. -Rows are sorted by adjusted p-value, raw p-value and taxon name, with missing -p-values last. The overall status is `partial` if any taxon failed, otherwise `ok`. Throws `ArgumentError` for unusable input (including a zero replacement that cannot be made), `ScalingRefusal` when the offset cannot be computed, `MASSUnavailable` or `ZCompositionsUnavailable` when the R package a method -needs is missing, `RBusyError` when the R runtime stays busy, and an -`ErrorException` when no taxon could be fitted at all or zero replacement changes -the table's shape. Other R evaluation or conversion errors propagate. +needs is missing, and an `ErrorException` when no taxon could be fitted at all. """ function differential_abundance(counts::AbstractMatrix{<:Real}, samples::AbstractVector{<:AbstractString}, @@ -781,8 +651,7 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, factors = size_factors(counts, samples, config.offset) tested = findall(>=(config.min_prevalence), prevalence) fits = isempty(tested) ? nothing : - fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels, - dispersion=config.dispersion) + fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels) if !isnothing(fits) for (k, j) in enumerate(tested) f = fits[k, :] @@ -800,8 +669,7 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, size_factor_rows = [Dict("sample" => String(samples[i]), "group" => String(groups[i]), "factor" => factors[i]) for i in eachindex(samples)] config_echo = Dict{String,Any}("method" => config.method, "offset" => config.offset, - "min_prevalence" => config.min_prevalence, - "dispersion" => config.dispersion) + "min_prevalence" => config.min_prevalence) else (n_ref >= 2 && n_con >= 2) || throw(ArgumentError( "Welch's t-test needs at least 2 samples in each group to estimate its variance; " * @@ -882,9 +750,8 @@ end A Plotly volcano plot of a `differential_abundance` result: the effect named by `result["effect"]` (log2 fold change, or CLR difference) against -log10 of the raw p-value, coloured by whether the BH-adjusted p-value is below `alpha`. Taxa -without an adjusted p-value are omitted here; they remain in the results table -with their reason. Without `effect`, uses log2 fold change. Zero raw p-values -are plotted at the smallest positive normal `Float64` and identified in hover text. +without a p-value are omitted here; they remain in the results table with their +reason. """ function volcano_chart(result::AbstractDict; alpha::Real=0.05) ref = result["groups"]["reference"] diff --git a/src/canonical.jl b/src/canonical.jl deleted file mode 100644 index 04363037..00000000 --- a/src/canonical.jl +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# JCS (RFC 8785) canonical JSON: deterministic bytes for hashing, signing and -# content-addressing. The same value always yields the same byte string, so a -# chart spec or an analysis result can be hashed where it is produced and the -# hash compared across machines, Julia versions and refactorings that build -# the same object in a different order. -# -# What this implements, and what it deliberately does not: -# -# * Object keys are sorted by UTF-16 code unit, the rule that makes an -# astral-plane key sort BEFORE a key such as U+FF21, because its high -# surrogate (U+D800..U+DBFF) is a smaller code unit than U+FF21. -# * Float64 values serialize per ECMAScript's Number::toString, the rules -# RFC 8785 builds on: shortest round-trip digits, positional notation for -# 1e-6 <= |x| < 1e21, exponent notation outside it, a signed exponent -# ("1e+30"), and -0.0 collapsing to 0. -# * Integers outside I-JSON's exact range (-(2^53)+1 .. 2^53-1) are -# refused, as the estate's ijson-jcs crate refuses them, because a -# consumer cannot read them back as the same number. -# * NaN and Inf are refused for the same reason. -# -# The implementation is verified against the estate's ijson-jcs crate -# (Rust, RFC 7493 + RFC 8785): the fixtures in -# test/fixtures/jcs_oracle.jsonl were produced BY that crate, and -# test/unit/test_canonical.jl asserts this module reproduces them -# byte-for-byte. The crate is the oracle; this is the second -# implementation, and any disagreement between them is a bug in one of -# them. - -module Canonical - - -using SHA - -export jcs_bytes, jcs_sha256 - -# The I-JSON exact-integer range. -const SAFE_INT_MAX = 2^53 - 1 - -""" - jcs_bytes(x) -> Vector{UInt8} - -The RFC 8785 canonical form of `x`, as bytes. Accepts the JSON-shaped -values the server produces: `Dict` and `JSON3.Object` (key order is -irrelevant; it is sorted away), `AbstractVector` and `JSON3.Array` (order -is preserved, as JCS preserves array order), `AbstractString`, `Integer`, -`AbstractFloat`, `Bool` and `nothing`. Floats serialize per ECMAScript's -Number::toString; integers outside the exact I-JSON range, and NaN or Inf -floats, throw `ArgumentError`. -""" -function jcs_bytes(x) - io = IOBuffer() - _write!(io, x) - take!(io) -end - -""" - jcs_sha256(x) -> String - -The SHA-256 of `jcs_bytes(x)`, as lowercase hex -- the content address of -a JSON value: two producers that build the same value hash identically -whatever order they built it in. -""" -jcs_sha256(x) = bytes2hex(sha256(jcs_bytes(x))) - -# Object keys sort by UTF-16 code unit, which for astral-plane characters -# means their surrogate pair, a smaller first unit than any BMP code point -# at or above U+E000. transcoding the key to UInt16 gives exactly those -# units to compare. -_utf16_units(s::AbstractString) = transcode(UInt16, String(s)) - -_write!(io, ::Nothing) = print(io, "null") -_write!(io, b::Bool) = print(io, b ? "true" : "false") - -function _write!(io, n::Integer) - # I-JSON: only |n| <= 2^53-1 survives a JSON round trip exactly. - -SAFE_INT_MAX <= n <= SAFE_INT_MAX || throw(ArgumentError( - "canonical JSON refuses the integer $n: it is outside the exact " * - "IEEE-754 binary64 range I-JSON allows")) - print(io, n) -end - -function _write!(io, x::AbstractFloat) - _write_float!(io, Float64(x)) -end - -function _write!(io, s::AbstractString) - print(io, '"') - for c in s - _write_char!(io, c) - end - print(io, '"') -end - -# ECMAScript's JSON.stringify escapes exactly these: the two mandatory -# characters, the five with short forms, and every other control -# character as \u00xx with lowercase hex. -function _write_char!(io, c::Char) - if c == '"' - print(io, "\\\"") - elseif c == '\\' - print(io, "\\\\") - elseif c == '\b' - print(io, "\\b") - elseif c == '\t' - print(io, "\\t") - elseif c == '\n' - print(io, "\\n") - elseif c == '\f' - print(io, "\\f") - elseif c == '\r' - print(io, "\\r") - elseif c < ' ' - print(io, "\\u", string(UInt32(c); base=16, pad=4)) - else - print(io, c) - end -end - -# AbstractDict (Dict and JSON3.Object alike): sorted keys, then each pair. -function _write!(io, d::AbstractDict) - print(io, '{') - first = true - for k in sort!(collect(String.(keys(d))); by = _utf16_units) - first || print(io, ',') - first = false - _write!(io, string(k)) - print(io, ':') - _write!(io, d[k]) - end - print(io, '}') -end - -# AbstractVector (Vector and JSON3.Array alike): order preserved. -function _write!(io, a::AbstractVector) - print(io, '[') - for (i, v) in enumerate(a) - i > 1 && print(io, ',') - _write!(io, v) - end - print(io, ']') -end - -# RFC 8785 §3.2.2.3: ECMAScript's Number::toString. Julia's own repr gives -# the shortest round-trip digits (Ryu, the same family ES specifies), so -# the work is reformatting those digits under ES's notation rules. -function _write_float!(io, x::Float64) - isnan(x) && throw(ArgumentError( - "canonical JSON refuses NaN: no consumer reads it back as the same value")) - isinf(x) && throw(ArgumentError( - "canonical JSON refuses Inf: no consumer reads it back as the same value")) - if x == 0 - print(io, "0") # +0.0 and -0.0 both collapse to 0, as ECMAScript does. - return - end - if signbit(x) - print(io, '-') - x = -x - end - digits, n = _shortest_digits(x) - k = length(digits) - if n > 21 || n <= -6 - # Exponential: one digit, then the fraction, then e with an - # explicit sign (ES prints "1e+30", not "1e30"). - print(io, digits[1]) - k > 1 && (print(io, '.'); print(io, digits[2:end])) - e = n - 1 - print(io, 'e', e < 0 ? '-' : '+', string(abs(e))) - elseif n >= k - # An integer too big for the fraction: digits followed by zeros. - print(io, digits, "0"^(n - k)) - elseif n > 0 - # Digits with the point inside them. - print(io, digits[1:n], '.', digits[n+1:end]) - else - # Below one, positional as far as ES goes: 0.000001, not 1e-6. - print(io, "0.", "0"^(-n), digits) - end -end - -# The shortest round-trip decimal form of x as (digits, n) with -# x = 0.digits * 10^n: repr yields either "123", "4.5" or "1.5e-10". -function _shortest_digits(x::Float64) - s = repr(x) - mantissa, exponent = occursin('e', s) ? (split(s, 'e') .|> String) : (s, "") - e = isempty(exponent) ? 0 : parse(Int, exponent) - if occursin('.', mantissa) - intpart, fracpart = split(mantissa, '.') .|> String - else - intpart, fracpart = mantissa, "" - end - digits = string(intpart, fracpart) - n = length(intpart) + e - # Trailing zeros are notation, not value: 1.0e30 is digits "1", n 31. - while length(digits) > 1 && endswith(digits, '0') - digits = digits[1:end-1] - end - digits, n -end - -end # module diff --git a/src/core/memory_probe.jl b/src/core/memory_probe.jl index a4ad4304..6406c37a 100644 --- a/src/core/memory_probe.jl +++ b/src/core/memory_probe.jl @@ -50,16 +50,8 @@ mutable struct TaskTrace samples :: Int # samples that found at least one live process peak_kib :: Int hwm_kib :: Int - # The sample window the task's children were live in: the index of the first - # sample taken after the first of them started, and the last taken before - # the last was waited for. 0 means never. Two tasks overlap when the later - # one's `first_sample` falls at or before the earlier one's `last_sample`, - # which is the interleaving the grade algebra sequences over. - first_sample :: Int - last_sample :: Int end -TaskTrace(label::AbstractString) = - TaskTrace(String(label), Int[], 0, 0, 0, 0, 0, 0) +TaskTrace(label::AbstractString) = TaskTrace(String(label), Int[], 0, 0, 0, 0) mutable struct Probe interval :: Float64 @@ -277,10 +269,7 @@ function watch!(proc::Base.Process) end lock(p.lock) do t.processes += 1 - t.processes == 1 && (t.first_sample = p.samples + 1) - if pid !== nothing - push!(t.live, pid) - end + pid === nothing || push!(t.live, pid) end pid === nothing || _sample!(p) pid @@ -312,235 +301,16 @@ function unwatch!(pid::Int) p === nothing && return nothing lock(p.lock) do filter!(!=(pid), t.live) - isempty(t.live) && (t.last_sample = p.samples) - end - nothing -end - -## Grading a stage against the high-water-mark algebra -# `docs/occupancy` reads a stage as a timeline of grades and checks the peak the -# algebra predicts against the one the probe measured. The grade is the pinned -# HWM instance's (`occupancy-types`' sibling `resource-algebra`, -# `ResourceAlgebra.Instances.HWM`; both are fetched by `scripts/fetch-siblings.sh`): -# -# hwm peak drift with drift ≤ peak -# -# * `peak` — the high-water mark this interaction reached, in KiB; -# * `drift` — the change in the live peak sum across the interaction's -# boundary, so a boundary that also releases an earlier task is negative; -# * `release` = peak − drift — everything freed between the interaction's start -# and the next boundary, which can exceed the interaction's own peak. -# -# A task's drift subtracts earlier tasks that exited before the next start, so it -# can be negative — the signed-drift extension `Instances/HWM`'s header records -# as H-2. The folded drift is the sum of peaks still live at the stage's end. -# -# A task's drift subtracts earlier tasks that exited before the next start, so -# it can be negative. The folded drift is the sum of peaks still live there. -# -# Sequencing starts the next interaction from the previous one's drift: -# -# hwm p₁ n₁ ⊗ hwm p₂ n₂ = hwm (max p₁ (n₁ + p₂)) (n₁ + n₂) -# -# and `OccupancyTypes.Reclaim` proves of that definition -# -# peak (a ⊗ b) = max (peak a) (drift a + peak b) (reclaim-law) -# peak (a ⊗ b) ≤ peak a + peak b (occupancy≤cost) -# drift a = peak a ⇒ peak (a ⊗ c) = peak a + peak c (no-reclaim) -# drift a < peak a, 0 < peak c ⇒ peak (a ⊗ c) < peak a + peak c (reclaimed) -# -# The last two are the two shapes a stage actually has. Run together, the first -# task is still resident when the second starts (drift = peak) and the stage peak -# is the sum; run one after the other it has exited (drift = 0) and the stage peak -# is the larger. The samples already tell the two apart, so the drift is read from -# the task windows and never assumed. -# -# The prediction is an upper bound on the measurement: at a handover the task -# still live holds at most its peak, so the drift the algebra adds is at least -# what a sample could have seen. `disagreement` therefore reports only what that -# bound forbids — a measured stage peak above the prediction — which is the "one -# of them is a bug" the occupancy reading asks about. - -const GRADE_ALGEBRA = "ResourceAlgebra.Instances.HWM" - -"A grade of the pinned high-water-mark algebra: a peak and a net memory change." -struct Grade - peak :: Int - drift :: Int - function Grade(peak::Integer, drift::Integer) - drift <= peak || throw(ArgumentError( - "a grade holds at most what it reached: drift $drift > peak $peak")) - new(Int(peak), Int(drift)) - end -end - -"`release(grade)` — `peak − drift`: everything freed by the interaction's boundary." -release(g::Grade) = g.peak - g.drift - -"`seq(a, b)` — sequence `b` after `a`: the suffix starts from `a`'s drift." -seq(a::Grade, b::Grade) = Grade(max(a.peak, a.drift + b.peak), a.drift + b.drift) - -"`alt(a, b)` — the larger grade, componentwise (what the timeline never needs, kept for the pair of laws)." -alt(a::Grade, b::Grade) = Grade(max(a.peak, b.peak), max(a.drift, b.drift)) - -"Nothing allocated and nothing held: the algebra's unit." -const NO_GRADE = Grade(0, 0) - -# The trace's sample window, or `nothing` for a record that carries none (one -# written before the windows were sampled). -function _window(rec::AbstractDict) - first_seen = get(rec, "first_sample", nothing) - last_seen = get(rec, "last_sample", nothing) - (first_seen isa Integer && last_seen isa Integer) ? (Int(first_seen), Int(last_seen)) : nothing -end - -_is_windowed(rec::AbstractDict) = _window(rec) !== nothing - -# Whether `prev` was still live when `next` started, so the handover hands its -# peak on rather than reclaiming it. A started `prev` never unwatched (window -# end 0) was still live; a task that never started (window start 0) never overlapped. -_overlaps(prev::Tuple{Int,Int}, next::Tuple{Int,Int}) = - prev[1] != 0 && next[1] != 0 && (prev[2] == 0 || next[1] <= prev[2]) - -# Grades and labels in sampled order, or `nothing` when the record carries no -# windows: a comparison is then skipped rather than made on an assumed order. -function _grades_and_labels(occ::AbstractDict) - recs = get(occ, "tasks", nothing) - recs isa AbstractVector || return Grade[], String[] - all(_is_windowed, recs) || return nothing - # Sampled order, ties broken by record order so the result never depends on - # the sort's stability. - order = sort(collect(eachindex(recs)); by = i -> (_window(recs[i])[1], i)) - ordered = collect(recs)[order] - windows = [_window(r) for r in ordered] - peaks = [Int(get(r, "peak_rss_kib", 0)) for r in ordered] - grades = Grade[] - labels = String[] - live = Int[] - held = 0 - for (i, rec) in enumerate(ordered) - push!(live, i) - if i == length(ordered) - # The stage waits for its children: a task still resident at the end - # keeps its peak in the folded drift, an exited one holds nothing. - filter!(j -> windows[j][1] != 0 && windows[j][2] == 0, live) - else - filter!(j -> _overlaps(windows[j], windows[i + 1]), live) - end - remaining = sum(j -> peaks[j], live; init = 0) - # seq adds drifts, so reclaim expired predecessors as well as this task. - push!(grades, Grade(peaks[i], remaining - held)) - held = remaining - push!(labels, String(get(rec, "label", "?"))) end - grades, labels -end - -""" - task_grades(occupancy) -> Union{Vector{Grade},Nothing} - -One grade per task of a stage's `occupancy` record, in sampled order. Its drift -is the change in the live peak sum by the next task's start, including releases -of earlier tasks; the final boundary retains only unclosed windows. Folding -with `seq` carries exactly the peaks still live at each boundary. `nothing` when -the record carries no sample windows. -""" -function task_grades(occ::AbstractDict) - graded = _grades_and_labels(occ) - graded === nothing ? nothing : graded[1] -end - -""" - stage_grade(occupancy) -> Union{Grade,Nothing} - -The stage's grade: its tasks' grades folded with `seq`, the algebra's sequencing. -`nothing` when the record carries no sample windows. -""" -function stage_grade(occ::AbstractDict) - grades = task_grades(occ) - grades === nothing && return nothing - isempty(grades) ? NO_GRADE : foldl(seq, grades) -end - -"`predicted_peak_rss_kib(occupancy)` — the peak the algebra predicts, or `nothing`." -function predicted_peak_rss_kib(occ::AbstractDict) - g = stage_grade(occ) - g === nothing ? nothing : g.peak -end - -"`cost_bound_rss_kib(occupancy)` — the peaks added: the peak if no cut reclaimed anything (occupancy≤cost's right-hand side)." -function cost_bound_rss_kib(occ::AbstractDict) - grades = task_grades(occ) - grades === nothing ? nothing : sum(g -> g.peak, grades; init = 0) -end - -"`reclaimed_kib(occupancy)` — the slack between the cost bound and the predicted peak: what the cuts reclaim." -function reclaimed_kib(occ::AbstractDict) - cost = cost_bound_rss_kib(occ) - predicted = predicted_peak_rss_kib(occ) - (cost === nothing || predicted === nothing) ? nothing : cost - predicted -end - -""" - disagreement(occupancy) -> Union{String,Nothing} - -Why the measured stage peak contradicts the grade algebra, or `nothing` when it -does not. The prediction is an upper bound, so the one contradiction is a -measured peak above it: a sample then saw more resident memory than the timed -timeline of tasks can hold, and one of the two is wrong. A record with no -windows, or none the probe graded, gives `nothing`: not measured is not a -disagreement. -""" -function disagreement(occ::AbstractDict) - predicted = predicted_peak_rss_kib(occ) - measured = get(occ, "child_peak_rss_kib", nothing) - (predicted === nothing || !(measured isa Integer)) && return nothing - measured > predicted && return "measured $(measured) KiB above the predicted $(predicted) KiB" nothing end -""" - graded(occupancy) -> Union{OrderedDict,Nothing} - -The grades of a stage's `occupancy` record, ready to sit beside it in the -attestation: per task `peak_kib`/`drift_kib`/`release_kib`, the stage's -`predicted_peak_kib`, the `cost_bound_kib` (its peaks added) and the -`reclaimed_kib` slack between the two, the `measured_peak_kib` it is checked -against, and the `disagreement` between them or `nothing`. `nothing` when the -record carries no sample windows. -""" -function graded(occ::AbstractDict) - graded_pair = _grades_and_labels(occ) - graded_pair === nothing && return nothing - grades, labels = graded_pair - stage = isempty(grades) ? NO_GRADE : foldl(seq, grades) - cost = sum(g -> g.peak, grades; init = 0) - OrderedDict{String,Any}( - "algebra" => GRADE_ALGEBRA, - "tasks" => Any[OrderedDict{String,Any}( - "label" => labels[i], - "peak_kib" => grades[i].peak, - "drift_kib" => grades[i].drift, - "release_kib" => release(grades[i])) - for i in eachindex(grades)], - "predicted_peak_kib" => stage.peak, - "drift_kib" => stage.drift, - "release_kib" => release(stage), - "cost_bound_kib" => cost, - "reclaimed_kib" => cost - stage.peak, - "measured_peak_kib" => get(occ, "child_peak_rss_kib", nothing), - "disagreement" => disagreement(occ), - ) -end - _task_record(t::TaskTrace) = OrderedDict{String,Any}( "label" => t.label, "processes" => t.processes, "samples" => t.samples, "peak_rss_kib" => t.peak_kib, "hwm_kib" => t.hwm_kib, - "first_sample" => t.first_sample, - "last_sample" => t.last_sample, ) """ @@ -567,11 +337,6 @@ function finish!(p::Probe) "tasks" => Any[_task_record(t) for t in traces], ) p.error === nothing || (p.record["error"] = p.error) - # The grades the sample windows imply, beside the samples they came from. A - # record whose tasks carry no windows cannot be graded, and is left as it is - # rather than graded on an assumed order. - g = graded(p.record) - g === nothing || (p.record["grades"] = g) p.record end diff --git a/src/pipeline/dada2/dada2_functions.r b/src/pipeline/dada2/dada2_functions.r index 5d61fa6c..8fd9fd40 100644 --- a/src/pipeline/dada2/dada2_functions.r +++ b/src/pipeline/dada2/dada2_functions.r @@ -213,19 +213,41 @@ write_fasta <- function(seq_table, tables_dir, prefix) { # taxonomic rank) # Returns taxa_df (SeqName + Sequence + taxonomy), used by write_combined_table(). # -# dplyr::left_join(x, y, by) keeps x's rows in x's order with x's columns first -# and then y's (the key once). merge() alone sorts the rows and reorders the -# columns, so both are restored explicitly: a temporary position column restores -# x's row order, and the column order is rebuilt from x's columns plus y's non-key -# ones. The CI oracle checks the written CSVs byte-for-byte against the previous output. +# Preserve left-row order, right-row order within matches, and the original +# column order when assembling the taxonomy and count tables. base_left_join <- function(x, y, by) { - x_cols <- colnames(x) - position <- ".left_join_position" - while (position %in% c(colnames(x), colnames(y))) position <- paste0(position, "_") - x[[position]] <- seq_len(nrow(x)) - joined <- merge(x, y, by = by, all.x = TRUE, sort = FALSE) - joined <- joined[order(joined[[position]]), - c(x_cols, setdiff(colnames(y), by)), drop = FALSE] + x_cols <- names(x) + y_cols <- names(y) + stopifnot(!anyDuplicated(x_cols), !anyDuplicated(y_cols), + all(by %in% x_cols), all(by %in% y_cols)) + suffix_names <- function(values, reserved, suffix) { + out <- c(reserved, values) + while (anyDuplicated(out)) { + duplicate <- which(duplicated(out)) + out[duplicate] <- paste0(out[duplicate], suffix) + } + tail(out, length(values)) + } + # Match left_join's repeated suffixing, including names such as Species.x. + x_out <- x_cols + x_aux <- !x_cols %in% by + y_aux <- !y_cols %in% by + x_out[x_aux] <- suffix_names(x_cols[x_aux], c(by, y_cols[y_aux]), ".x") + y_out <- suffix_names(y_cols, x_cols, ".y")[y_aux] + + # Merge only unique internal names, so character selection can never confuse + # a taxonomy column with a sample whose name resembles a generated suffix. + x_ids <- paste0(".x", seq_along(x_cols)) + y_ids <- paste0(".y", seq_along(y_cols)) + names(x) <- x_ids + names(y) <- y_ids + x$.left_position <- seq_len(nrow(x)) + y$.right_position <- seq_len(nrow(y)) + joined <- merge(x, y, by.x = x_ids[match(by, x_cols)], + by.y = y_ids[match(by, y_cols)], all.x = TRUE, sort = FALSE) + joined <- joined[order(joined$.left_position, joined$.right_position), + c(x_ids, y_ids[y_aux]), drop = FALSE] + names(joined) <- c(x_out, y_out) rownames(joined) <- NULL joined } diff --git a/src/server/routes/analysis.jl b/src/server/routes/analysis.jl index 6154f5b7..0dd506a3 100644 --- a/src/server/routes/analysis.jl +++ b/src/server/routes/analysis.jl @@ -1203,8 +1203,7 @@ function _differential_config(study::String) DifferentialConfig(String(val("analysis.differential.offset", "tss")), Float64(val("analysis.differential.min_prevalence", 0.0)); method=String(val("analysis.differential.method", "nb_glm")), - replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65)), - dispersion=String(val("analysis.differential.dispersion", "ml"))) + replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65))) end """ @@ -1213,11 +1212,7 @@ end Run a differential abundance test between exactly two conditions (runs or pooled-run sub-groups) and return the per-taxon table with a volcano plot. The first condition is the reference, so a positive estimate means more abundant in -the second: a natural-log fold change for `nb_glm`, or a mean CLR difference for -`clr_welch`. `ArgumentError`s from config construction or the test return 400; -scaling refusals and analysis `ErrorException`s return 422, and missing MASS or -zCompositions returns 503. Missing studies or results return 404. -`RBusyError` and other uncaught errors propagate to the caller. +the second. Every refusal is an explicit JSON error. """ function _differential(study::String, body) study in _study_names() || return json_error(404, "study_not_found", diff --git a/src/server/routes/config.jl b/src/server/routes/config.jl index 20b55a60..2f927ba1 100644 --- a/src/server/routes/config.jl +++ b/src/server/routes/config.jl @@ -125,10 +125,8 @@ const _NULLABLE_NUMBERS = ("phylogeny.placement.place.heuristic",) """ _config_value_error(key, value) -> Union{String,Nothing} -Return a rejection message for `value` under the dotted pipeline `key`, or -`nothing`. Enums, command arguments, remote settings and file-name parts have -specific checks; otherwise the type follows the factory default, with no type -restriction for a default of ~. Does not write config or check key recognition. +Why a value cannot be saved under this key, or nothing. The type follows the +factory default; a key whose default is ~ accepts a string, number or boolean. """ function _config_value_error(key::String, value) v = value isa JSON3.Array ? collect(value) : value @@ -294,13 +292,6 @@ end # Read the document; an unparseable file becomes a 400 naming it. PrimersLibrary.load # raises on a bad file because an empty document would pass validation and the next # Save would overwrite the real primers. Returns the document, or an HTTP.Response error. -""" - _read_primers() -> Union{Dict,HTTP.Response} - -Read the canonical primers document, or an empty document if the file is absent. -Loading errors become a 400 `primers_unreadable` response naming the file and -reason; the file is not modified. -""" function _read_primers() try PrimersLibrary.load(_primers_path()) diff --git a/src/server/routes/databases.jl b/src/server/routes/databases.jl index 6ff74393..2d22fd0e 100644 --- a/src/server/routes/databases.jl +++ b/src/server/routes/databases.jl @@ -15,13 +15,6 @@ end # Read the document; an unparseable file becomes a 400 naming it. DatabasesLibrary.load # raises on a bad file because an empty document would pass validation and the next # Save would overwrite the real config. Returns the document, or an HTTP.Response error. -""" - _read_databases() -> Union{Dict,HTTP.Response} - -Read the canonical databases document, or an empty document if the file is absent. -Loading errors become a 400 `databases_unreadable` response naming the file and -reason; the file is not modified. -""" function _read_databases() try DatabasesLibrary.load(_databases_path()) diff --git a/src/server/server.jl b/src/server/server.jl index 880a3f84..cef70d33 100644 --- a/src/server/server.jl +++ b/src/server/server.jl @@ -221,15 +221,11 @@ module Server _allowed_origins() -> Set{String} The browser origins allowed to call the API cross-origin: the origin this - server is served from over HTTP (its bound port, spelled with `localhost` and + server is served from (its bound port, spelled with `localhost` and `127.0.0.1`, plus the bound host itself when it is a specific name), the Vite dev server, and any listed comma-separated in `JULIA_METAMANIFOLD_ALLOWED_ORIGINS`. Every other origin is refused, including other ports and schemes on localhost. - - Wildcard bind addresses are not included. Read the environment on each call, - ignoring blank entries and normalising case, surrounding whitespace and - trailing slashes; entries are not validated as URLs. """ function _allowed_origins() port = _bound_port[] @@ -245,43 +241,12 @@ module Server allowed end - _is_wildcard(h::AbstractString) = lowercase(h) in ("0.0.0.0", "::", "[::]") - - """ - _bound_ip_origin(origin, host_header) -> Bool - - Whether `origin` names the exact IP address the browser reached a wildcard - bind on: an `http://` origin whose host is an IP literal (not a name), whose - port is the bound port, and whose host-and-port equals the request's `Host` - header. A browser only sends such an `Origin` for a page it loaded from that - address, so the request is same-origin by construction. A DNS-rebinding page - arrives with the attacker's domain as both `Host` and `Origin`; a name is not - an IP literal, so it is refused here as before. Hostnames (e.g. - `http://lab-box:8080`) and any address behind a proxy on another port must be - listed in `JULIA_METAMANIFOLD_ALLOWED_ORIGINS`. - """ - function _bound_ip_origin(origin::AbstractString, host_header::AbstractString) - o = _normalise_origin(origin) - startswith(o, "http://") || return false - hostport = o[length("http://") + 1:end] - hostport == lowercase(strip(host_header)) || return false - m = match(r"^(?:\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-f:]+\]):(\d+)$"i, hostport) - !isnothing(m) && parse(Int, m.captures[1]) == _bound_port[] - end - """ _cors_middleware(next) - Return a request handler that answers 403 for a non-empty, non-loopback Host - on a loopback bind, or a non-empty `Origin` outside `_allowed_origins()` after - normalisation. On a wildcard bind an origin naming the request's own IP host - on the bound port is also admitted (`_bound_ip_origin`): the UI is then - served, and writable, at whatever address the machine is reached on, without - trusting a hostname from the `Host` header. The Host check also applies to - requests without an Origin. Allowed origins get CORS headers echoing the - supplied Origin; OPTIONS gets 204 without calling `next`. Otherwise call - `next(req)`, adding CORS headers only when the Origin is non-empty. Errors - from `next` propagate. + Refuse requests whose Host is not loopback on a loopback bind, and requests + carrying an `Origin` outside `_allowed_origins()`. Allowed origins get CORS + headers and preflight answers; requests with no `Origin` pass unchanged. """ function _cors_middleware(next) function(req::HTTP.Request) @@ -296,8 +261,6 @@ module Server return next(req) end _normalise_origin(origin) in _allowed_origins() || - (_is_wildcard(_bound_host[]) && - _bound_ip_origin(origin, HTTP.header(req, "Host", ""))) || return HTTP.Response(403, "Forbidden: origin not allowed") cors_headers = [ "Access-Control-Allow-Origin" => origin, @@ -329,13 +292,8 @@ module Server _file_middleware(next) Serve run outputs under `/files/{study}/runs/...` from the projects tree and - the frontend build for other non-API paths, passing `/api/` requests to `next`. - Return a request handler; query strings are ignored when locating files. - Missing run files return 404; existing files resolving outside the projects - tree return 403. For other paths, serve a matching frontend file or fall back - to `index.html`, calling `next` if neither exists. + the frontend build for every other non-API path, passing `/api/` requests on. Run files are sent with `nosniff`; report uploads also get `_UPLOAD_CSP`. - File resolution and read errors, and errors from `next`, propagate. """ function _file_middleware(next) function(req::HTTP.Request) @@ -438,19 +396,6 @@ module Server # above, so they would bypass the Host and Origin checks; nothing uses them. const _SERVE_OPTIONS = (access_log=nothing, show_errors=false, docs=false, metrics=false) - """ - start(; root=pwd(), host="127.0.0.1", port=8080, listen=true) - - Set the server's root for `data/`, `projects/` and `config/`, record the bind - host and port for request guards, and initialise missing study projects. - Individual project creation failures are caught and startup continues; - errors enumerating studies propagate. - - With `listen=false`, perform this initialisation and return `nothing` without - opening a listener. Otherwise serve the API and frontend with the request - guards enabled and Oxygen docs and metrics disabled, waiting until the server - stops and returning its server object. Listener startup errors propagate. - """ function start(; root=pwd(), host="127.0.0.1", port=8080, listen::Bool=true) global_logger(_SuppressEpipe(global_logger())) ServerState.set_root!(root) diff --git a/test/fixtures/jcs_oracle.jsonl b/test/fixtures/jcs_oracle.jsonl deleted file mode 100644 index a61e6e37..00000000 --- a/test/fixtures/jcs_oracle.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"input":{"b":1,"a":2,"c":[]},"expected":"{\"a\":2,\"b\":1,\"c\":[]}"} -{"input":{"€":"euro","Ａ":"fullwidth A","a":"first"},"expected":"{\"a\":\"first\",\"€\":\"euro\",\"Ａ\":\"fullwidth A\"}"} -{"input":{"😀":"astral first?","Ａ":"high bmp"},"expected":"{\"😀\":\"astral first?\",\"Ａ\":\"high bmp\"}"} -{"input":{"d":[3,1,2],"e":{"z":1,"y":{"x":[true,false,null]}}},"expected":"{\"d\":[3,1,2],\"e\":{\"y\":{\"x\":[true,false,null]},\"z\":1}}"} -{"input":[4.50,24499999999999998,1E+30,0.000001,1E-7,1E+21,1E+20,1E-33,-0.0,0.0,9.9999999999999997E+22],"expected":"[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,1e-33,0,0,1e+23]"} -{"input":[0.000001,1E-7,1.5E-10,9007199254740991,-42],"expected":"[0.000001,1e-7,1.5e-10,9007199254740991,-42]"} -{"input":{"s\":with\"quotes":"tab\there\nnewline","esc\\":"\\backslash","ctl":"\u0001\u001f"},"expected":"{\"ctl\":\"\\u0001\\u001f\",\"esc\\\\\":\"\\\\backslash\",\"s\\\":with\\\"quotes\":\"tab\\there\\nnewline\"}"} -{"input":{"nested":{"arr":[{"n":1.0},{"m":2}],"deep":{"deeper":{"deepest":"value"}}}},"expected":"{\"nested\":{\"arr\":[{\"n\":1},{\"m\":2}],\"deep\":{\"deeper\":{\"deepest\":\"value\"}}}}"} -{"input":{"literals":true,"n":null,"f":false,"empty_obj":{},"empty_arr":[]},"expected":"{\"empty_arr\":[],\"empty_obj\":{},\"f\":false,\"literals\":true,\"n\":null}"} -{"input":[5E-324,1.7976931348623157E+308,2.2250738585072014E-308,0.3,0.1,0.30000000000000004],"expected":"[5e-324,1.7976931348623157e+308,2.2250738585072014e-308,0.3,0.1,0.30000000000000004]"} -{"input":{"Plotly":{"layout":{"title":{"text":"CO₂"}},"data":[{"x":[1,2,3],"y":[4.5,5.5,6.5],"type":"scatter"}]}},"expected":"{\"Plotly\":{\"data\":[{\"type\":\"scatter\",\"x\":[1,2,3],\"y\":[4.5,5.5,6.5]}],\"layout\":{\"title\":{\"text\":\"CO₂\"}}}}"} diff --git a/test/integration/test_pipeline.jl b/test/integration/test_pipeline.jl index f3a42bc0..bd3b5d6f 100644 --- a/test/integration/test_pipeline.jl +++ b/test/integration/test_pipeline.jl @@ -261,7 +261,7 @@ run_dir(run) = joinpath(PROJECT_ROOT, "projects", PROJECT_NAME, run) probe_cfg = Dict("occupancy_probe" => Dict("enabled" => true, "interval_ms" => 10)) old_root = SV.ServerState._root[] - original_configs = Dict{String,Union{Nothing,Vector{UInt8}}}() + written = String[] try SV.ServerState.set_root!(PROJECT_ROOT) @@ -269,18 +269,9 @@ rm(joinpath(run_dir("run_A"), "vsearch"); recursive=true, force=true) rm(joinpath(run_dir("run_A"), "swarm", "vsearch"); recursive=true, force=true) yaml_a = joinpath(PROJECT_ROOT, "data", PROJECT_NAME, "run_A", "pipeline.yml") - original_configs[yaml_a] = isfile(yaml_a) ? read(yaml_a) : nothing YAML.write_file(yaml_a, probe_cfg) - response = SV.Oxygen.internalrequest(SV.HTTP.Request( - "POST", "/api/v1/studies/$PROJECT_NAME/runs/run_A/stages/vsearch")) - @test response.status == 200 - job = SV.JobQueue.get_job(String(JSON3.read(String(response.body)).id)) - @test !isnothing(job) - task = lock(SV.JobQueue._jobs_lock) do - job.task - end - isnothing(task) || wait(task) - @test job.status == SV.JobQueue.complete + push!(written, yaml_a) + SV._run_stage(PROJECT_NAME, "run_A", "vsearch", db_config, dbs_loaded) @test isfile(joinpath(run_dir("run_A"), "vsearch", "taxonomy.tsv")) @test isfile(joinpath(run_dir("run_A"), "swarm", "vsearch", "taxonomy.tsv")) @@ -288,8 +279,8 @@ rm(joinpath(run_dir("run_B"), "vsearch"); recursive=true, force=true) rm(joinpath(run_dir("run_B"), "swarm", "vsearch"); recursive=true, force=true) yaml_b = joinpath(PROJECT_ROOT, "data", PROJECT_NAME, "run_B", "pipeline.yml") - original_configs[yaml_b] = isfile(yaml_b) ? read(yaml_b) : nothing YAML.write_file(yaml_b, probe_cfg) + push!(written, yaml_b) merged = SV._run_full_pipeline(PROJECT_NAME, "run_B", db_config, dbs_loaded) @test haskey(merged.tables, "merged") @test isfile(joinpath(run_dir("run_B"), "vsearch", "taxonomy.tsv")) @@ -309,9 +300,7 @@ @test labels == ["vsearch:asv", "vsearch:otu"] end finally - for (path, contents) in original_configs - isnothing(contents) ? rm(path; force=true) : write(path, contents) - end + foreach(p -> rm(p; force=true), written) SV.ServerState._root[] = old_root end end diff --git a/test/occupancy_disagreement_test.jl b/test/occupancy_disagreement_test.jl index 15df70fd..00837263 100644 --- a/test/occupancy_disagreement_test.jl +++ b/test/occupancy_disagreement_test.jl @@ -341,25 +341,6 @@ _task(rec, label) = only(t for t in rec["tasks"] if t["label"] == label) @info "occupancy: concurrent stage peak vs the larger serial stage peak" concurrent = conc["child_peak_rss_kib"] serial = serial_max overlapped = overlap cost_spread = round(spread; sigdigits = 3) overlap || @warn "occupancy: the concurrent searches did not overlap in memory on this runner, so this run cannot show the order dependence" - # The grades the sample windows imply, on this same run - # (fork issue #105). The peak is recorded, not asserted — a - # runner that never has both searches resident cannot show it — - # but the algebra's own claims are asserted of every order: - # every task is timed, the prediction at least reaches what the - # samples saw (the prediction is an upper bound: at a handover - # the live task holds at most its peak), it never exceeds the - # cost bound (Reclaim.occupancy≤cost), and the slack between - # them is never negative. - for o in ORDERS - r = records[o] - @test all(t -> haskey(t, "first_sample") && haskey(t, "last_sample"), r["tasks"]) - @test haskey(r, "grades") - @test MP.predicted_peak_rss_kib(r) >= r["child_peak_rss_kib"] - @test MP.predicted_peak_rss_kib(r) <= MP.cost_bound_rss_kib(r) - @test MP.reclaimed_kib(r) >= 0 - @test MP.disagreement(r) === nothing - end - path = get(ENV, "OCCUPANCY_RESULTS", "") if !isempty(path) io = IOBuffer() diff --git a/test/runtests.jl b/test/runtests.jl index 77e15bcf..ed351f13 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -45,7 +45,6 @@ const UNIT_FILES = [ "test_remote_stages.jl", "test_jobs.jl", "test_provenance.jl", - "test_occupancy_grade.jl", "test_install_pins.jl", "test_migrate_composition.jl", "test_sample_reads.jl", @@ -54,7 +53,6 @@ const UNIT_FILES = [ "test_permanova.jl", "test_write_gates.jl", "test_server_security.jl", - "test_canonical.jl", ] # The files named on the command line ("test_routes.jl", "routes" or diff --git a/test/unit/test_canonical.jl b/test/unit/test_canonical.jl deleted file mode 100644 index 1a9bb01b..00000000 --- a/test/unit/test_canonical.jl +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) -# -# Unit tests for the JCS (RFC 8785) canonical JSON writer. The oracle is the -# estate's ijson-jcs crate (Rust): every fixture in -# test/fixtures/jcs_oracle.jsonl was PRODUCED by that crate, and this file -# asserts MetaManifold.Canonical reproduces it byte-for-byte. Any -# disagreement is a bug in one of the two implementations, and the fixture -# names the case. -using MetaManifold -using MetaManifold.Canonical: jcs_bytes, jcs_sha256 -using JSON3, SHA, Test - -@testset "canonical JSON (JCS)" begin - # Byte-for-byte against the Rust crate's output, over the fixture - # corpus: key sorting (including the astral-before-high-BMP UTF-16 - # case), ES number formatting, string escaping, nesting, literals. - fixtures = joinpath(@__DIR__, "..", "fixtures", "jcs_oracle.jsonl") - for (i, line) in enumerate(eachline(fixtures)) - case = JSON3.read(line) - @testset "oracle case $i" begin - value = case.input - # The input arrives already parsed; the expected is the oracle's text. - @test String(jcs_bytes(value)) == case.expected - end - end - - @testset "key order is irrelevant, bytes are not" begin - a = Dict("b" => 1, "a" => Dict("y" => [1.0, 2], "x" => "s")) - b = Dict("a" => Dict("x" => "s", "y" => [1.0, 2]), "b" => 1) - @test jcs_bytes(a) == jcs_bytes(b) - # ...but array order is preserved, as JCS requires. - c = Dict("a" => [2, 1]) - @test jcs_bytes(a) != jcs_bytes(c) - end - - @testset "the astral key sorts before the high BMP key" begin - # U+1F600 arrives as the surrogate pair D83D DE00; UTF-16 code-unit - # order puts it before U+FF21. Naive code-point order would agree - # here; naive byte order would too -- this case pins the rule. - d = Dict("\uff21" => "high bmp", "\U0001f600" => "astral") - @test String(jcs_bytes(d)) == "{\"😀\":\"astral\",\"Ａ\":\"high bmp\"}" - end - - @testset "float notation follows ECMAScript" begin - @test String(jcs_bytes([4.50, 2.4499999999999998e16, 1e30, 1e-6, 1e-7, - 1e21, 1e20, -0.0, 5e-324])) == - "[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,0,5e-324]" - end - - @testset "refusals" begin - @test_throws ArgumentError jcs_bytes(2^53) - @test_throws ArgumentError jcs_bytes(-(2^53)) - @test_throws ArgumentError jcs_bytes(Inf) - @test_throws ArgumentError jcs_bytes(NaN) - @test jcs_bytes(2^53 - 1) == b"9007199254740991" - end - - @testset "a content address is stable" begin - # The use this exists for: a chart-like spec built in any key - # order hashes identically, and the hash can be recorded where the - # bytes cannot. - spec = Dict( - "layout" => Dict("title" => Dict("text" => "CO₂")), - "data" => [Dict("x" => [1, 2, 3], "y" => [4.5, 5.5, 6.5], - "type" => "scatter")]) - same = Dict( - "data" => [Dict("type" => "scatter", "y" => [4.5, 5.5, 6.5], - "x" => [1, 2, 3])], - "layout" => Dict("title" => Dict("text" => "CO₂"))) - @test jcs_sha256(spec) == jcs_sha256(same) - @test jcs_sha256(spec) == bytes2hex(sha256(jcs_bytes(spec))) - end -end diff --git a/test/unit/test_dada2_tables.R b/test/unit/test_dada2_tables.R new file mode 100644 index 00000000..228bae1b --- /dev/null +++ b/test/unit/test_dada2_tables.R @@ -0,0 +1,74 @@ +# Regression checks for CSV assembly, without loading DADA2 or dplyr. +# Run from the repository root: Rscript --vanilla test/unit/test_dada2_tables.R +expressions <- parse("src/pipeline/dada2/dada2_functions.r") +wanted <- c("base_left_join", "write_taxa_table", "write_combined_table") +loaded <- character() +for (expr in expressions) { + if (is.call(expr) && identical(expr[[1]], as.name("<-")) && + is.symbol(expr[[2]]) && as.character(expr[[2]]) %in% wanted) { + eval(expr) + loaded <- c(loaded, as.character(expr[[2]])) + } +} +stopifnot(setequal(loaded, wanted)) + +# Unsorted keys, repeated matches, unmatched rows, overlapping column names, +# NA keys, and helper-column name collisions must preserve the intended order. +x <- data.frame(value = 1:4, key = c("b", "a", "z", NA), + .left_join_position = 11:14) +y <- data.frame(key = c("a", "b", "b", NA), value = 21:24, + .right_join_position = 31:34) +got <- base_left_join(x, y, "key") +stopifnot(identical(got$key, c("b", "b", "a", "z", NA_character_)), + identical(got$value.x, c(1L, 1L, 2L, 3L, 4L)), + identical(got$value.y, c(22L, 23L, 21L, NA_integer_, 24L)), + identical(names(got), c("value.x", "key", ".left_join_position", + "value.y", ".right_join_position"))) +stopifnot(nrow(base_left_join(x[FALSE, ], y, "key")) == 0L, + nrow(base_left_join(x, y[FALSE, ], "key")) == nrow(x)) +# Multiple keys: matching only one component must not produce a match. +a <- data.frame(id = c("a", "a"), key = c("x", "y"), value = 1:2) +b <- data.frame(id = c("a", "b"), key = c("y", "x"), other = 3:4) +stopifnot(identical(base_left_join(a, b, c("id", "key"))$other, + c(NA_integer_, 3L))) + +# Rank names and their suffixed variants are also valid sample IDs. +# Repeated suffixing must preserve both the taxonomy strings and count columns. +tax_collision <- data.frame(Sequence = c("AA", "GG"), Species = c("taxA", "taxG")) +count_collision <- data.frame(Sequence = c("GG", "AA"), Species = c(10L, 20L), + Species.x = c(30L, 40L), Species.x.x = c(50L, 60L)) +collision <- base_left_join(tax_collision, count_collision, "Sequence") +stopifnot(identical(names(collision), + c("Sequence", "Species.x.x.x", "Species.y", "Species.x", "Species.x.x")), + identical(collision[[2]], c("taxA", "taxG")), + identical(collision[[3]], c(20L, 10L)), + identical(collision[[4]], c(40L, 30L)), + identical(collision[[5]], c(60L, 50L))) + +# Fixed expected CSV bytes cover every writer output, including non-syntactic +# sample names and an unmatched taxonomy row. No sequencing dependencies needed. +out <- tempfile("dada2-tables-") +dir.create(out) +index <- data.frame(SeqName = c("ASV3", "ASV1", "ASV2"), + Sequence = c("TT", "GG", "AA")) +tax <- matrix(c("G", "A"), ncol = 1, + dimnames = list(c("GG", "AA"), "Kingdom")) +boot <- matrix(c(90L, 80L), ncol = 1, dimnames = dimnames(tax)) +taxa <- write_taxa_table(tax, boot, index, out, "taxonomy") +counts <- matrix(c(1L, 2L, 3L, 4L, 5L, 6L), nrow = 2, + dimnames = list(c("sample-1", "sample 2"), c("AA", "TT", "GG"))) +write_combined_table(taxa, index, counts, out, "tax_counts.csv", "asv_counts.csv") +expected <- list( + taxonomy.csv = c("SeqName,Sequence,Kingdom", "ASV3,TT,NA", "ASV1,GG,G", "ASV2,AA,A"), + taxonomy_bootstraps.csv = c("SeqName,Sequence,Kingdom_boot", "ASV3,TT,NA", "ASV1,GG,90", "ASV2,AA,80"), + taxonomy_combined.csv = c("SeqName,Sequence,Kingdom,Kingdom_boot", "ASV3,TT,NA,NA", "ASV1,GG,G,90", "ASV2,AA,A,80"), + tax_counts.csv = c("SeqName,Sequence,Kingdom,sample-1,sample 2", "ASV3,TT,NA,3,4", "ASV1,GG,G,5,6", "ASV2,AA,A,1,2"), + asv_counts.csv = c("SeqName,Sequence,sample-1,sample 2", "ASV3,TT,3,4", "ASV1,GG,5,6", "ASV2,AA,1,2") +) +for (name in names(expected)) { + path <- file.path(out, name) + bytes <- readBin(path, "raw", n = file.info(path)$size) + stopifnot(identical(bytes, charToRaw(paste0(paste(expected[[name]], collapse = "\n"), "\n")))) +} +unlink(out, recursive = TRUE) +cat("DADA2 join and five CSV fixtures passed\n") diff --git a/test/unit/test_differential.jl b/test/unit/test_differential.jl index e46559f4..3b034896 100644 --- a/test/unit/test_differential.jl +++ b/test/unit/test_differential.jl @@ -172,14 +172,6 @@ end @test_throws ArgumentError DifferentialConfig(; method="ancombc") # The pre-review name, never released, is not accepted. @test_throws ArgumentError DifferentialConfig(; method="clr_lm") - # The dispersion option: ml is the default, glmGamPoi is the alternative, - # and it is a negative-binomial setting only. - @test DifferentialConfig().dispersion == "ml" - @test DifferentialConfig(dispersion="glmGamPoi").dispersion == "glmGamPoi" - @test DifferentialConfig(dispersion=" GLMGAMPOI ").dispersion == "glmGamPoi" - @test_throws ArgumentError DifferentialConfig(dispersion="shrinkage") - @test_throws ArgumentError DifferentialConfig(method="clr_welch", dispersion="glmGamPoi") - @test DifferentialConfig(method="clr_welch").dispersion == "ml" for d in (0, 1, -0.5, NaN) @test_throws ArgumentError DifferentialConfig(; method="clr_welch", replacement_delta=d) end @@ -728,48 +720,4 @@ end @test rcopy(Bool, reval("requireNamespace('zCompositions', quietly = TRUE)")) == HAVE_ZCOMP end end - - ## Without glmGamPoi, dispersion: glmGamPoi refuses with - # GlmGamPoiUnavailable. Same trick as the zCompositions testset above: a - # `requireNamespace` bound in R's global environment reports glmGamPoi - # missing and defers to base for every other package, so the production - # probe sees the package as absent on any machine. MASS is probed first - # and must be real, so this testset needs it. - @testset "nb_glm without glmGamPoi refuses with GlmGamPoiUnavailable" begin - if !HAVE_MASS - @test_skip false # no MASS means the ml path's own guard fires first - else - reval("""requireNamespace <- function(package, ...) { - if (identical(package, "glmGamPoi")) FALSE - else base::requireNamespace(package, ...) - }""") - try - @test !rcopy(Bool, reval("requireNamespace('glmGamPoi', quietly = TRUE)")) - e = try - fit_nb([5 6; 7 3; 9 4; 4 8], ["A", "A", "B", "B"], [0.0, 0.0, 0.0, 0.0]; - levels=("A", "B"), dispersion="glmGamPoi") - nothing - catch err - err - end - @test e isa GlmGamPoiUnavailable - @test occursin("renv::restore()", sprint(showerror, e)) - # The override defers to base for everything else, so the default - # dispersion must still fit normally with it in place. The - # injected-effect dataset, not the minimal matrix above: - # MEASURED with MASS::glm.nb, with 4 samples of small counts the - # alternating fit runs theta to its upper bound (status boundary) - # and one taxon's fit even produced NaNs (status failed), so that - # data never reaches status ok on any machine; the dataset - # "Negative-binomial fit recovers an injected effect" already - # proves fits. - r = fit_nb(effect, effect_groups, log.(tss_factors(effect, effect_samples)); - levels=("A", "B")) - @test all(r.status .== "ok") - finally - reval("rm(requireNamespace)") - end - @test !rcopy(Bool, reval("exists('requireNamespace', envir = globalenv(), inherits = FALSE)")) - end - end end diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index bfa9bbb5..a3053b97 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -14,9 +14,8 @@ using SHA const REPO_ROOT = normpath(joinpath(@__DIR__, "..", "..")) const PINS_PATH = joinpath(REPO_ROOT, "config", "defaults", "tool_versions.yml") -const INSTALL_JL = joinpath(REPO_ROOT, "scripts", "install.jl") +const INSTALL_JL = joinpath(REPO_ROOT, "install.jl") const CI_PATH = joinpath(REPO_ROOT, ".github", "workflows", "ci.yml") -const WORKFLOWS = joinpath(REPO_ROOT, ".github", "workflows") const MANIFEST = joinpath(REPO_ROOT, "Manifest.toml") const RENV_LOCK = joinpath(REPO_ROOT, "renv.lock") @@ -30,11 +29,13 @@ const EXPECTED_PLATFORMS = ["linux-x86_64", "linux-aarch64", "macos-x86_64", "ma is_sha256(s) = s isa AbstractString && occursin(r"^[0-9a-f]{64}$", s) -# Allow GitHub's own actions, this fork's actions, and Claude Code, each pinned to -# a full commit SHA. -allowed_action(uses) = - occursin(r"^(actions|hyperpolymath)/[A-Za-z0-9_.-]+(/[A-Za-z0-9_./-]+)?@[0-9a-f]{40}$", uses) || - occursin(r"^anthropics/claude-code-action@[0-9a-f]{40}$", uses) +"""Return the first step of `job` whose `uses:` names `action`, or nothing.""" +function ci_step_using(job, action) + for step in get(job, "steps", []) + startswith(get(step, "uses", ""), action) && return step + end + return nothing +end @testset "install pins" begin @@ -186,18 +187,18 @@ allowed_action(uses) = ci = YAML.load_file(CI_PATH) job = ci["jobs"]["test"] - # Julia is the one runtime CI cannot read from the pin file, because nothing can - # be read before Julia exists. Its archive URL and checksum are therefore - # duplicated in the job's env, and this is the test that makes the duplication - # safe. The pinned URL names the version, which must equal Manifest.toml's - # (tested above). + # Julia is the one version CI cannot read from the pin file, because nothing can + # be read before Julia exists. It is therefore duplicated, and this is the test + # that makes the duplication safe. # - # Read from the job's env rather than from a matrix: the matrix was removed - # because GitHub appends a matrix combination to the posted check name (see "a - # required check name is a stable identifier" below). - archive = pins["runtimes"]["julia"]["archives"]["linux-x86_64"] - @test job["env"]["JULIA_URL"] == archive["url"] - @test job["env"]["JULIA_SHA256"] == archive["sha256"] + # Read from the `Set up Julia` step rather than from a matrix: the matrix was + # removed because GitHub appends a matrix combination to the posted check name + # (see "a required check name is a stable identifier" below). The step is located + # by its `uses:` rather than by index, so reordering the steps cannot make this + # assertion quietly vanish. + setup = ci_step_using(job, "julia-actions/setup-julia") + @test setup !== nothing + @test setup["with"]["version"] == pins["runtimes"]["julia"]["version"] # A floating runner would carry the R apt pin, which names a 24.04 build, off to # whatever the next LTS ships. @@ -247,32 +248,9 @@ allowed_action(uses) = @test !occursin("set +e", fetch_code) @test occursin("source scripts/ci/fetch_pinned.sh", runs) - for ref in (raw"$VSEARCH_URL", raw"$SWARM_URL", raw"$FASTQC_URL", - raw"$JULIA_URL", raw"$BUN_URL") + for ref in (raw"$VSEARCH_URL", raw"$SWARM_URL", raw"$FASTQC_URL") @test occursin("fetch_pinned \"$ref\"", runs) end - - # Julia and Bun, which actions used to install, are refused unless they hash - # to their pins. - for (sha, file) in ((raw"$JULIA_SHA256", "julia.tar.gz"), (raw"$BUN_SHA256", "bun.zip")) - @test occursin("echo \"$sha $file\" | sha256sum -c -", runs) - end - end - - @testset "every action is one this account allows" begin - # Anything else ends the whole workflow in startup_failure before any job runs, - # so the run names no failing step. That is how ci.yml failed on every run on - # this fork, from at least 2026-10-02 until it stopped using julia-actions, - # oven-sh/setup-bun and codecov/codecov-action. - files = filter(f -> endswith(f, ".yml") || endswith(f, ".yaml"), readdir(WORKFLOWS)) - @test "ci.yml" in files - @testset "$file" for file in files - for (_, job) in YAML.load_file(joinpath(WORKFLOWS, file))["jobs"], - step in get(job, "steps", []) - uses = get(step, "uses", nothing) - uses === nothing || @test allowed_action(uses) - end - end end @testset "a required check name is a stable identifier" begin @@ -309,96 +287,3 @@ allowed_action(uses) = end end end - -@testset "the root keeps its shape" begin - # Adapted from the RSR root-cleanup doctrine (rsr-template-repo, - # docs/proposals/root-cleanup.adoc): the repository root holds authority - # files, the entry points a newcomer runs, tool-mandated build metadata, and - # one directory per concern -- nothing else. The check is BIDIRECTIONAL, - # because a one-directional allowlist only ever ratchets open: a tracked - # root entry that is not listed here is drift, and a listed entry that has - # gone missing is drift too. The tracked tree is what is checked, so local - # untracked build output (bin/, web/) does not fail a developer's run. To - # add a root entry, add it here with its reason in the same commit. - root_entries = [ - # Authority files. GitHub reads CITATION.cff from the default-branch - # root only, so it cannot move. - "README.md", "LICENSE", "CITATION.cff", - # The two entry points a newcomer runs (the launcher stays at the - # root); the installer itself lives under scripts/. - "install.sh", "start.sh", "launcher.sh", - # The RSR scheme's task entry and changelog index. - "Justfile", "CHANGELOG.md", - # Build metadata rooted by its tooling: Julia Pkg, and renv, which - # also scans R/ for dependency discovery. - "Project.toml", "Manifest.toml", "renv.lock", "renv", ".Rprofile", "R", - # One directory per concern. Health files (CONTRIBUTING.md) and CI - # configs (codecov.yml) live under .github/. - ".github", "bench", "config", "data", "docs", "frontend", - "proofs", "scripts", "src", "test", - # Agent instructions (the agents.md convention): coding agents load - # this at startup, and it only works from the root, which is why it - # is listed here rather than living under .github/ with the other - # health files. - "AGENTS.md", - # VCS housekeeping, and the editor configuration. - ".gitignore", ".editorconfig", - ] - tracked = Set(split(readchomp(`git -C $(REPO_ROOT) ls-tree --name-only HEAD`), "\n")) - for entry in root_entries - @test entry in tracked - end - extras = sort(collect(setdiff(tracked, Set(root_entries)))) - if !isempty(extras) - println("root entries outside the allowlist: ", join(extras, ", ")) - end - @test isempty(extras) -end - -@testset "dependabot never batches a major" begin - # #124 grouped every frontend update, and with nothing requiring the - # checks it merged five framework majors (React 18->19, vite 6->8, - # TypeScript 5->7, react-router-dom 6->7, plotly 2->4) in one PR, with - # tsc already red until #140 repaired it. A major is a migration, not an - # update, so it must open its own PR: a Dependabot group only claims an - # update when its patterns match AND its update-types admit it -- a - # dependency matching no group gets its own PR. Every group must - # therefore NAME its update-types, and never "major". The frontend must - # also stay on the "bun" ecosystem: the 2026-09 security-update runs - # proved this account's npm_and_yarn updater refuses a Bun-managed - # project (see the header of .github/dependabot.yml). - dependabot = YAML.load_file(joinpath(REPO_ROOT, ".github", "dependabot.yml")) - @test dependabot["version"] == 2 - ecosystems = Dict{String,Any}() - for u in dependabot["updates"] - @test !haskey(ecosystems, u["package-ecosystem"]) - ecosystems[u["package-ecosystem"]] = u - end - @test sort(collect(keys(ecosystems))) == ["bun", "github-actions"] - @test ecosystems["bun"]["directory"] == "/frontend" - for u in dependabot["updates"] - for (_, group) in get(u, "groups", Dict{String,Any}()) - update_types = get(group, "update-types", nothing) - @test update_types !== nothing - @test !("major" in update_types) - end - end -end - -@testset "KYAML is the authoring dialect, ratcheted" begin - # YAML-POLICY Y-3 (standards, owner rulings D280/D307): KYAML is the - # estate's authoring dialect; a *.kyaml file that is not in KYAML form - # is a violation, an unconverted *.yml is migration backlog. The - # formatter is the standards repo's kyaml-format.sh, verbatim: the one - # rewriter the step-2 comment-preservation proof covers. Tool-named - # files (workflows, dependabot, codecov) keep their extensions; the - # shared config defaults convert as an upstream unit, not unilaterally. - formatter = joinpath(REPO_ROOT, "scripts", "kyaml-format.sh") - tracked = split(readchomp(`git -C $(REPO_ROOT) ls-files`), "\n") - kyamls = [joinpath(REPO_ROOT, f) for f in tracked if endswith(f, ".kyaml")] - for path in kyamls - proc = run(pipeline(ignorestatus(`bash $formatter --check $path`), - stdout = devnull, stderr = devnull)) - @test success(proc) - end -end diff --git a/test/unit/test_occupancy_grade.jl b/test/unit/test_occupancy_grade.jl deleted file mode 100644 index 8b94a0a5..00000000 --- a/test/unit/test_occupancy_grade.jl +++ /dev/null @@ -1,270 +0,0 @@ -# © 2026 Joshua Benjamin Jewell. All rights reserved. -# Licensed under the GNU Affero General Public License version 3 (AGPLv3). - -## Grading a stage against the high-water-mark algebra (MMW-01, fork issue #105) -# The algebra is proved in occupancy-types' `OccupancyTypes.Reclaim`, over the HWM -# instance in resource-algebra; `scripts/fetch-siblings.sh` pins both and -# `scripts/check-occupancy.sh` type-checks what consumes them. Nothing here -# restates a sibling's definition: what is tested is the *reading* of a probe's -# samples as grades — that a task is sequenced from the previous one's drift, -# that the siblings' laws come out of that reading, and that the prediction -# reproduces the measurement the occupancy reading recorded on 2026-10-14. - -using MetaManifold.MemoryProbe -using OrderedCollections, YAML - -const MG = MetaManifold.MemoryProbe - -const RECORDED_OCCUPANCY = - normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "results-2026-10-14.yml")) -const RECORDED_SCHEDULES = - normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "schedules.yml")) - -# One task as the probe records it, with the sample window a schedule gives it. -# The windows determine which peaks remain live at each handover. -function _occ_task(peak; label = "t", first_sample = 1, last_sample = 2) - OrderedDict{String,Any}("label" => label, "processes" => 1, "samples" => 3, - "peak_rss_kib" => peak, "hwm_kib" => peak, - "first_sample" => first_sample, "last_sample" => last_sample) -end - -# One stage as `finish!` writes it: its tasks and the peak the samples saw. -_occ_stage(tasks; measured) = OrderedDict{String,Any}( - "probe" => MG.PROBE, "interval_ms" => 10, "samples" => 10, - "julia_allocated_bytes" => 1000, "julia_live_bytes_delta" => 500, - "child_peak_rss_kib" => measured, "tasks" => Any[tasks...]) - -@testset "occupancy grades" begin - @testset "a grade is (peak, drift), drift ≤ peak" begin - g = MG.Grade(90, 40) - @test g.peak == 90 - @test g.drift == 40 - @test MG.release(g) == 50 # what the frontier reclaims - # The invariant the HWM instance carries as its `bounded` field. - @test_throws ArgumentError MG.Grade(40, 90) - end - - @testset "sequencing starts the suffix from the drift (Reclaim.reclaim-law)" begin - a = MG.Grade(100, 40) - b = MG.Grade(60, 10) - s = MG.seq(a, b) - # peak (a ⊗ b) ≡ max (peak a) (drift a + peak b) - @test s.peak == max(a.peak, a.drift + b.peak) - @test s.drift == a.drift + b.drift - @test MG.seq(a, MG.NO_GRADE) == a # the unit holds nothing - @test s.peak <= a.peak + b.peak # Reclaim.occupancy≤cost - # Alternatives keep the larger of each component. - @test MG.alt(a, b) == MG.Grade(100, 40) - end - - @testset "the two shapes a stage has: serial and concurrent" begin - # One after the other: the first has exited (drift = 0), so the peak is - # the larger of the two and the cut reclaims the difference. - serial = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 2), - _occ_task(60; label = "otu", first_sample = 3, last_sample = 4)]; - measured = 100) - @test MG.predicted_peak_rss_kib(serial) == 100 - @test MG.cost_bound_rss_kib(serial) == 160 - @test MG.reclaimed_kib(serial) == 60 # Reclaim.reclaimed: strictly below cost - @test MG.disagreement(serial) === nothing - - # Together: the first is still resident when the second starts - # (drift = peak), so the peak is the sum and nothing is reclaimed. - together = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 4), - _occ_task(60; label = "otu", first_sample = 2, last_sample = 5)]; - measured = 160) - @test MG.predicted_peak_rss_kib(together) == 160 - @test MG.cost_bound_rss_kib(together) == 160 - @test MG.reclaimed_kib(together) == 0 # Reclaim.no-reclaim: drift = peak - @test MG.disagreement(together) === nothing - end - - @testset "overlap expires at each handover" begin - for (windows, predicted, remaining) in ( - ([(1, 2), (2, 3), (4, 5)], 160, 0), # an overlapping pair, then a serial task - ([(1, 3), (2, 5), (4, 6)], 160, 0), # pairwise overlap, never all three - ([(1, 6), (2, 3), (4, 5)], 180, 0), # the first survives a shorter task - ([(1, 5), (2, 5), (3, 5)], 240, 0), # genuine three-way overlap - ([(1, 2), (1, 2), (3, 4)], 160, 0), # children registered before sample 1 - ([(1, 0), (2, 3), (4, 0)], 180, 180), # unclosed windows remain live - ) - tasks = [_occ_task(peak; label = "t$i", first_sample = window[1], - last_sample = window[2]) - for (i, (peak, window)) in enumerate(zip((100, 60, 80), windows))] - occ = _occ_stage(tasks; measured = predicted) - grades = MG.task_grades(occ) - folded = foldl(MG.seq, grades) - @test folded.peak == predicted - @test folded.drift == remaining - @test MG.release(folded) == predicted - remaining - @test MG.cost_bound_rss_kib(occ) == 240 - @test MG.reclaimed_kib(occ) == 240 - predicted - @test MG.disagreement(occ) === nothing - @test MG.graded(occ)["predicted_peak_kib"] == predicted - # Record order does not change the sampled order. - @test MG.stage_grade(_occ_stage(reverse(tasks); measured = predicted)) == folded - end - @test !MG._overlaps((0, 0), (1, 2)) # a never-started task is not live - @test !MG._overlaps((1, 0), (0, 0)) - @test MG._overlaps((1, 2), (1, 2)) - @test MG._overlaps((1, 2), (2, 3)) # the end sample is included - @test !MG._overlaps((1, 2), (3, 4)) - end - - @testset "the reviewer's three-task example: expiry, not the sum of drifts" begin - # Windows [1,3], [2,10], [4,5] and peaks 100, 1, 100 KiB — CodeRabbit's - # example on #117. A overlaps B and exits before C starts, so the largest - # simultaneous sum is 101 (A+B at samples 2-3, then B+C at 4-5), not the - # 201 a rule that kept every earlier drift would read, and not the 201 KiB - # cost. A's release at C's boundary makes B's delta negative, which is how - # the fold stops charging for it. - tasks = [_occ_task(100; label = "A", first_sample = 1, last_sample = 3), - _occ_task(1; label = "B", first_sample = 2, last_sample = 10), - _occ_task(100; label = "C", first_sample = 4, last_sample = 5)] - occ = _occ_stage(tasks; measured = 101) - @test [g.drift for g in MG.task_grades(occ)] == [100, -99, -1] - @test MG.predicted_peak_rss_kib(occ) == 101 - @test MG.cost_bound_rss_kib(occ) == 201 - @test MG.reclaimed_kib(occ) == 100 # the 100 KiB that never coexist - @test MG.disagreement(occ) === nothing - # The bound the expiry sharpens: 201 KiB resident at once is now the - # disagreement, where before the fix it was the prediction. - @test MG.disagreement(_occ_stage(tasks; measured = 201)) isa String - end - - @testset "registration keeps the first child's sample boundary" begin - # No background sampler: watch! takes the first sample deterministically. - probe = MG.Probe(0.01, ReentrantLock(), MG.TaskTrace[], MG.TaskTrace("(stage)"), - 0, 0, Base.gc_num(), Base.gc_live_bytes(), false, nothing, - nothing, nothing, nothing, Vector{UInt8}(undef, 8192), - Vector{UInt8}(undef, 64)) - function child!() - proc = run(`sleep 0.01`; wait = false) - pid = MG.watch!(proc) - try - wait(proc) - finally - MG.unwatch!(pid) - end - end - MG.within(probe) do - MG.task("two children") do - child!() - @test probe.samples == 1 - @test probe.tasks[1].first_sample == 1 - @test probe.tasks[1].last_sample == 1 - MG.task("between children") do - child!() - end - child!() - end - end - first_task, between = probe.tasks - @test first_task.processes == 2 - @test first_task.first_sample == 1 - @test first_task.last_sample == 3 - @test between.first_sample == 2 - @test MG._overlaps((first_task.first_sample, first_task.last_sample), - (between.first_sample, between.last_sample)) - @test MG.TaskTrace("never started").first_sample == 0 - end - - @testset "the graded record says what it is" begin - rec = _occ_stage([_occ_task(100; label = "vsearch:asv", first_sample = 1, last_sample = 4), - _occ_task(60; label = "vsearch:otu", first_sample = 2, last_sample = 5)]; - measured = 160) - g = MG.graded(rec) - @test g["algebra"] == "ResourceAlgebra.Instances.HWM" - @test g["predicted_peak_kib"] == 160 - @test g["cost_bound_kib"] == 160 - @test g["reclaimed_kib"] == 0 - @test g["measured_peak_kib"] == 160 - @test g["disagreement"] === nothing - @test [t["label"] for t in g["tasks"]] == ["vsearch:asv", "vsearch:otu"] - @test g["tasks"][1]["drift_kib"] == 100 # still resident at the handover - @test g["tasks"][2]["drift_kib"] == -100 # the earlier task also exited - @test g["tasks"][2]["release_kib"] == 160 # both peaks are reclaimed - @test g["drift_kib"] == 0 - end - - @testset "a measured peak above the prediction is a disagreement" begin - # The prediction is an upper bound, so this is the one contradiction it - # allows: a sample saw more than the timed timeline can hold. - rec = _occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; measured = 250) - d = MG.disagreement(rec) - @test d isa String - @test occursin("250", d) && occursin("100", d) - # At or below the bound there is nothing to report. - @test MG.disagreement(_occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; - measured = 100)) === nothing - end - - @testset "a record with no sample windows is not graded" begin - # Records written before the windows were sampled cannot be read as a - # timeline, so nothing is predicted for them rather than an order assumed. - rec = OrderedDict{String,Any}("child_peak_rss_kib" => 100, - "tasks" => Any[OrderedDict{String,Any}("label" => "t", "peak_rss_kib" => 100)]) - @test MG.predicted_peak_rss_kib(rec) === nothing - @test MG.graded(rec) === nothing - @test MG.disagreement(rec) === nothing # not measured is not a disagreement - end - - @testset "finish! grades the record it writes" begin - # A record the probe writes itself carries its tasks' sample windows, so - # `finish!` attaches the grades read from them; the run with real vsearch - # children is test/occupancy_disagreement_test.jl. - if !Sys.islinux() - @info "occupancy grades: not Linux, so there is no /proc to sample; the finish! grading check is SKIPPED" - else - probe = MG.start(Dict("occupancy_probe" => Dict("enabled" => true, - "interval_ms" => 10))) - MG.within(probe) do - MG.task("sleep") do - proc = run(`sleep 0.2`; wait = false) - pids = MG.watch!(proc) - try - wait(proc) - finally - MG.unwatch!(pids) - end - end - end - rec = MG.finish!(probe) - @test length(rec["tasks"]) == 1 # one task; no stage trace - t = rec["tasks"][1] - @test t["label"] == "sleep" && t["samples"] > 0 - @test t["peak_rss_kib"] > 0 # sampled from /proc - @test t["last_sample"] >= t["first_sample"] # the window watch!/unwatch! set - @test haskey(rec, "grades") - @test rec["grades"]["predicted_peak_kib"] == MG.predicted_peak_rss_kib(rec) - @test rec["grades"]["measured_peak_kib"] == rec["child_peak_rss_kib"] - @test MG.disagreement(rec) === nothing - end - end - - @testset "the prediction reproduces the recorded measurement" begin - # docs/occupancy/results-2026-10-14.yml is the probe's own record of the - # run docs/occupancy-2026-10.adoc publishes. The schedules are the ones the - # .sir files in docs/occupancy state — serial for two orders, together for - # the third — and the peaks are the recorded ones: nothing here is invented. - orders = YAML.load_file(RECORDED_OCCUPANCY)["orders"] - # The interleavings the run used, from the same file the gate reads - # (scripts/check-occupancy-grades.jl), so the two cannot disagree. - schedules = YAML.load_file(RECORDED_SCHEDULES) - for (name, rec) in orders - windows = schedules[name] - tasks = [_occ_task(t["peak_rss_kib"]; label = t["label"], - first_sample = windows[i][1], last_sample = windows[i][2]) - for (i, t) in enumerate(rec["tasks"])] - occ = _occ_stage(tasks; measured = rec["child_peak_rss_kib"]) - graded = MG.graded(occ) - @test MG.predicted_peak_rss_kib(occ) == rec["child_peak_rss_kib"] - @test MG.disagreement(occ) === nothing - peaks = [t["peak_rss_kib"] for t in rec["tasks"]] - # Serial: the larger peak. Together: both resident, so nothing is reclaimed. - @test graded["reclaimed_kib"] == - (name == "concurrent" ? 0 : sum(peaks) - maximum(peaks)) - @test graded["cost_bound_kib"] == sum(peaks) - end - end -end diff --git a/test/unit/test_server_security.jl b/test/unit/test_server_security.jl index 9ef2fad6..2bb7086d 100644 --- a/test/unit/test_server_security.jl +++ b/test/unit/test_server_security.jl @@ -34,14 +34,6 @@ end SV.ServerState.set_root!(tmp) SV._bound_host[] = "127.0.0.1" SV._bound_port[] = 8080 - # The baseline assertions below hold only with no extra origins - # configured. JULIA_METAMANIFOLD_ALLOWED_ORIGINS is a supported - # deployment setting (split deployments, tunnels), and a developer - # running the suite with it set would otherwise fail them, so it is - # cleared here and restored in the finally. - saved_origins = get(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS", nothing) - haskey(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") && - delete!(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") mkpath(joinpath(tmp, "data", "StudyS", "run1")) touch(joinpath(tmp, "data", "StudyS", "run1", "s1_R1.fastq.gz")) mkpath(joinpath(tmp, "projects", "StudyS", "run1")) @@ -85,43 +77,6 @@ end @test _ss_request("GET", studies, ["Host" => "evil.example:8080"]).status == 403 end - @testset "Wildcard bind serves, and admits only, the reached IP" begin - studies = "/api/v1/studies" - SV._bound_host[] = "0.0.0.0" - lan = "Host" => "192.168.1.10:8080" - ui = "Origin" => "http://192.168.1.10:8080" - # The UI served at the machine's address: its origin is admitted, - # with CORS headers, a preflight answer, and writes that reach the - # route (this one fails in the route itself, not in the guard). - r = _ss_request("GET", studies, [lan, ui]) - @test r.status == 200 - @test SV.HTTP.header(r, "Access-Control-Allow-Origin") == "http://192.168.1.10:8080" - @test _ss_request("OPTIONS", studies, [lan, ui]).status == 204 - @test _ss_request("POST", "/api/v1/studies/StudyS/report?kind=bogus&ext=.txt", - [lan, ui, "Content-Type" => "text/plain"], "x").status == 400 - @test !isdir(joinpath(tmp, "projects", "StudyS", "report")) - # An IPv6 literal that matches its Host is admitted the same way. - @test _ss_request("GET", studies, - ["Host" => "[fd00::10]:8080", "Origin" => "http://[fd00::10]:8080"]).status == 200 - # Another address, another port, or another scheme: still refused. - for o in ("http://192.168.1.11:8080", "http://192.168.1.10:9090", - "https://192.168.1.10:8080") - @test _ss_request("GET", studies, [lan, "Origin" => o]).status == 403 - end - # A DNS-rebinding page: the domain matches its own Host, but a - # name is not an IP literal, so it is still refused. - @test _ss_request("GET", studies, - ["Host" => "evil.example:8080", "Origin" => "http://evil.example:8080"]).status == 403 - # A hostname the machine is reached by still has to be listed. - withenv("JULIA_METAMANIFOLD_ALLOWED_ORIGINS" => "http://lab-box:8080") do - @test _ss_request("GET", studies, - ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 200 - end - @test _ss_request("GET", studies, - ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 403 - SV._bound_host[] = "127.0.0.1" - end - @testset "Oxygen docs and metrics are not served" begin @test SV._SERVE_OPTIONS.docs === false @test SV._SERVE_OPTIONS.metrics === false @@ -166,8 +121,6 @@ end @test SV.HTTP.header(f, "Content-Security-Policy") == "" end finally - saved_origins === nothing || - (ENV["JULIA_METAMANIFOLD_ALLOWED_ORIGINS"] = saved_origins) SV.ServerState._root[] = old_root SV._bound_host[], SV._bound_port[] = old_host, old_port rm(tmp; recursive=true, force=true) From bafa9010e4f07e757231631ec96e1f31de330384 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:26:48 +0100 Subject: [PATCH 02/35] proofs: scope significance model as an experimental interface (#183) Recover the Agda interface and root import. Replace the overclaim that it models Julia representation with explicit limits: Julia can retain pairwise rows after omnibus failure; no verified refinement connects the representations. Documentation now calls it a proposed publication interface. Validation: axiom audit passes, all six modules reachable; whitespace check passes. Agda typechecking and Julia integration tests are pending. No claim of statistical validity or runtime enforcement. Next: specify the mapping and test publication paths. --- proofs/agda/MetaManifold/All.agda | 1 + proofs/agda/MetaManifold/Significance.agda | 80 ++++++++++++++++++++++ proofs/agda/README.md | 12 ++++ 3 files changed, 93 insertions(+) create mode 100644 proofs/agda/MetaManifold/Significance.agda diff --git a/proofs/agda/MetaManifold/All.agda b/proofs/agda/MetaManifold/All.agda index ff5c18c0..0770f154 100644 --- a/proofs/agda/MetaManifold/All.agda +++ b/proofs/agda/MetaManifold/All.agda @@ -13,3 +13,4 @@ import MetaManifold.Prelude import MetaManifold.BenjaminiHochberg import MetaManifold.Scaling import MetaManifold.Permanova +import MetaManifold.Significance diff --git a/proofs/agda/MetaManifold/Significance.agda b/proofs/agda/MetaManifold/Significance.agda new file mode 100644 index 00000000..d2341cc1 --- /dev/null +++ b/proofs/agda/MetaManifold/Significance.agda @@ -0,0 +1,80 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- A proposed publication interface, not a proof of the Julia representation. +-- This model stores numbers only in computed-result. Julia's AlphaSignificance +-- can retain pairwise rows after an omnibus failure, so the two representations +-- are not equivalent. No refinement mapping or extraction connects them. +-- The theorems below establish properties of this Agda interface only. They +-- establish neither statistical validity nor that Julia callers obey the gate. +-- A Julia integration needs explicit publication guards and regression tests. +-- Exact rationals and lists abstract Float64 p-values and DataFrame rows. + +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.Significance where + +open import Data.Bool using (Bool; true; false) +open import Data.List using (List; map) +open import Data.Rational using (ℚ) +open import Data.String using (String) +open import Relation.Nullary using (¬_) +open import Relation.Binary.PropositionalEquality using (_≡_; refl; cong) + +-- Julia's statuses: :computed, :r_unavailable, :no_paired_samples, :test_failed. +data Status : Set where + computed r-unavailable no-paired-samples test-failed : Status + +-- The discipline: the omnibus p-value and the pairwise p-values exist only +-- in the constructor that claims the test ran. A result that reveals nothing +-- names its status, carries a proof that the status is not `computed`, and a +-- reader-facing reason -- and no numbers, because there are none to read. +data AlphaSignificance : Set where + computed-result : (omnibus : ℚ) (pairwise : List ℚ) → AlphaSignificance + reveals-nothing : (status : Status) → ¬ (status ≡ computed) → String → AlphaSignificance + +-- The model's publication gate. +was-computed : AlphaSignificance → Bool +was-computed (computed-result _ _) = true +was-computed (reveals-nothing _ _ _) = false + +-- 1. Publishing the omnibus p-value requires evidence the test ran. On the +-- reveals-nothing branch the evidence is an inhabitant of `false ≡ true`, +-- which does not exist, so the branch cannot produce a number at all. +publish : (r : AlphaSignificance) → was-computed r ≡ true → ℚ +publish (computed-result omnibus _) refl = omnibus +publish (reveals-nothing _ _ _) () + +-- 2. The pairwise table, same discipline. +pairwise-published : (r : AlphaSignificance) → was-computed r ≡ true → List ℚ +pairwise-published (computed-result _ pairwise) refl = pairwise +pairwise-published (reveals-nothing _ _ _) () + +-- 3. The dual: the reader-facing reason can be read only from a result that +-- did not run. A computed result has no reason to show, and the evidence +-- it would need (`true ≡ false`) does not exist. +reason : (r : AlphaSignificance) → was-computed r ≡ false → String +reason (computed-result _ _) () +reason (reveals-nothing _ _ why) refl = why + +-- 4. The status of a result that did not run is never `computed`, by +-- construction: the proof obligation lives in the constructor, so a +-- `reveals-nothing` value cannot even be built with the `computed` +-- status. +unrun-status-not-computed : + (s : Status) (h : ¬ (s ≡ computed)) (why : String) → + let r = reveals-nothing s h why in was-computed r ≡ false +unrun-status-not-computed s h why = refl + +-- The status of a computed result is `computed`, and its evidence is the +-- constructor's own shape -- so the two constructors never meet: a result +-- is either readable as numbers or readable as a reason, never both. +computed-status : (o : ℚ) (p : List ℚ) → + was-computed (computed-result o p) ≡ true +computed-status o p = refl + +-- Map every pairwise p-value through a function without changing the +-- publish discipline: reading remains possible only on the computed branch. +map-pairwise : (f : ℚ → ℚ) (r : AlphaSignificance) → was-computed r ≡ true → List ℚ +map-pairwise f (computed-result _ pairwise) refl = map f pairwise +map-pairwise f (reveals-nothing _ _ _) () diff --git a/proofs/agda/README.md b/proofs/agda/README.md index a7d25cab..f055de52 100644 --- a/proofs/agda/README.md +++ b/proofs/agda/README.md @@ -189,3 +189,15 @@ returns. A concrete counterexample is `p = [0.01, 0.011]`, `n = 2`. Without the minimum the scaled values are `0.02` and `0.011`, which are not monotone. + +## Significance publication interface (experimental) + +`MetaManifold.Significance` models an interface that requires evidence of a +computed status before publishing numbers. Its noncomputed constructor stores +only a reason. This is a proposed interface, not a model equivalent to Julia's +`AlphaSignificance`: Julia may retain pairwise rows after omnibus failure. +There is no verified refinement or extracted implementation. These proofs alone +do not establish Julia reporting behavior or statistical validity. + +Before integration, specify the Julia-to-model mapping and test all publication +paths, including failed omnibus tests with retained pairwise results. From f151d290d4f35525ede863f4a145e7ef5431687b Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:27:30 +0100 Subject: [PATCH 03/35] fix(charts): isolate title markup stripping (#179) Recover the four-file title-text change from archived commit 1856722 without frontend dependency upgrades. Repeated stripping prevents nested tag fragments surviving a single pass at chart-title measurement and default-title sites. This is a text helper, not a general HTML sanitizer. Validation with Bun 1.4.3: frozen lockfile installation succeeded; `bun run typecheck` passed; `bun run test` passed all 42 tests across 7 files (86 assertions). Diff whitespace check passes. CI remains pending; Julia and R were not run. No dependency changes. Part of fork-work isolation after #178. Keep separate from the pending upstream repairs. --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- frontend/src/components/AnalysisChart.tsx | 8 +++++++- frontend/src/figure/charts.ts | 12 ++++++++++-- frontend/src/utils/text.test.ts | 24 +++++++++++++++++++++++ frontend/src/utils/text.ts | 19 ++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 frontend/src/utils/text.test.ts diff --git a/frontend/src/components/AnalysisChart.tsx b/frontend/src/components/AnalysisChart.tsx index 3e40f157..dcf1c775 100644 --- a/frontend/src/components/AnalysisChart.tsx +++ b/frontend/src/components/AnalysisChart.tsx @@ -1,13 +1,19 @@ // © 2026 Joshua Benjamin Jewell. All rights reserved. // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import Plotly from 'plotly.js-dist-min' +import { stripHtmlTags } from '../utils/text' import { AddToReport } from './AddToReport' import { PlotlyChart } from './PlotlyChart' +/** + * Read a Plotly title from `layout.title` or its `text` property and strip tags + * for the default report caption. A missing or empty title returns null; a + * title containing only tags returns an empty string. Whitespace is preserved. + */ const figureTitle = (fig: unknown): string | null => { const t = (fig as { layout?: { title?: unknown } })?.layout?.title const text = typeof t === 'string' ? t : (t as { text?: string } | undefined)?.text - return text ? text.replace(/<[^>]+>/g, '') : null + return text ? stripHtmlTags(text) : null } // The chart as standalone SVG, at a fixed print width. diff --git a/frontend/src/figure/charts.ts b/frontend/src/figure/charts.ts index 29845147..dcb58c29 100644 --- a/frontend/src/figure/charts.ts +++ b/frontend/src/figure/charts.ts @@ -2,6 +2,7 @@ // Licensed under the GNU Affero General Public License version 3 (AGPLv3). import { api } from '../api/client' import { extractAlphaPanel } from '../components/alphaMetrics' +import { stripHtmlTags } from '../utils/text' import { textWidth } from './legend' import { PX_PER_PT, tickPt, titlePt, type AlphaMetricKey, type ChartSpec, type FigureDoc, type FigurePane } from './types' @@ -101,7 +102,14 @@ function recolour(t: Record, colours: Record): return { ...t, marker: { ...marker, color: rgba(c, c[3]), ...(t.type === 'scatter' ? { line: outline } : {}) } } } -/** The chart sized, typeset and titled for its pane. */ +/** + * Return a JSON-cloned copy of the chart, sized, typeset and titled for its pane. + * A null pane title keeps the chart's title; an empty or whitespace-only title + * hides it. Title markup is retained for rendering but stripped for measuring + * the font size needed to fit the available width. + * Requires a browser document for text measurement. JSON serialisation, parsing + * and text measurement errors propagate to the caller. + */ export function paneFigure(raw: unknown, pane: FigurePane, doc: FigureDoc, opts: { showLegend: boolean widthPx: number @@ -119,7 +127,7 @@ export function paneFigure(raw: unknown, pane: FigurePane, doc: FigureDoc, opts: // A title wider than its pane shrinks to fit, clear of the group letter. const titleRoom = opts.widthPx - 2 * doc.letters.sizePt * PX_PER_PT const titleSize = Math.min(titlePt(doc) * PX_PER_PT, - titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(title.replace(/<[^>]+>/g, ''), titlePt(doc) * PX_PER_PT, family))) + titlePt(doc) * PX_PER_PT * titleRoom / Math.max(1, textWidth(stripHtmlTags(title), titlePt(doc) * PX_PER_PT, family))) for (const k of axisKeys(L, 'x').concat(axisKeys(L, 'y'))) { const ax = { ...(L[k] as Record) } diff --git a/frontend/src/utils/text.test.ts b/frontend/src/utils/text.test.ts new file mode 100644 index 00000000..f5ed71b9 --- /dev/null +++ b/frontend/src/utils/text.test.ts @@ -0,0 +1,24 @@ +// © 2026 Joshua Benjamin Jewell. All rights reserved. +// Licensed under the GNU Affero General Public License version 3 (AGPLv3). +import { describe, expect, test } from 'bun:test' +import { splitLines, stripHtmlTags } from './text' + +describe('stripHtmlTags', () => { + test('removes the markup Plotly allows in a title', () => { + expect(stripHtmlTags('CO2 by run')).toBe('CO2 by run') + }) + + test('a tag hiding inside another does not survive', () => { + expect(stripHtmlTags('ipt>alert(1)')).not.toMatch(/<[^>]+>/) + }) + + test('text without markup, including a bare >, is unchanged', () => { + expect(stripHtmlTags('p > 0.05 for all')).toBe('p > 0.05 for all') + }) +}) + +describe('splitLines', () => { + test('one trimmed value per line, blanks dropped', () => { + expect(splitLines(' a\n\n b \nc')).toEqual(['a', 'b', 'c']) + }) +}) diff --git a/frontend/src/utils/text.ts b/frontend/src/utils/text.ts index dc076655..fc28108a 100644 --- a/frontend/src/utils/text.ts +++ b/frontend/src/utils/text.ts @@ -9,3 +9,22 @@ export function splitLines(text: string): string[] { return text.split('\n').map(v => v.trim()).filter(v => v.length > 0) } + +/** + * Strip the pseudo-HTML markup Plotly accepts in titles and axis labels (, + *
, and friends), leaving the bare text for measuring or for a + * filename. A single pass is not enough: replacing "ipt>" once + * removes the inner tag and leaves the halves joined into another, so the + * strip repeats until the text stops changing. + * Entities are not decoded and removed tags add no spacing or line breaks. + * This is a text helper, not an HTML sanitiser. + */ +export function stripHtmlTags(text: string): string { + let stripped = text + let previous = '' + do { + previous = stripped + stripped = stripped.replace(/<[^>]+>/g, '') + } while (stripped !== previous) + return stripped +} From abf6f0d561b01eab59f00dec42a388dd16f67a71 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:28:37 +0100 Subject: [PATCH 04/35] feat(occupancy): isolate memory grading and proof integration (#184) Recover memory grading, unit/disagreement tests, proof scripts and occupancy documentation. Add a dedicated proof gate while preserving unrelated frontend and routing code. The historical records retain peak magnitudes and sample counts but no task windows. The empirical checker now reports INDETERMINATE (exit 2) for those records, rejects impossible windows, and never injects illustrative boundaries. Synthetic model-windows.yml and the Agda/Session IR examples state their assumptions explicitly; MMW-01.5 requires a real rerun retaining first_sample/last_sample before empirical correspondence can be claimed. The completed concurrent model releases its final 9176 KiB and has zero drift. Validation: four evidence-checker regressions and 129 occupancy assertions pass with native Julia 1.12.5. Two existing live-child /proc assertions fail in this sandbox because child PIDs are unavailable through mounted /proc; tests remain intact. The historical CLI returns exit 2 as intended. Agda and sibling proof integration have not run locally. Full fork CI remains blocked at startup by the repository Actions allowlist. Historical measurements are preserved, not newly reproduced. This is memory occupancy, not biological occupancy modelling. --- .github/workflows/ci.yml | 17 + .gitignore | 3 + docs/occupancy-2026-10.adoc | 168 +++++++++ docs/occupancy/GradedStages.agda | 114 +++++++ docs/occupancy/TypedPeak.agda | 49 +++ docs/occupancy/model-windows.yml | 13 + .../reject/ConcurrentDriftIsResident.agda | 10 + docs/occupancy/reject/ConcurrentIsSerial.agda | 14 + docs/occupancy/reject/SerialIsCost.agda | 14 + docs/occupancy/results-2026-10-14.yml | 63 ++++ docs/occupancy/vsearch-asv-then-otu.sir | 13 + docs/occupancy/vsearch-concurrent.sir | 12 + docs/occupancy/vsearch-otu-then-asv.sir | 9 + scripts/check-occupancy-grades.jl | 58 ++++ scripts/check-occupancy.sh | 101 ++++++ scripts/fetch-siblings.sh | 29 ++ scripts/lint-occupancy-readings.sh | 40 +++ src/core/memory_probe.jl | 240 ++++++++++++- test/occupancy_disagreement_test.jl | 19 ++ test/runtests.jl | 1 + test/unit/test_occupancy_grade.jl | 318 ++++++++++++++++++ 21 files changed, 1303 insertions(+), 2 deletions(-) create mode 100644 docs/occupancy-2026-10.adoc create mode 100644 docs/occupancy/GradedStages.agda create mode 100644 docs/occupancy/TypedPeak.agda create mode 100644 docs/occupancy/model-windows.yml create mode 100644 docs/occupancy/reject/ConcurrentDriftIsResident.agda create mode 100644 docs/occupancy/reject/ConcurrentIsSerial.agda create mode 100644 docs/occupancy/reject/SerialIsCost.agda create mode 100644 docs/occupancy/results-2026-10-14.yml create mode 100644 docs/occupancy/vsearch-asv-then-otu.sir create mode 100644 docs/occupancy/vsearch-concurrent.sir create mode 100644 docs/occupancy/vsearch-otu-then-asv.sir create mode 100755 scripts/check-occupancy-grades.jl create mode 100755 scripts/check-occupancy.sh create mode 100755 scripts/fetch-siblings.sh create mode 100755 scripts/lint-occupancy-readings.sh create mode 100644 test/unit/test_occupancy_grade.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bc3a6b2..df48fab7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -344,3 +344,20 @@ jobs: echo /usr/share/agda-stdlib/standard-library.agda-lib > "$AGDA_DIR/libraries" cd proofs/agda agda MetaManifold/All.agda + + occupancy: + name: Occupancy proof gate + runs-on: ubuntu-24.04 + container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Install proof tools + run: | + apt-get update -qq + apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 + - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 + with: + version: "1.12.5" + - name: Check Session IR and occupancy proofs + run: bash scripts/check-occupancy.sh diff --git a/.gitignore b/.gitignore index 6ef7e458..26faf843 100644 --- a/.gitignore +++ b/.gitignore @@ -335,3 +335,6 @@ dev/server.log # Agda interface files (proofs/agda) proofs/agda/_build/ + +# Occupancy proof dependencies and reproduction logs +/_build/ diff --git a/docs/occupancy-2026-10.adoc b/docs/occupancy-2026-10.adoc new file mode 100644 index 00000000..e1abbf98 --- /dev/null +++ b/docs/occupancy-2026-10.adoc @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) += Occupancy of the vsearch stage +:revdate: 2026-10-14 + +The vsearch stage runs two independent searches, ASV and OTU. The full +pipeline spawns them together; a single-stage re-run runs them one after the +other. The high-water-mark (HWM) algebra of occupancy-types predicts that the +stage's cost is the same in every order but its memory peak is not: serially +the peak is the larger search's, concurrently it can be their sum +(choreographic-types A-2 and A-3). This page records one measurement of that, +on 2026-10-14, and checks it against the typed reading. + +== What was measured + +`MemoryProbe` (`src/core/memory_probe.jl`), off unless a pipeline.yml sets +`occupancy_probe.enabled: true`, records for each attested stage: + +* `julia_allocated_bytes`: the `Base.gc_num()` delta over the stage (cost); +* per task (one spawned search): the largest sampled VmRSS of the child + process it started (`peak_rss_kib`), the kernel's own VmHWM for it + (`hwm_kib`), and the sample window its children were live in — `first_sample`, + the first sample taken after its first child started, and `last_sample`, the + last taken before its last was waited for, 0 for never — which its grade is + read from; +* `child_peak_rss_kib`: the largest sum of the stage's children's VmRSS at + one sampling instant (the stage's occupancy). + +The run: `test/occupancy_disagreement_test.jl`, which runs the same two searches +in three orders under the probe, on a fixture it generates: 1000 reference +sequences of 400 bp, 300 ASV and 150 OTU queries. Linux 6.18 with 4 CPUs, +Julia 1.12.5 with 2 threads, vsearch 2.30.5 with `--threads 1`. Sampling +interval 10 ms. + +== Results + +[cols="2,1,1,1,1,1,1",options="header"] +|=== +| Order | ASV peak (KiB) | OTU peak (KiB) | Stage peak, measured (KiB) | Counted bytes, Session IR | Typed peak, HWM | Julia allocation (bytes) +| ASV then OTU | 9204 | 9180 | 9204 | 9204 | 9204 | 978432 +| OTU then ASV | 9152 | 9192 | 9192 | 9192 | 9192 | 978720 +| Concurrent | 9176 | 9204 | 18380 | 18380 | 18380 | 978816 +|=== + +* *Cost commutes.* The allocation totals agree to 0.04%. The test asserts + agreement within 1%; a planted extra search in one order breaks it (spread + 25%). +* *Each search's peak is the same in every order*, to within 0.6% + (9152-9204 KiB). Each search's VmHWM equals its sampled peak, so the + sampling caught the peaks. +* *The stage peak is not the same.* Serially it is the larger search's + (9192-9204 KiB); concurrently it is 18380 KiB, twice that. This is the A-3 + order dependence, and nothing in the stage's outputs shows it: the hits are + byte-identical in all three orders. The prediction would have been falsified + had the concurrent peak matched the serial one; it did not. +* *Counted bytes equal the typed peak*, in all three orders. The counted bytes + come from occupancy-types' Session IR stepper running + `docs/occupancy/vsearch-*.sir`, one buffer per search. The typed peak is + computed, by `refl`, from occupancy-types' `Timeline` over resource-algebra's + HWM grade (`docs/occupancy/TypedPeak.agda`). A search alone exits holding + nothing (drift 0); in the concurrent order the first is still resident + when the second starts (drift = peak). The sequence law + `max (peak a) (drift a + peak b)` then gives the larger peak serially and the + sum concurrently. +* *The measured stage peak also equals the typed peak* here. For the concurrent + order the typed peak is the interleaving in which both are resident at once; + a measurement could come in lower if the two searches' peaks did not + coincide. Here they did: each search's memory plateaus once its database is + loaded. + +== Reading the numbers + +A recorded `peak_rss_kib` is a reading, not an occupancy: what a stage occupied +depends on what the next stage inherits from it. So the probe grades a stage's +whole window (`MemoryProbe`'s `task_grades`, `stage_grade`, +`predicted_peak_rss_kib`, `cost_bound_rss_kib`, `reclaimed_kib`) and attaches the +result to a recorded stage as `grades`; a record with no sample windows is left +ungraded rather than graded on an assumed order. Tasks are read in sampled order +(first sample, ties by record order). Each handover keeps only the tasks still +live at it, so a task's `drift_kib` is the change in the live peak sum across its +boundary — negative when the boundary also releases an earlier task — and +`release_kib` is everything freed between its start and the next boundary, which +can exceed that task's own peak. The folded drift is the sum of the peaks still +live when the stage ends. The pinned Agda instance has natural drift only; +its H-2 signed extension is a proposal, not an established general proof of +Julia's signed per-task deltas. GradedStages explicitly adds the concurrent +example's final delta of -9176 KiB and proves that completed drift is zero. + +The historical records retain peak magnitudes and sample counts, but no +`first_sample`/`last_sample` windows. Their empirical grade correspondence is +*indeterminate*. `scripts/check-occupancy-grades.jl` checks recorded windows only +and exits 2 for these incomplete records. A new probe run retaining real windows +is required; no windows can be reconstructed from peak magnitudes alone. + +`docs/occupancy/model-windows.yml` supplies explicitly synthetic boundaries for +unit examples. Combined with the historical peak magnitudes, the assumed serial +and concurrent interleavings produce 9204, 9192 and 18380 KiB, respectively. +Matching those numbers does not validate the historical trace or measure the +bytes reclaimed at a cut. The Agda examples (`GradedStages.agda`, `TypedPeak.agda`) +prove conditional algebraic statements at those magnitudes and assumed schedules. +They do not establish empirical correspondence. The completed concurrent example +adds a final release and has drift zero. + +`scripts/lint-occupancy-readings.sh` keeps production code from reading the key +directly: outside `src/core/memory_probe.jl`, an occupancy figure must go through +those helpers. + +== Status + +[cols="1,3,1,2",options="header"] +|=== +| ID | Claim | Status | Artefact +| MMW-01.1 | With the flag off, a stage record has exactly its earlier keys; with it on, it gains `occupancy`; old records still parse and merge | TESTED | `test/occupancy_disagreement_test.jl` ("flag off", "round trip", "through the pipeline's _attest") +| MMW-01.2 | For this workload, the vsearch stage's cost (Julia allocation) is the same in all three orders | TESTED | same file, "two-thread disagreement" (asserted, within 1%) +| MMW-01.3 | For this workload, the stage peak depends on the order (A-3), and each search's own peak does not | TESTED | same file, recorded in the test log; table above +| MMW-01.4 | For the three Session IR texts, the stepper's counted bytes equal the HWM typed peak | TESTED | `scripts/check-occupancy.sh` (stepper output against `TypedPeak.agda`; three rejections in `docs/occupancy/reject`) +| MMW-01.5 | Recorded windows support the graded prediction and reclamation interpretation | INDETERMINATE | Historical records lack windows; rerun required. Synthetic unit examples and Agda examples establish conditional model statements only. +| bridge | Counted bytes are bounded by the typed peak in general (occupancy-types' v0 → typed bridge) | OPEN | unchanged; this page gives three instances, not a proof +|=== + +== Limits + +* *Sampled.* A per-task peak is the largest VmRSS seen at 10 ms intervals. VmHWM + is exact up to the last sample, but growth after it is not seen. A child that + lives for less than one interval may not be sampled at all (`samples: 0`). +* *Child memory only.* The children are counted; the Julia process is not. Its + `julia_live_bytes_delta` is recorded, but without a forced collection it + reflects when the GC last ran, not what the stage kept. +* *Allocation is process-wide.* While the study loop runs several runs' + pipelines at once, a stage's `julia_allocated_bytes` includes the others' + allocations too. The child figures are per stage, since each stage samples + only the children it started. +* *Linux only*, since it reads /proc. For a stage offloaded with `remote:`, + the local child is an ssh client, so its memory says nothing about the work. +* *One thread for vsearch.* With several threads, vsearch writes its hits in + whatever order the threads finish. `_sort_hits_by_query!` then does a + different amount of work, because its sort key splits each line on every + comparison. That moved the stage's allocation total by up to 12% between + repeats of the same order. So in production the cost is order-independent + only up to that noise; precomputing the sort key would remove it. This PR + does not change it. +* *One host, one day.* CI runs the same test on every push and logs its own + numbers; they are not compared with these. + +== Reproducing + +---- +julia --project=. -t 2 test/occupancy_disagreement_test.jl # set OCCUPANCY_RESULTS= for YAML +bash scripts/check-occupancy.sh # needs julia and agda 2.6.4.x +julia --project=. scripts/check-occupancy-grades.jl # historical evidence check; currently exits 2 (missing windows) +bash scripts/lint-occupancy-readings.sh # no ungraded readings in src/ +---- + +`scripts/fetch-siblings.sh` fetches the siblings at their pins into +`_build/dependencies`: occupancy-types `af95728` and resource-algebra +`cf9235c`. + +== As the proofs progress + +* *Per-buffer (n-ary) occupancy* in occupancy-types: each search is already one + buffer in the Session IR texts, so the typed peak could then be read per + search. +* *Signed drift*: frees could be graded as negative drift, so the Julia side's + collections (the pipeline's `GC.gc(false)` hints) become modelled + reclamation rather than unmodelled noise. +* *choreographic-types' operational `step` as a function*: the three orders + become three traces of one typed protocol. The frontier certificate its + two-thread application note describes could then be displayed in the WebUI. diff --git a/docs/occupancy/GradedStages.agda b/docs/occupancy/GradedStages.agda new file mode 100644 index 00000000..b29e8aca --- /dev/null +++ b/docs/occupancy/GradedStages.agda @@ -0,0 +1,114 @@ +-- Model example: assumed interleavings, historical peak magnitudes only. +-- Historical sample windows were not retained; empirical correspondence is indeterminate. +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- The vsearch stage's grades, and occupancy-types' reclamation theorems at the +-- measured numbers. Fork issue #105 asks for exactly this: grade the stages +-- (alloc/hold/release) and compare the predicted peak with the measured one. +-- TypedPeak.agda computes the typed peak by the algebra's sequence law; this +-- module assigns conditional model grades, and applies the laws that say +-- what the difference between a peak and a cost reading is: occupancy≤cost, and +-- the strict reclamation at a cut where something is released. +-- +-- Consumed, never re-declared, from the siblings scripts/fetch-siblings.sh pins +-- and scripts/check-occupancy.sh checks: +-- occupancy-types af957280075699c659de0abd9a3b0defade77309 +-- resource-algebra cf9235c532712a783a404c359a1055f40134794a +-- Everything is stated in this file's own Timeline instance (`TH`, as +-- TypedPeak.agda does): the Grade-level laws are Reclaim's as they stand, and +-- the two protocol-level ones are Reclaim's own two-step — its `cut-factor` over +-- its `reclaim-law`/`occupancy≤cost` — applied to `xs` and `ys` of that instance. +-- +-- A grade is `hwm peak drift _`: what an interaction reached, and what it still +-- holds when the next one begins. The peaks below are the 2026-10-14 run's +-- (docs/occupancy-2026-10.adoc, docs/occupancy/results-2026-10-14.yml), in KiB. +module GradedStages where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import OccupancyTypes.Order +open import OccupancyTypes.Reclaim + +import OccupancyTypes.Timeline +module TH = OccupancyTypes.Timeline HWM + +-- A search that has not been waited for when the next one starts: it still holds +-- everything it reached, so the cut reclaims none of it. +still : Nat → Grade +still p = hwm p p (≤-refl p) + +-- A search that has exited before the next one starts: it holds nothing, so the +-- cut reclaims all of it. +gone : Nat → Grade +gone p = hwm p 0 z≤n + +-- This is the handover prefix, not the completed stage. ASV is still resident when +-- OTU starts, so the stage peak is their sum and nothing is reclaimed — +-- Reclaim.no-reclaim's equality, whose `refl` argument discharges the theorem's +-- own hypothesis (drift = peak) at the measured grade. +run-together : peak (still 9176 ⊗h gone 9204) ≡ 18380 +run-together = no-reclaim (still 9176) refl (gone 9204) + +-- A single-stage re-run runs them one after the other: ASV has exited, so the +-- cut reclaims its 9204 KiB and the stage peak is the larger search's +-- (Reclaim.reclaim-law, by `refl`). +run-in-turn : peak (gone 9204 ⊗h gone 9180) ≡ 9204 +run-in-turn = refl + +-- The same claim as Reclaim.reclaimed states it: with drift < peak at the cut +-- and a suffix that does anything, occupancy is strictly below cost. 9204 KiB +-- are released at the frontier, so the cost reading (18384 KiB) overstates the +-- stage. +run-in-turn +-- +-- The vsearch stage's typed peak, computed (TESTED, by refl) with +-- occupancy-types' Timeline over resource-algebra's HWM grade, both consumed +-- by pin through scripts/fetch-siblings.sh: +-- occupancy-types af957280075699c659de0abd9a3b0defade77309 +-- resource-algebra cf9235c532712a783a404c359a1055f40134794a +-- Nothing here re-declares their definitions. +-- +-- Each search is one interaction graded by its measured peak, in KiB, from the +-- 2026-10-14 run in docs/occupancy-2026-10.adoc. A search's process exits +-- holding nothing, so run alone its drift is 0. Run concurrently, the first is +-- still resident when the second starts, so that is graded with drift = peak. +-- This is an assumed interleaving, not a recovered historical sample trace. +-- The algebra's sequence law, max (peak a) (drift a + peak b), then gives: +-- serial = the larger peak; concurrent = the sum. These are peak claims. +-- The concurrent timeline is a handover prefix; GradedStages explicitly +-- closes its signed final boundary before stating the completed-stage drift. +module TypedPeak where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM + +import OccupancyTypes.Timeline +module TH = OccupancyTypes.Timeline HWM + +-- A search whose process has exited before the next one starts. +exits : Nat → Grade +exits p = hwm p 0 z≤n + +-- A search still resident when the next one starts. +resident : Nat → Grade +resident p = hwm p p (≤-refl p) + +-- ASV 9204 then OTU 9180. +asv-then-otu : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 9204 +asv-then-otu = refl + +-- OTU 9192 then ASV 9152. +otu-then-asv : peak (TH.grade (exits 9192 TH.∷ exits 9152 TH.∷ TH.[])) ≡ 9192 +otu-then-asv = refl + +-- ASV 9176 and OTU 9204 together. +concurrent : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 18380 +concurrent = refl diff --git a/docs/occupancy/model-windows.yml b/docs/occupancy/model-windows.yml new file mode 100644 index 00000000..245da598 --- /dev/null +++ b/docs/occupancy/model-windows.yml @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Synthetic model boundaries, NOT probe sample indices or historical evidence. +# Peak magnitudes come from results-2026-10-14.yml; these assumed interleavings +# illustrate the algebra only. They cannot establish empirical correspondence. +asv_then_otu: + - [1, 2] + - [3, 4] +otu_then_asv: + - [1, 2] + - [3, 4] +concurrent: + - [1, 4] + - [2, 5] diff --git a/docs/occupancy/reject/ConcurrentDriftIsResident.agda b/docs/occupancy/reject/ConcurrentDriftIsResident.agda new file mode 100644 index 00000000..65cd5bd1 --- /dev/null +++ b/docs/occupancy/reject/ConcurrentDriftIsResident.agda @@ -0,0 +1,10 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- A completed stage has released both tasks; retaining ASV is the wrong claim. +module ConcurrentDriftIsResident where +open import ResourceAlgebra.Prelude +open import GradedStages +import Data.Integer as Int +wrong : concurrent-completed-drift ≡ Int.+ 9176 +wrong = refl diff --git a/docs/occupancy/reject/ConcurrentIsSerial.agda b/docs/occupancy/reject/ConcurrentIsSerial.agda new file mode 100644 index 00000000..592de6ee --- /dev/null +++ b/docs/occupancy/reject/ConcurrentIsSerial.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: the claim that running the two searches concurrently leaves +-- the peak where a serial run puts it (the larger of the two). +module ConcurrentIsSerial where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (resident 9176 TH.∷ exits 9204 TH.∷ TH.[])) ≡ 9204 +wrong = refl diff --git a/docs/occupancy/reject/SerialIsCost.agda b/docs/occupancy/reject/SerialIsCost.agda new file mode 100644 index 00000000..dbb6f4ba --- /dev/null +++ b/docs/occupancy/reject/SerialIsCost.agda @@ -0,0 +1,14 @@ +{-# OPTIONS --safe --without-K #-} +-- SPDX-License-Identifier: AGPL-3.0-only +-- SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Must be REFUSED: reading a serial run's peak as the sum of the two peaks, +-- i.e. occupancy as cost (occupancy-types R-6). +module SerialIsCost where + +open import ResourceAlgebra.Prelude +open import ResourceAlgebra.Instances.HWM +open import TypedPeak + +wrong : peak (TH.grade (exits 9204 TH.∷ exits 9180 TH.∷ TH.[])) ≡ 18384 +wrong = refl diff --git a/docs/occupancy/results-2026-10-14.yml b/docs/occupancy/results-2026-10-14.yml new file mode 100644 index 00000000..394236d6 --- /dev/null +++ b/docs/occupancy/results-2026-10-14.yml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# The raw MemoryProbe records behind docs/occupancy-2026-10.adoc, written by +# test/occupancy_disagreement_test.jl with OCCUPANCY_RESULTS set (2026-10-14). +vsearch: "vsearch v2.30.5_linux_x86_64, 15.7GB RAM, 4 cores" +julia: "1.12.5" +threads: 2 +cpus: 4 +cost_spread: 0.00039231071008238524 +orders: + asv_then_otu: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 56 + julia_allocated_bytes: 978432 + julia_live_bytes_delta: 978544 + child_peak_rss_kib: 9204 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9204 + hwm_kib: 9204 + - label: "vsearch:otu" + processes: 1 + samples: 20 + peak_rss_kib: 9180 + hwm_kib: 9180 + otu_then_asv: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 57 + julia_allocated_bytes: 978720 + julia_live_bytes_delta: 978832 + child_peak_rss_kib: 9192 + tasks: + - label: "vsearch:otu" + processes: 1 + samples: 22 + peak_rss_kib: 9192 + hwm_kib: 9192 + - label: "vsearch:asv" + processes: 1 + samples: 33 + peak_rss_kib: 9152 + hwm_kib: 9152 + concurrent: + probe: "MemoryProbe 1" + interval_ms: 10 + samples: 33 + julia_allocated_bytes: 978816 + julia_live_bytes_delta: 978928 + child_peak_rss_kib: 18380 + tasks: + - label: "vsearch:asv" + processes: 1 + samples: 30 + peak_rss_kib: 9176 + hwm_kib: 9176 + - label: "vsearch:otu" + processes: 1 + samples: 24 + peak_rss_kib: 9204 + hwm_kib: 9204 diff --git a/docs/occupancy/vsearch-asv-then-otu.sir b/docs/occupancy/vsearch-asv-then-otu.sir new file mode 100644 index 00000000..e462fb5e --- /dev/null +++ b/docs/occupancy/vsearch-asv-then-otu.sir @@ -0,0 +1,13 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-asv-then-otu.sir - the vsearch stage with the two searches run one + after the other, ASV first: the single-stage re-run's order. Each search is + one buffer, sized by its measured peak in KiB (docs/occupancy-2026-10.adoc, + 2026-10-14 run); each process exits holding nothing, so its buffer is dropped + before the next search starts. + + Expected: steps = 0 (no communication), peak_live = max 9204 9180 = 9204. *) +let asv = alloc 9204 in +let _ = drop asv in +let otu = alloc 9180 in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-concurrent.sir b/docs/occupancy/vsearch-concurrent.sir new file mode 100644 index 00000000..26cebe5d --- /dev/null +++ b/docs/occupancy/vsearch-concurrent.sir @@ -0,0 +1,12 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-concurrent.sir - the two searches spawned together, as the full + pipeline runs them. Session IR has no parallel composition, so this is the + assumed model interleaving: both buffers live at once, then both + dropped. + + Expected: steps = 0, peak_live = 9176 + 9204 = 18380. *) +let asv = alloc 9176 in +let otu = alloc 9204 in +let _ = drop asv in +let _ = drop otu in +unit diff --git a/docs/occupancy/vsearch-otu-then-asv.sir b/docs/occupancy/vsearch-otu-then-asv.sir new file mode 100644 index 00000000..ba575591 --- /dev/null +++ b/docs/occupancy/vsearch-otu-then-asv.sir @@ -0,0 +1,9 @@ +(* SPDX-License-Identifier: AGPL-3.0-only *) +(* vsearch-otu-then-asv.sir - the same two searches, OTU first. + + Expected: steps = 0, peak_live = max 9192 9152 = 9192. *) +let otu = alloc 9192 in +let _ = drop otu in +let asv = alloc 9152 in +let _ = drop asv in +unit diff --git a/scripts/check-occupancy-grades.jl b/scripts/check-occupancy-grades.jl new file mode 100755 index 00000000..ac443134 --- /dev/null +++ b/scripts/check-occupancy-grades.jl @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# Validate empirical records only; never supply missing windows from a model. +# Exit 0: comparisons passed; 1: malformed evidence/disagreement; 2: incomplete evidence. +# Reuse the package binding when included by its registered unit tests. +if !isdefined(@__MODULE__, :MemoryProbe) + include(joinpath(@__DIR__, "..", "src", "core", "memory_probe.jl")) +end +using .MemoryProbe +using YAML + +function check_records(orders; io=stdout) + failures = 0 + incomplete = 0 + for name in sort(collect(keys(orders))) + rec = orders[name] + tasks = rec["tasks"] + if isempty(tasks) || any(t -> !haskey(t, "first_sample") || !haskey(t, "last_sample"), tasks) + println(io, "INDETERMINATE: $name: no recorded task sample windows; rerun required") + incomplete += 1 + continue + end + valid = all(tasks) do t + first, last, samples = t["first_sample"], t["last_sample"], t["samples"] + all(x -> x isa Integer && !(x isa Bool), (first, last, samples)) && + ((first == last == samples == 0) || + (first > 0 && last >= first && samples >= 0 && + samples <= last - first + 1 && last <= rec["samples"])) + end + if !valid + println(io, "FAIL: $name: inconsistent or unclosed recorded sample window") + failures += 1 + continue + end + if any(t -> t["samples"] == 0, tasks) + println(io, "INDETERMINATE: $name: a child was not sampled; no empirical bound established") + incomplete += 1 + continue + end + predicted = MemoryProbe.predicted_peak_rss_kib(rec) + cost = MemoryProbe.cost_bound_rss_kib(rec) + reclaimed = MemoryProbe.reclaimed_kib(rec) + grade = MemoryProbe.stage_grade(rec) + if rec["child_peak_rss_kib"] > predicted || predicted > cost || reclaimed < 0 || grade.drift != 0 + println(io, "FAIL: $name: model disagrees with recorded evidence") + failures += 1 + else + println(io, "PASS: $name: measured peak is within the conditional model bound") + end + end + isempty(orders) && (incomplete += 1; println(io, "INDETERMINATE: no orders recorded")) + failures > 0 ? 1 : incomplete > 0 ? 2 : 0 +end + +if abspath(PROGRAM_FILE) == @__FILE__ + results = isempty(ARGS) ? joinpath(@__DIR__, "..", "docs", "occupancy", "results-2026-10-14.yml") : only(ARGS) + exit(check_records(YAML.load_file(results)["orders"])) +end diff --git a/scripts/check-occupancy.sh b/scripts/check-occupancy.sh new file mode 100755 index 00000000..185934a6 --- /dev/null +++ b/scripts/check-occupancy.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Gate for the occupancy reading of the vsearch stage (docs/occupancy-2026-10.adoc): +# 1. each Session IR text checks, and occupancy-types' stepper counts its bytes; +# 2. TypedPeak.agda and GradedStages.agda type-check against the pinned +# siblings, and each module in docs/occupancy/reject is refused at its own +# declaration; +# 3. the counted bytes equal the typed peak, order by order. +# Needs julia and agda (2.6.4.x) on PATH. Exit 0 only if all three hold. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +export LC_ALL="${LC_ALL:-C.UTF-8}" + +bash scripts/fetch-siblings.sh +deps=_build/dependencies +sir="$deps/occupancy-types/src/julia" +agda --version | head -1 +flags=(--safe --without-K --no-libraries --ignore-interfaces --double-check + -i docs/occupancy -i /usr/share/agda-stdlib -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") +logs=_build/check-logs +mkdir -p "$logs" +failures=0 + +# GitHub renders an `::error` line as a check annotation, and annotations -- unlike +# job logs -- can be read back through the API. A failure that only happens in CI +# must be readable from a machine that cannot download the log, so every failure +# below says what it was and repeats the prover's own last lines: % escapes to +# %25 and newlines to %0A, as the workflow-command syntax requires. +annotate() { # title file log + [ -n "${GITHUB_ACTIONS:-}" ] || return 0 + printf '::error file=%s,title=%s::%s\n' "$2" "$1" \ + "$(awk 'NR > 1 { printf "%%0A" } { gsub(/%/, "%25"); printf "%s", $0 }' "$3" | + tail -c 6000)" +} + +# One Agda module that must type-check, with its own log. +check_agda() { # file label + if ! agda "${flags[@]}" "$1" >"$logs/$2.log" 2>&1; then + cat "$logs/$2.log" + annotate "Agda refused $2" "$1" "$logs/$2.log" + echo "ERROR: $1 does not type-check" >&2 + exit 1 + fi + echo "PASS: $2" +} + +# Order -> Session IR file and TypedPeak lemma. +orders=(asv-then-otu otu-then-asv concurrent) + +echo "== Session IR (occupancy-types v0 stepper)" +declare -A counted +for o in "${orders[@]}"; do + file="docs/occupancy/vsearch-$o.sir" + julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" check "$file" + out="$(julia --startup-file=no --project="$sir" "$sir/bin/sessionir.jl" run "$file" --trace)" + echo "$out" + counted[$o]="$(sed -n 's/.*peak_live=\([0-9]*\).*/\1/p' <<<"$out" | head -1)" +done + +echo "== Typed peak (Agda, HWM Timeline)" +check_agda docs/occupancy/TypedPeak.agda TypedPeak.agda + +# Conditional algebra examples at historical peak magnitudes, with assumed +# interleavings. This proof gate does not validate the missing historical windows. +check_agda docs/occupancy/GradedStages.agda GradedStages.agda +for file in docs/occupancy/reject/*.agda; do + name="$(basename -- "$file" .agda)" + log="$logs/$name.log" + if agda "${flags[@]}" -i docs/occupancy/reject "$file" >"$log" 2>&1; then + cat "$log" + annotate "Invalid claim accepted: $name" "$file" "$log" + echo "ERROR: invalid claim accepted: $name" >&2; failures=$((failures + 1)); continue + fi + if ! grep -Eq "/$name\.agda:[0-9]+,[0-9]+" "$log" || ! grep -q 'when checking that' "$log" || + grep -Eq 'Not in scope|Failed to find|Unsolved|Parse error' "$log"; then + cat "$log" + annotate "$name failed for the wrong reason" "$file" "$log" + echo "ERROR: $name failed for the wrong reason" >&2; failures=$((failures + 1)); continue + fi + echo "PASS: rejected $name ($(grep -m1 -E '[0-9]+ != [0-9]+' "$log"))" +done + +echo "== Counted bytes against typed peak (KiB)" +printf '%-14s %10s %10s\n' order counted typed +for o in "${orders[@]}"; do + typed="$(grep -A1 -E "^$o :" docs/occupancy/TypedPeak.agda | sed -n 's/.*≡ \([0-9]*\)$/\1/p' | head -1)" + printf '%-14s %10s %10s\n' "$o" "${counted[$o]}" "$typed" + if [ -z "${counted[$o]}" ] || [ -z "$typed" ] || [ "${counted[$o]}" != "$typed" ]; then + echo "ERROR: $o: counted ${counted[$o]:-?} != typed ${typed:-?}" >&2; failures=$((failures + 1)) + echo "counted ${counted[$o]:-?} KiB != typed ${typed:-?} KiB" >"$logs/counted-$o.log" + annotate "Counted bytes != typed peak ($o)" docs/occupancy/TypedPeak.agda "$logs/counted-$o.log" + fi +done + +if (( failures > 0 )); then + echo "check-occupancy: $failures failure(s)" >&2 + exit 1 +fi +echo "check-occupancy: OK" diff --git a/scripts/fetch-siblings.sh b/scripts/fetch-siblings.sh new file mode 100755 index 00000000..31a98bc1 --- /dev/null +++ b/scripts/fetch-siblings.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fetch the proof siblings the occupancy reading (docs/occupancy-2026-10.adoc) +# consumes, each at its pinned commit, into _build/dependencies. Their +# definitions are used, never copied. Re-pin one sibling per commit. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +# occupancy-types: the HWM Timeline (Agda) and the SessionIR stepper (Julia). +OCCUPANCY_TYPES_COMMIT="${OCCUPANCY_TYPES_COMMIT:-af957280075699c659de0abd9a3b0defade77309}" +# resource-algebra: the HWM grade algebra occupancy-types builds on. +RESOURCE_ALGEBRA_COMMIT="${RESOURCE_ALGEBRA_COMMIT:-cf9235c532712a783a404c359a1055f40134794a}" + +fetch() { + local name="$1" commit="$2" dir="_build/dependencies/$1" + if [ ! -d "$dir/.git" ]; then + git init --quiet "$dir" + git -C "$dir" remote add origin "https://github.com/hyperpolymath/$name.git" + fi + git -C "$dir" fetch --quiet --depth 1 origin "$commit" + git -C "$dir" checkout --quiet --detach "$commit" + echo "$name at $(git -C "$dir" rev-parse --short HEAD)" +} + +mkdir -p _build/dependencies +fetch occupancy-types "$OCCUPANCY_TYPES_COMMIT" +fetch resource-algebra "$RESOURCE_ALGEBRA_COMMIT" diff --git a/scripts/lint-occupancy-readings.sh b/scripts/lint-occupancy-readings.sh new file mode 100755 index 00000000..114779a0 --- /dev/null +++ b/scripts/lint-occupancy-readings.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fork issue #105: a task's recorded `peak_rss_kib` is a reading, not an +# occupancy. What a stage occupied is determined only by what the next stage +# inherits from it — the drift — which is why the probe grades a whole window and +# why src/core/memory_probe.jl's helpers (`graded`, `predicted_peak_rss_kib`, +# `cost_bound_rss_kib`, `reclaimed_kib`, `task_grades`) exist. Code outside the +# probe that reaches for the key itself is comparing measured figures without +# that context; on the recorded run that reading is off by 9180 KiB for +# `asv_then_otu` and 9204 KiB for `otu_then_asv` (the reclaimed suffix). +# +# So: production code may name the key only in src/core/memory_probe.jl, where +# the record is built and graded. Tests may read it (they assert the recorded +# numbers directly, which is fine — they are evidence, not presentation). +# +# Run from the repository root: bash scripts/lint-occupancy-readings.sh +set -u -o pipefail + +if [[ ! -f src/core/memory_probe.jl ]]; then + echo "lint-occupancy-readings: run from the repository root" >&2 + exit 2 +fi + +failures=0 +while IFS= read -r hit; do + [[ -z "$hit" ]] && continue + echo "ERROR: ungraded occupancy reading: $hit" >&2 + failures=$((failures + 1)) +done < <(grep -rn --include='*.jl' '"peak_rss_kib"' src/ | + grep -v '^src/core/memory_probe\.jl:' || true) + +if (( failures > 0 )); then + echo "lint-occupancy-readings: $failures ungraded reading(s); use the grading helpers in" >&2 + echo "MemoryProbe (see docs/occupancy-2026-10.adoc, 'Reading the numbers') instead." >&2 + exit 1 +fi + +echo "lint-occupancy-readings: OK — every peak_rss_kib reading in src/ goes through the probe's grades" diff --git a/src/core/memory_probe.jl b/src/core/memory_probe.jl index 6406c37a..e18a7b53 100644 --- a/src/core/memory_probe.jl +++ b/src/core/memory_probe.jl @@ -50,8 +50,16 @@ mutable struct TaskTrace samples :: Int # samples that found at least one live process peak_kib :: Int hwm_kib :: Int + # The sample window the task's children were live in: the index of the first + # sample taken after the first of them started, and the last taken before + # the last was waited for. 0 means never. Two tasks overlap when the later + # one's `first_sample` falls at or before the earlier one's `last_sample`, + # which is the interleaving the grade algebra sequences over. + first_sample :: Int + last_sample :: Int end -TaskTrace(label::AbstractString) = TaskTrace(String(label), Int[], 0, 0, 0, 0) +TaskTrace(label::AbstractString) = + TaskTrace(String(label), Int[], 0, 0, 0, 0, 0, 0) mutable struct Probe interval :: Float64 @@ -269,7 +277,10 @@ function watch!(proc::Base.Process) end lock(p.lock) do t.processes += 1 - pid === nothing || push!(t.live, pid) + if pid !== nothing + t.first_sample == 0 && (t.first_sample = p.samples + 1) + push!(t.live, pid) + end end pid === nothing || _sample!(p) pid @@ -301,16 +312,236 @@ function unwatch!(pid::Int) p === nothing && return nothing lock(p.lock) do filter!(!=(pid), t.live) + isempty(t.live) && (t.last_sample = p.samples) + end + nothing +end + +## Grading a stage against the high-water-mark algebra +# `docs/occupancy` reads a stage as a timeline of grades and checks the peak the +# algebra predicts against the one the probe measured. The grade is the pinned +# HWM instance's (`occupancy-types`' sibling `resource-algebra`, +# `ResourceAlgebra.Instances.HWM`; both are fetched by `scripts/fetch-siblings.sh`): +# +# hwm peak drift with drift ≤ peak +# +# * `peak` — the high-water mark this interaction reached, in KiB; +# * `drift` — the change in the live peak sum across the interaction's +# boundary, so a boundary that also releases an earlier task is negative; +# * `release` = peak − drift — everything freed between the interaction's start +# and the next boundary, which can exceed the interaction's own peak. +# +# A task's drift subtracts earlier tasks that exited before the next start, so it +# can be negative. The pinned Agda HWM instance has natural drift only; +# this Julia extension is not covered by its general algebra proofs. The +# concrete final boundary is checked separately in GradedStages.agda. +# +# A task's drift subtracts earlier tasks that exited before the next start, so +# it can be negative. The folded drift is the sum of peaks still live there. +# +# Sequencing starts the next interaction from the previous one's drift: +# +# hwm p₁ n₁ ⊗ hwm p₂ n₂ = hwm (max p₁ (n₁ + p₂)) (n₁ + n₂) +# +# and `OccupancyTypes.Reclaim` proves of that definition +# +# peak (a ⊗ b) = max (peak a) (drift a + peak b) (reclaim-law) +# peak (a ⊗ b) ≤ peak a + peak b (occupancy≤cost) +# drift a = peak a ⇒ peak (a ⊗ c) = peak a + peak c (no-reclaim) +# drift a < peak a, 0 < peak c ⇒ peak (a ⊗ c) < peak a + peak c (reclaimed) +# +# The last two are the two shapes a stage actually has. Run together, the first +# task is still resident when the second starts (drift = peak) and the stage peak +# is the sum; run one after the other it has exited (drift = 0) and the stage peak +# is the larger. The samples already tell the two apart, so the drift is read from +# the task windows and never assumed. +# +# The prediction is an upper bound on the measurement: at a handover the task +# still live holds at most its peak, so the drift the algebra adds is at least +# what a sample could have seen. `disagreement` therefore reports only what that +# bound forbids — a measured stage peak above the prediction — which is the "one +# of them is a bug" the occupancy reading asks about. + +const GRADE_ALGEBRA = "ResourceAlgebra.Instances.HWM" + +"A grade of the pinned high-water-mark algebra: a peak and a net memory change." +struct Grade + peak :: Int + drift :: Int + function Grade(peak::Integer, drift::Integer) + drift <= peak || throw(ArgumentError( + "a grade holds at most what it reached: drift $drift > peak $peak")) + new(Int(peak), Int(drift)) + end +end + +"`release(grade)` — `peak − drift`: everything freed by the interaction's boundary." +release(g::Grade) = g.peak - g.drift + +"`seq(a, b)` — sequence `b` after `a`: the suffix starts from `a`'s drift." +seq(a::Grade, b::Grade) = Grade(max(a.peak, a.drift + b.peak), a.drift + b.drift) + +"`alt(a, b)` — the larger grade, componentwise (what the timeline never needs, kept for the pair of laws)." +alt(a::Grade, b::Grade) = Grade(max(a.peak, b.peak), max(a.drift, b.drift)) + +"Nothing allocated and nothing held: the algebra's unit." +const NO_GRADE = Grade(0, 0) + +# The trace's sample window, or `nothing` for a record that carries none (one +# written before the windows were sampled). +function _window(rec::AbstractDict) + first_seen = get(rec, "first_sample", nothing) + last_seen = get(rec, "last_sample", nothing) + (first_seen isa Integer && last_seen isa Integer) ? (Int(first_seen), Int(last_seen)) : nothing +end + +_is_windowed(rec::AbstractDict) = _window(rec) !== nothing + +# Whether `prev` was still live when `next` started, so the handover hands its +# peak on rather than reclaiming it. A started `prev` never unwatched (window +# end 0) was still live; a task that never started (window start 0) never overlapped. +_overlaps(prev::Tuple{Int,Int}, next::Tuple{Int,Int}) = + prev[1] != 0 && next[1] != 0 && (prev[2] == 0 || next[1] <= prev[2]) + +# Grades and labels in sampled order, or `nothing` when the record carries no +# windows: a comparison is then skipped rather than made on an assumed order. +function _grades_and_labels(occ::AbstractDict) + recs = get(occ, "tasks", nothing) + recs isa AbstractVector || return Grade[], String[] + all(_is_windowed, recs) || return nothing + # Sampled order, ties broken by record order so the result never depends on + # the sort's stability. + order = sort(collect(eachindex(recs)); by = i -> (_window(recs[i])[1], i)) + ordered = collect(recs)[order] + windows = [_window(r) for r in ordered] + peaks = [Int(get(r, "peak_rss_kib", 0)) for r in ordered] + grades = Grade[] + labels = String[] + live = Int[] + held = 0 + for (i, rec) in enumerate(ordered) + push!(live, i) + if i == length(ordered) + # The stage waits for its children: a task still resident at the end + # keeps its peak in the folded drift, an exited one holds nothing. + filter!(j -> windows[j][1] != 0 && windows[j][2] == 0, live) + else + filter!(j -> _overlaps(windows[j], windows[i + 1]), live) + end + remaining = sum(j -> peaks[j], live; init = 0) + # seq adds drifts, so reclaim expired predecessors as well as this task. + push!(grades, Grade(peaks[i], remaining - held)) + held = remaining + push!(labels, String(get(rec, "label", "?"))) end + grades, labels +end + +""" + task_grades(occupancy) -> Union{Vector{Grade},Nothing} + +One grade per task of a stage's `occupancy` record, in sampled order. Its drift +is the change in the live peak sum by the next task's start, including releases +of earlier tasks; the final boundary retains only unclosed windows. Folding +with `seq` carries exactly the peaks still live at each boundary. `nothing` when +the record carries no sample windows. +""" +function task_grades(occ::AbstractDict) + graded = _grades_and_labels(occ) + graded === nothing ? nothing : graded[1] +end + +""" + stage_grade(occupancy) -> Union{Grade,Nothing} + +The stage's grade: its tasks' grades folded with `seq`, the algebra's sequencing. +`nothing` when the record carries no sample windows. +""" +function stage_grade(occ::AbstractDict) + grades = task_grades(occ) + grades === nothing && return nothing + isempty(grades) ? NO_GRADE : foldl(seq, grades) +end + +"`predicted_peak_rss_kib(occupancy)` — the peak the algebra predicts, or `nothing`." +function predicted_peak_rss_kib(occ::AbstractDict) + g = stage_grade(occ) + g === nothing ? nothing : g.peak +end + +"`cost_bound_rss_kib(occupancy)` — the peaks added: the peak if no cut reclaimed anything (occupancy≤cost's right-hand side)." +function cost_bound_rss_kib(occ::AbstractDict) + grades = task_grades(occ) + grades === nothing ? nothing : sum(g -> g.peak, grades; init = 0) +end + +"`reclaimed_kib(occupancy)` — the slack between the cost bound and the predicted peak: what the cuts reclaim." +function reclaimed_kib(occ::AbstractDict) + cost = cost_bound_rss_kib(occ) + predicted = predicted_peak_rss_kib(occ) + (cost === nothing || predicted === nothing) ? nothing : cost - predicted +end + +""" + disagreement(occupancy) -> Union{String,Nothing} + +Why the measured stage peak contradicts the grade algebra, or `nothing` when it +does not. The prediction is an upper bound, so the one contradiction is a +measured peak above it: a sample then saw more resident memory than the timed +timeline of tasks can hold, and one of the two is wrong. A record with no +windows, or none the probe graded, gives `nothing`: not measured is not a +disagreement. +""" +function disagreement(occ::AbstractDict) + predicted = predicted_peak_rss_kib(occ) + measured = get(occ, "child_peak_rss_kib", nothing) + (predicted === nothing || !(measured isa Integer)) && return nothing + measured > predicted && return "measured $(measured) KiB above the predicted $(predicted) KiB" nothing end +""" + graded(occupancy) -> Union{OrderedDict,Nothing} + +The grades of a stage's `occupancy` record, ready to sit beside it in the +attestation: per task `peak_kib`/`drift_kib`/`release_kib`, the stage's +`predicted_peak_kib`, the `cost_bound_kib` (its peaks added) and the +`reclaimed_kib` slack between the two, the `measured_peak_kib` it is checked +against, and the `disagreement` between them or `nothing`. `nothing` when the +record carries no sample windows. +""" +function graded(occ::AbstractDict) + graded_pair = _grades_and_labels(occ) + graded_pair === nothing && return nothing + grades, labels = graded_pair + stage = isempty(grades) ? NO_GRADE : foldl(seq, grades) + cost = sum(g -> g.peak, grades; init = 0) + OrderedDict{String,Any}( + "algebra" => GRADE_ALGEBRA, + "tasks" => Any[OrderedDict{String,Any}( + "label" => labels[i], + "peak_kib" => grades[i].peak, + "drift_kib" => grades[i].drift, + "release_kib" => release(grades[i])) + for i in eachindex(grades)], + "predicted_peak_kib" => stage.peak, + "drift_kib" => stage.drift, + "release_kib" => release(stage), + "cost_bound_kib" => cost, + "reclaimed_kib" => cost - stage.peak, + "measured_peak_kib" => get(occ, "child_peak_rss_kib", nothing), + "disagreement" => disagreement(occ), + ) +end + _task_record(t::TaskTrace) = OrderedDict{String,Any}( "label" => t.label, "processes" => t.processes, "samples" => t.samples, "peak_rss_kib" => t.peak_kib, "hwm_kib" => t.hwm_kib, + "first_sample" => t.first_sample, + "last_sample" => t.last_sample, ) """ @@ -337,6 +568,11 @@ function finish!(p::Probe) "tasks" => Any[_task_record(t) for t in traces], ) p.error === nothing || (p.record["error"] = p.error) + # The grades the sample windows imply, beside the samples they came from. A + # record whose tasks carry no windows cannot be graded, and is left as it is + # rather than graded on an assumed order. + g = graded(p.record) + g === nothing || (p.record["grades"] = g) p.record end diff --git a/test/occupancy_disagreement_test.jl b/test/occupancy_disagreement_test.jl index 00837263..15df70fd 100644 --- a/test/occupancy_disagreement_test.jl +++ b/test/occupancy_disagreement_test.jl @@ -341,6 +341,25 @@ _task(rec, label) = only(t for t in rec["tasks"] if t["label"] == label) @info "occupancy: concurrent stage peak vs the larger serial stage peak" concurrent = conc["child_peak_rss_kib"] serial = serial_max overlapped = overlap cost_spread = round(spread; sigdigits = 3) overlap || @warn "occupancy: the concurrent searches did not overlap in memory on this runner, so this run cannot show the order dependence" + # The grades the sample windows imply, on this same run + # (fork issue #105). The peak is recorded, not asserted — a + # runner that never has both searches resident cannot show it — + # but the algebra's own claims are asserted of every order: + # every task is timed, the prediction at least reaches what the + # samples saw (the prediction is an upper bound: at a handover + # the live task holds at most its peak), it never exceeds the + # cost bound (Reclaim.occupancy≤cost), and the slack between + # them is never negative. + for o in ORDERS + r = records[o] + @test all(t -> haskey(t, "first_sample") && haskey(t, "last_sample"), r["tasks"]) + @test haskey(r, "grades") + @test MP.predicted_peak_rss_kib(r) >= r["child_peak_rss_kib"] + @test MP.predicted_peak_rss_kib(r) <= MP.cost_bound_rss_kib(r) + @test MP.reclaimed_kib(r) >= 0 + @test MP.disagreement(r) === nothing + end + path = get(ENV, "OCCUPANCY_RESULTS", "") if !isempty(path) io = IOBuffer() diff --git a/test/runtests.jl b/test/runtests.jl index ed351f13..d107e559 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -45,6 +45,7 @@ const UNIT_FILES = [ "test_remote_stages.jl", "test_jobs.jl", "test_provenance.jl", + "test_occupancy_grade.jl", "test_install_pins.jl", "test_migrate_composition.jl", "test_sample_reads.jl", diff --git a/test/unit/test_occupancy_grade.jl b/test/unit/test_occupancy_grade.jl new file mode 100644 index 00000000..0308b442 --- /dev/null +++ b/test/unit/test_occupancy_grade.jl @@ -0,0 +1,318 @@ +# © 2026 Joshua Benjamin Jewell. All rights reserved. +# Licensed under the GNU Affero General Public License version 3 (AGPLv3). + +## Grading a stage against the high-water-mark algebra (MMW-01, fork issue #105) +# The algebra is proved in occupancy-types' `OccupancyTypes.Reclaim`, over the HWM +# instance in resource-algebra; `scripts/fetch-siblings.sh` pins both and +# `scripts/check-occupancy.sh` type-checks what consumes them. Nothing here +# restates a sibling's definition: what is tested is the *reading* of a probe's +# samples as grades — that a task is sequenced from the previous one's drift, +# that the siblings' laws come out of that reading, and that the prediction +# illustrates conditional predictions; historical windows were not retained. + +using MetaManifold.MemoryProbe +using OrderedCollections, YAML + +const MG = MetaManifold.MemoryProbe + +const RECORDED_OCCUPANCY = + normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "results-2026-10-14.yml")) +const MODEL_WINDOWS = + normpath(joinpath(@__DIR__, "..", "..", "docs", "occupancy", "model-windows.yml")) + +# A synthetic task with model boundaries, not a historical probe record. +# The windows determine which peaks remain live at each handover. +function _occ_task(peak; label = "t", first_sample = 1, last_sample = 2) + OrderedDict{String,Any}("label" => label, "processes" => 1, "samples" => (last_sample >= first_sample ? last_sample - first_sample + 1 : 0), + "peak_rss_kib" => peak, "hwm_kib" => peak, + "first_sample" => first_sample, "last_sample" => last_sample) +end + +# One stage as `finish!` writes it: its tasks and the peak the samples saw. +_occ_stage(tasks; measured) = OrderedDict{String,Any}( + "probe" => MG.PROBE, "interval_ms" => 10, "samples" => 10, + "julia_allocated_bytes" => 1000, "julia_live_bytes_delta" => 500, + "child_peak_rss_kib" => measured, "tasks" => Any[tasks...]) + +@testset "empirical checker rejects missing or inconsistent windows" begin + include(joinpath(@__DIR__, "..", "..", "scripts", "check-occupancy-grades.jl")) + @test MemoryProbe === MG # checker must reuse the imported package module + historical = YAML.load_file(RECORDED_OCCUPANCY)["orders"] + @test check_records(historical; io=devnull) == 2 + @test check_records(Dict(); io=devnull) == 2 + rec = deepcopy(historical["asv_then_otu"]) + for t in rec["tasks"] + t["first_sample"], t["last_sample"] = 1, 2 + end + @test check_records(Dict("impossible" => rec); io=devnull) == 1 + valid = Dict("samples" => 4, "child_peak_rss_kib" => 100, + "tasks" => [Dict("label" => "a", "samples" => 2, "peak_rss_kib" => 100, + "first_sample" => 1, "last_sample" => 2), + Dict("label" => "b", "samples" => 2, "peak_rss_kib" => 60, + "first_sample" => 3, "last_sample" => 4)]) + @test check_records(Dict("synthetic" => valid); io=devnull) == 0 + overlap = deepcopy(valid) + overlap["tasks"][2]["first_sample"] = 2 + overlap["child_peak_rss_kib"] = 120 # below 160: individual maxima need not coincide + @test check_records(Dict("noncoincident" => overlap); io=devnull) == 0 + overlap["child_peak_rss_kib"] = 161 + @test check_records(Dict("contradiction" => overlap); io=devnull) == 1 + unsampled = deepcopy(valid) + unsampled["tasks"][1]["first_sample"] = 0 + unsampled["tasks"][1]["last_sample"] = 0 + unsampled["tasks"][1]["samples"] = 0 + @test check_records(Dict("unsampled" => unsampled); io=devnull) == 2 +end + +@testset "occupancy grades" begin + @testset "a grade is (peak, drift), drift ≤ peak" begin + g = MG.Grade(90, 40) + @test g.peak == 90 + @test g.drift == 40 + @test MG.release(g) == 50 # what the frontier reclaims + # The invariant the HWM instance carries as its `bounded` field. + @test_throws ArgumentError MG.Grade(40, 90) + end + + @testset "sequencing starts the suffix from the drift (Reclaim.reclaim-law)" begin + a = MG.Grade(100, 40) + b = MG.Grade(60, 10) + s = MG.seq(a, b) + # peak (a ⊗ b) ≡ max (peak a) (drift a + peak b) + @test s.peak == max(a.peak, a.drift + b.peak) + @test s.drift == a.drift + b.drift + @test MG.seq(a, MG.NO_GRADE) == a # the unit holds nothing + @test s.peak <= a.peak + b.peak # Reclaim.occupancy≤cost + # Alternatives keep the larger of each component. + @test MG.alt(a, b) == MG.Grade(100, 40) + end + + @testset "the two shapes a stage has: serial and concurrent" begin + # One after the other: the first has exited (drift = 0), so the peak is + # the larger of the two and the cut reclaims the difference. + serial = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 2), + _occ_task(60; label = "otu", first_sample = 3, last_sample = 4)]; + measured = 100) + @test MG.predicted_peak_rss_kib(serial) == 100 + @test MG.cost_bound_rss_kib(serial) == 160 + @test MG.reclaimed_kib(serial) == 60 # Reclaim.reclaimed: strictly below cost + @test MG.disagreement(serial) === nothing + + # Together: the first is still resident when the second starts + # (drift = peak), so the peak is the sum and nothing is reclaimed. + together = _occ_stage([_occ_task(100; label = "asv", first_sample = 1, last_sample = 4), + _occ_task(60; label = "otu", first_sample = 2, last_sample = 5)]; + measured = 160) + @test MG.predicted_peak_rss_kib(together) == 160 + @test MG.cost_bound_rss_kib(together) == 160 + @test MG.reclaimed_kib(together) == 0 # Reclaim.no-reclaim: drift = peak + @test MG.disagreement(together) === nothing + @test MG.task_grades(together)[1].drift == 100 + @test MG.task_grades(together)[2].drift == -100 + @test MG.stage_grade(together).drift == 0 + end + + @testset "overlap expires at each handover" begin + for (windows, predicted, remaining) in ( + ([(1, 2), (2, 3), (4, 5)], 160, 0), # an overlapping pair, then a serial task + ([(1, 3), (2, 5), (4, 6)], 160, 0), # pairwise overlap, never all three + ([(1, 6), (2, 3), (4, 5)], 180, 0), # the first survives a shorter task + ([(1, 5), (2, 5), (3, 5)], 240, 0), # genuine three-way overlap + ([(1, 2), (1, 2), (3, 4)], 160, 0), # children registered before sample 1 + ([(1, 0), (2, 3), (4, 0)], 180, 180), # unclosed windows remain live + ) + tasks = [_occ_task(peak; label = "t$i", first_sample = window[1], + last_sample = window[2]) + for (i, (peak, window)) in enumerate(zip((100, 60, 80), windows))] + occ = _occ_stage(tasks; measured = predicted) + grades = MG.task_grades(occ) + folded = foldl(MG.seq, grades) + @test folded.peak == predicted + @test folded.drift == remaining + @test MG.release(folded) == predicted - remaining + @test MG.cost_bound_rss_kib(occ) == 240 + @test MG.reclaimed_kib(occ) == 240 - predicted + @test MG.disagreement(occ) === nothing + @test MG.graded(occ)["predicted_peak_kib"] == predicted + # Record order does not change the sampled order. + @test MG.stage_grade(_occ_stage(reverse(tasks); measured = predicted)) == folded + end + @test !MG._overlaps((0, 0), (1, 2)) # a never-started task is not live + @test !MG._overlaps((1, 0), (0, 0)) + @test MG._overlaps((1, 2), (1, 2)) + @test MG._overlaps((1, 2), (2, 3)) # the end sample is included + @test !MG._overlaps((1, 2), (3, 4)) + end + + @testset "the reviewer's three-task example: expiry, not the sum of drifts" begin + # Windows [1,3], [2,10], [4,5] and peaks 100, 1, 100 KiB — CodeRabbit's + # example on #117. A overlaps B and exits before C starts, so the largest + # simultaneous sum is 101 (A+B at samples 2-3, then B+C at 4-5), not the + # 201 a rule that kept every earlier drift would read, and not the 201 KiB + # cost. A's release at C's boundary makes B's delta negative, which is how + # the fold stops charging for it. + tasks = [_occ_task(100; label = "A", first_sample = 1, last_sample = 3), + _occ_task(1; label = "B", first_sample = 2, last_sample = 10), + _occ_task(100; label = "C", first_sample = 4, last_sample = 5)] + occ = _occ_stage(tasks; measured = 101) + @test [g.drift for g in MG.task_grades(occ)] == [100, -99, -1] + @test MG.predicted_peak_rss_kib(occ) == 101 + @test MG.cost_bound_rss_kib(occ) == 201 + @test MG.reclaimed_kib(occ) == 100 # the 100 KiB that never coexist + @test MG.disagreement(occ) === nothing + # The bound the expiry sharpens: 201 KiB resident at once is now the + # disagreement, where before the fix it was the prediction. + @test MG.disagreement(_occ_stage(tasks; measured = 201)) isa String + end + + @testset "registration keeps the first child's sample boundary" begin + # No background sampler: watch! takes the first sample deterministically. + probe = MG.Probe(0.01, ReentrantLock(), MG.TaskTrace[], MG.TaskTrace("(stage)"), + 0, 0, Base.gc_num(), Base.gc_live_bytes(), false, nothing, + nothing, nothing, nothing, Vector{UInt8}(undef, 8192), + Vector{UInt8}(undef, 64)) + function child!() + proc = run(`sleep 0.01`; wait = false) + pid = MG.watch!(proc) + try + wait(proc) + finally + MG.unwatch!(pid) + end + end + MG.within(probe) do + MG.task("two children") do + child!() + @test probe.samples == 1 + @test probe.tasks[1].first_sample == 1 + @test probe.tasks[1].last_sample == 1 + MG.task("between children") do + child!() + end + child!() + end + end + first_task, between = probe.tasks + @test first_task.processes == 2 + @test first_task.first_sample == 1 + @test first_task.last_sample == 3 + @test between.first_sample == 2 + @test MG._overlaps((first_task.first_sample, first_task.last_sample), + (between.first_sample, between.last_sample)) + @test MG.TaskTrace("never started").first_sample == 0 + end + + @testset "already-exited children stay unsampled" begin + probe = MG.start(Dict("occupancy_probe" => Dict("enabled" => true))) + MG.within(probe) do + MG.task("already exited") do + proc = run(`true`; wait=false) + wait(proc) + @test MG.watch!(proc) === nothing + MG.unwatch!(nothing) + end + end + rec = MG.finish!(probe) + t = only(rec["tasks"]) + @test t["processes"] == 1 + @test t["samples"] == 0 + @test (t["first_sample"], t["last_sample"]) == (0, 0) + @test check_records(Dict("already exited" => rec); io=devnull) == 2 + end + + @testset "the graded record says what it is" begin + rec = _occ_stage([_occ_task(100; label = "vsearch:asv", first_sample = 1, last_sample = 4), + _occ_task(60; label = "vsearch:otu", first_sample = 2, last_sample = 5)]; + measured = 160) + g = MG.graded(rec) + @test g["algebra"] == "ResourceAlgebra.Instances.HWM" + @test g["predicted_peak_kib"] == 160 + @test g["cost_bound_kib"] == 160 + @test g["reclaimed_kib"] == 0 + @test g["measured_peak_kib"] == 160 + @test g["disagreement"] === nothing + @test [t["label"] for t in g["tasks"]] == ["vsearch:asv", "vsearch:otu"] + @test g["tasks"][1]["drift_kib"] == 100 # still resident at the handover + @test g["tasks"][2]["drift_kib"] == -100 # the earlier task also exited + @test g["tasks"][2]["release_kib"] == 160 # both peaks are reclaimed + @test g["drift_kib"] == 0 + end + + @testset "a measured peak above the prediction is a disagreement" begin + # The prediction is an upper bound, so this is the one contradiction it + # allows: a sample saw more than the timed timeline can hold. + rec = _occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; measured = 250) + d = MG.disagreement(rec) + @test d isa String + @test occursin("250", d) && occursin("100", d) + # At or below the bound there is nothing to report. + @test MG.disagreement(_occ_stage([_occ_task(100; first_sample = 1, last_sample = 2)]; + measured = 100)) === nothing + end + + @testset "a record with no sample windows is not graded" begin + # Records written before the windows were sampled cannot be read as a + # timeline, so nothing is predicted for them rather than an order assumed. + rec = OrderedDict{String,Any}("child_peak_rss_kib" => 100, + "tasks" => Any[OrderedDict{String,Any}("label" => "t", "peak_rss_kib" => 100)]) + @test MG.predicted_peak_rss_kib(rec) === nothing + @test MG.graded(rec) === nothing + @test MG.disagreement(rec) === nothing # not measured is not a disagreement + end + + @testset "finish! grades the record it writes" begin + # A record the probe writes itself carries its tasks' sample windows, so + # `finish!` attaches the grades read from them; the run with real vsearch + # children is test/occupancy_disagreement_test.jl. + if !Sys.islinux() + @info "occupancy grades: not Linux, so there is no /proc to sample; the finish! grading check is SKIPPED" + else + probe = MG.start(Dict("occupancy_probe" => Dict("enabled" => true, + "interval_ms" => 10))) + MG.within(probe) do + MG.task("sleep") do + proc = run(`sleep 0.2`; wait = false) + pids = MG.watch!(proc) + try + wait(proc) + finally + MG.unwatch!(pids) + end + end + end + rec = MG.finish!(probe) + @test length(rec["tasks"]) == 1 # one task; no stage trace + t = rec["tasks"][1] + @test t["label"] == "sleep" && t["samples"] > 0 + @test t["peak_rss_kib"] > 0 # sampled from /proc + @test t["last_sample"] >= t["first_sample"] # the window watch!/unwatch! set + @test haskey(rec, "grades") + @test rec["grades"]["predicted_peak_kib"] == MG.predicted_peak_rss_kib(rec) + @test rec["grades"]["measured_peak_kib"] == rec["child_peak_rss_kib"] + @test MG.disagreement(rec) === nothing + end + end + + @testset "synthetic interleavings illustrate conditional predictions" begin + # Only peak magnitudes are historical. Boundaries are assumed examples. + orders = YAML.load_file(RECORDED_OCCUPANCY)["orders"] + schedules = YAML.load_file(MODEL_WINDOWS) + for (name, rec) in orders + @test MG.stage_grade(rec) === nothing + @test MG.predicted_peak_rss_kib(rec) === nothing + windows = schedules[name] + tasks = [_occ_task(t["peak_rss_kib"]; label = t["label"], + first_sample = windows[i][1], last_sample = windows[i][2]) + for (i, t) in enumerate(rec["tasks"])] + occ = _occ_stage(tasks; measured = rec["child_peak_rss_kib"]) + graded = MG.graded(occ) + @test MG.predicted_peak_rss_kib(occ) == rec["child_peak_rss_kib"] + @test MG.disagreement(occ) === nothing + peaks = [t["peak_rss_kib"] for t in rec["tasks"]] + # Serial: the larger peak. Together: both resident, so nothing is reclaimed. + @test graded["reclaimed_kib"] == + (name == "concurrent" ? 0 : sum(peaks) - maximum(peaks)) + @test graded["cost_bound_kib"] == sum(peaks) + end + end +end From a843a83d4dae244916a2a9e83a958b73d0285789 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:28:57 +0100 Subject: [PATCH 05/35] fix(server): isolate wildcard IP origins and reject overflowing ports (#180) Two incremental commits recover the wildcard-bind origin change and then replace throwing port parsing with tryparse. A regression test covers a 100-digit request port. Existing tests cover IP matches, mismatched origins, DNS-rebinding-shaped hostnames and environment isolation. Validation: whitespace checks pass. Julia and the server tests have not run in this environment. Keep draft until native tests and CI pass; the address regex still needs validation against malformed IPv4/IPv6 literals. No dependency changes or broad documentation rewrite. Recovered from b672663 after #178; all original work remains on archive/pre-sync-2026-10-17. --- src/server/server.jl | 35 ++++++++++++++-- test/unit/test_server_security.jl | 68 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/src/server/server.jl b/src/server/server.jl index cef70d33..d5673f9c 100644 --- a/src/server/server.jl +++ b/src/server/server.jl @@ -241,12 +241,39 @@ module Server allowed end + _is_wildcard(h::AbstractString) = lowercase(h) in ("0.0.0.0", "::", "[::]") + + """ + _bound_ip_origin(origin, host_header) -> Bool + + Whether `origin` names the exact IP address the browser reached a wildcard + bind on: an `http://` origin whose host is an IP literal (not a name), whose + port is the bound port, and whose host-and-port equals the request's `Host` + header. A browser only sends such an `Origin` for a page it loaded from that + address, so the request is same-origin by construction. A DNS-rebinding page + arrives with the attacker's domain as both `Host` and `Origin`; a name is not + an IP literal, so it is refused here as before. Hostnames (e.g. + `http://lab-box:8080`) and any address behind a proxy on another port must be + listed in `JULIA_METAMANIFOLD_ALLOWED_ORIGINS`. + """ + function _bound_ip_origin(origin::AbstractString, host_header::AbstractString) + o = _normalise_origin(origin) + startswith(o, "http://") || return false + hostport = o[length("http://") + 1:end] + hostport == lowercase(strip(host_header)) || return false + m = match(r"^(?:\d{1,3}(?:\.\d{1,3}){3}|\[[0-9a-f:]+\])(?::(\d+))?$"i, hostport) + isnothing(m) && return false + port = isnothing(m.captures[1]) ? 80 : tryparse(Int, m.captures[1]) + !isnothing(port) && port == _bound_port[] + end + """ _cors_middleware(next) - Refuse requests whose Host is not loopback on a loopback bind, and requests - carrying an `Origin` outside `_allowed_origins()`. Allowed origins get CORS - headers and preflight answers; requests with no `Origin` pass unchanged. + Refuse unexpected hosts on loopback binds and origins outside the allowlist. + On wildcard binds, also admit an HTTP origin matching the request's literal + IP address and bound port. Domain names still require explicit configuration. + Requests without an Origin continue through the host guard. """ function _cors_middleware(next) function(req::HTTP.Request) @@ -261,6 +288,8 @@ module Server return next(req) end _normalise_origin(origin) in _allowed_origins() || + (_is_wildcard(_bound_host[]) && + _bound_ip_origin(origin, HTTP.header(req, "Host", ""))) || return HTTP.Response(403, "Forbidden: origin not allowed") cors_headers = [ "Access-Control-Allow-Origin" => origin, diff --git a/test/unit/test_server_security.jl b/test/unit/test_server_security.jl index 2bb7086d..b108de6e 100644 --- a/test/unit/test_server_security.jl +++ b/test/unit/test_server_security.jl @@ -34,6 +34,14 @@ end SV.ServerState.set_root!(tmp) SV._bound_host[] = "127.0.0.1" SV._bound_port[] = 8080 + # The baseline assertions below hold only with no extra origins + # configured. JULIA_METAMANIFOLD_ALLOWED_ORIGINS is a supported + # deployment setting (split deployments, tunnels), and a developer + # running the suite with it set would otherwise fail them, so it is + # cleared here and restored in the finally. + saved_origins = get(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS", nothing) + haskey(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") && + delete!(ENV, "JULIA_METAMANIFOLD_ALLOWED_ORIGINS") mkpath(joinpath(tmp, "data", "StudyS", "run1")) touch(joinpath(tmp, "data", "StudyS", "run1", "s1_R1.fastq.gz")) mkpath(joinpath(tmp, "projects", "StudyS", "run1")) @@ -77,6 +85,64 @@ end @test _ss_request("GET", studies, ["Host" => "evil.example:8080"]).status == 403 end + @testset "Wildcard bind serves, and admits only, the reached IP" begin + studies = "/api/v1/studies" + SV._bound_host[] = "0.0.0.0" + lan = "Host" => "192.168.1.10:8080" + ui = "Origin" => "http://192.168.1.10:8080" + # The UI served at the machine's address: its origin is admitted, + # with CORS headers, a preflight answer, and writes that reach the + # route (this one fails in the route itself, not in the guard). + r = _ss_request("GET", studies, [lan, ui]) + @test r.status == 200 + @test SV.HTTP.header(r, "Access-Control-Allow-Origin") == "http://192.168.1.10:8080" + @test _ss_request("OPTIONS", studies, [lan, ui]).status == 204 + @test _ss_request("POST", "/api/v1/studies/StudyS/report?kind=bogus&ext=.txt", + [lan, ui, "Content-Type" => "text/plain"], "x").status == 400 + @test !isdir(joinpath(tmp, "projects", "StudyS", "report")) + # An IPv6 literal that matches its Host is admitted the same way. + @test _ss_request("GET", studies, + ["Host" => "[fd00::10]:8080", "Origin" => "http://[fd00::10]:8080"]).status == 200 + # Another address, another port, or another scheme: still refused. + for o in ("http://192.168.1.11:8080", "http://192.168.1.10:9090", + "https://192.168.1.10:8080") + @test _ss_request("GET", studies, [lan, "Origin" => o]).status == 403 + end + # A DNS-rebinding page: the domain matches its own Host, but a + # name is not an IP literal, so it is still refused. + @test _ss_request("GET", studies, + ["Host" => "evil.example:8080", "Origin" => "http://evil.example:8080"]).status == 403 + # A hostname the machine is reached by still has to be listed. + withenv("JULIA_METAMANIFOLD_ALLOWED_ORIGINS" => "http://lab-box:8080") do + @test _ss_request("GET", studies, + ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 200 + end + @test _ss_request("GET", studies, + ["Host" => "lab-box:8080", "Origin" => "http://lab-box:8080"]).status == 403 + SV._bound_host[] = "127.0.0.1" + end + + @testset "Implicit HTTP port on wildcard binds" begin + SV._bound_host[] = "0.0.0.0" + SV._bound_port[] = 80 + for host in ("192.168.1.10", "[2001:db8::1]") + @test SV._bound_ip_origin("http://" * host, host) + r = _ss_request("OPTIONS", "/api/v1/studies", + ["Host" => host, "Origin" => "http://" * host]) + @test r.status == 204 + @test !SV._bound_ip_origin("https://" * host, host) + end + @test !SV._bound_ip_origin("http://attacker.example", "attacker.example") + SV._bound_port[] = 8080 + @test !SV._bound_ip_origin("http://192.168.1.10", "192.168.1.10") + SV._bound_host[] = "127.0.0.1" + end + + @testset "Oversized origin ports are refused without throwing" begin + host = "192.168.1.10:" * repeat("9", 100) + @test !SV._bound_ip_origin("http://" * host, host) + end + @testset "Oxygen docs and metrics are not served" begin @test SV._SERVE_OPTIONS.docs === false @test SV._SERVE_OPTIONS.metrics === false @@ -121,6 +187,8 @@ end @test SV.HTTP.header(f, "Content-Security-Policy") == "" end finally + saved_origins === nothing || + (ENV["JULIA_METAMANIFOLD_ALLOWED_ORIGINS"] = saved_origins) SV.ServerState._root[] = old_root SV._bound_host[], SV._bound_port[] = old_host, old_port rm(tmp; recursive=true, force=true) From 00c36130964d2766a16127ea9e113d7df0bfcb06 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:31:12 +0100 Subject: [PATCH 06/35] feat(canonical): isolate JSON canonicalization utility (#182) Isolate the writer, module inclusion, fixtures and test registration. Excludes KYAML policy/tooling. Corrects a misleading UTF-16 ordering test comment. Validation: all 11 stored fixtures independently agree with ECMAScript JSON.stringify and UTF-16 key ordering under Node. This checks the fixtures, not execution of the Julia writer. Julia tests and full RFC conformance remain pending; original Rust fixture provenance has not been independently reproduced. Audit invalid Unicode, non-string keys and float conversion before integration. No production hashing route is added. --- src/MetaManifold.jl | 3 + src/canonical.jl | 216 +++++++++++++++++++++++++++++++++ test/fixtures/jcs_oracle.jsonl | 11 ++ test/runtests.jl | 1 + test/unit/test_canonical.jl | 93 ++++++++++++++ 5 files changed, 324 insertions(+) create mode 100644 src/canonical.jl create mode 100644 test/fixtures/jcs_oracle.jsonl create mode 100644 test/unit/test_canonical.jl diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index 31db2a5c..b5f13a59 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -33,6 +33,9 @@ include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/differential.jl") +# Canonical JSON: deterministic bytes for hashing and content-addressing. +include("canonical.jl") + # Web server include("server/server.jl") diff --git a/src/canonical.jl b/src/canonical.jl new file mode 100644 index 00000000..b287b7fb --- /dev/null +++ b/src/canonical.jl @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# JCS (RFC 8785) canonical JSON: deterministic bytes for hashing, signing and +# content-addressing. The same value always yields the same byte string, so a +# chart spec or an analysis result can be hashed where it is produced and the +# hash compared across machines, Julia versions and refactorings that build +# the same object in a different order. +# +# What this implements, and what it deliberately does not: +# +# * Object keys are sorted by UTF-16 code unit, the rule that makes an +# astral-plane key sort BEFORE a key such as U+FF21, because its high +# surrogate (U+D800..U+DBFF) is a smaller code unit than U+FF21. +# * Float64 values serialize per ECMAScript's Number::toString, the rules +# RFC 8785 builds on: shortest round-trip digits, positional notation for +# 1e-6 <= |x| < 1e21, exponent notation outside it, a signed exponent +# ("1e+30"), and -0.0 collapsing to 0. +# * Integers outside I-JSON's exact range (-(2^53)+1 .. 2^53-1) are +# refused, as the estate's ijson-jcs crate refuses them, because a +# consumer cannot read them back as the same number. +# * NaN and Inf are refused for the same reason. +# +# The stored fixture strings were checked against ECMAScript JSON.stringify +# with UTF-16 key ordering, and the Julia tests compare this writer with them. +# The inherited claim of Rust ijson-jcs fixture provenance has not been +# independently reproduced; no cross-language Rust verification is claimed. + +module Canonical + + +using SHA + +export jcs_bytes, jcs_sha256 + +# The I-JSON exact-integer range. +const SAFE_INT_MAX = 2^53 - 1 + +""" + jcs_bytes(x) -> Vector{UInt8} + +The RFC 8785 canonical form of `x`, as bytes. Accepts the JSON-shaped +values the server produces: `Dict` and `JSON3.Object` (key order is +irrelevant; it is sorted away), `AbstractVector` and `JSON3.Array` (order +is preserved, as JCS preserves array order), `AbstractString`, `Integer`, +`Float16`, `Float32`, `Float64`, `Bool` and `nothing`. Floats serialize per ECMAScript's +Number::toString; integers outside the exact I-JSON range, and NaN or Inf +floats, throw `ArgumentError`. +""" +function jcs_bytes(x) + io = IOBuffer() + _write!(io, x) + take!(io) +end + +""" + jcs_sha256(x) -> String + +The SHA-256 of `jcs_bytes(x)`, as lowercase hex -- the content address of +a JSON value: two producers that build the same value hash identically +whatever order they built it in. +""" +jcs_sha256(x) = bytes2hex(sha256(jcs_bytes(x))) + +# Object keys sort by UTF-16 code unit, which for astral-plane characters +# means their surrogate pair, a smaller first unit than any BMP code point +# at or above U+E000. transcoding the key to UInt16 gives exactly those +# units to compare. +_utf16_units(s::AbstractString) = transcode(UInt16, String(s)) + +_write!(io, ::Nothing) = print(io, "null") +_write!(io, b::Bool) = print(io, b ? "true" : "false") + +function _write!(io, n::Integer) + # I-JSON: only |n| <= 2^53-1 survives a JSON round trip exactly. + -SAFE_INT_MAX <= n <= SAFE_INT_MAX || throw(ArgumentError( + "canonical JSON refuses the integer $n: it is outside the exact " * + "IEEE-754 binary64 range I-JSON allows")) + print(io, string(n; base=10)) +end + +function _write!(io, x::Union{Float16,Float32,Float64}) + _write_float!(io, Float64(x)) +end + +_write!(io, ::AbstractFloat) = throw(ArgumentError( + "canonical JSON supports only Float16, Float32 and Float64")) + +function _write!(io, s::AbstractString) + isvalid(s) || throw(ArgumentError("canonical JSON refuses invalid Unicode")) + print(io, '"') + for c in s + _write_char!(io, c) + end + print(io, '"') +end + +# ECMAScript's JSON.stringify escapes exactly these: the two mandatory +# characters, the five with short forms, and every other control +# character as \u00xx with lowercase hex. +function _write_char!(io, c::Char) + if c == '"' + print(io, "\\\"") + elseif c == '\\' + print(io, "\\\\") + elseif c == '\b' + print(io, "\\b") + elseif c == '\t' + print(io, "\\t") + elseif c == '\n' + print(io, "\\n") + elseif c == '\f' + print(io, "\\f") + elseif c == '\r' + print(io, "\\r") + elseif c < ' ' + print(io, "\\u", string(UInt32(c); base=16, pad=4)) + else + print(io, c) + end +end + +# AbstractDict (Dict and JSON3.Object alike): sorted keys, then each pair. +function _write!(io, d::AbstractDict) + print(io, '{') + first = true + entries = Pair{String,Any}[] + names = Set{String}() + for (key, value) in pairs(d) + key isa Union{AbstractString,Symbol} || throw(ArgumentError( + "canonical JSON object keys must be strings or symbols")) + name = String(key) + isvalid(name) || throw(ArgumentError("canonical JSON refuses invalid Unicode keys")) + name in names && throw(ArgumentError("canonical JSON refuses duplicate property names")) + push!(names, name) + push!(entries, name => value) + end + sort!(entries; by = entry -> _utf16_units(entry.first)) + for (name, value) in entries + first || print(io, ',') + first = false + _write!(io, name) + print(io, ':') + _write!(io, value) + end + print(io, '}') +end + +# AbstractVector (Vector and JSON3.Array alike): order preserved. +function _write!(io, a::AbstractVector) + print(io, '[') + for (i, v) in enumerate(a) + i > 1 && print(io, ',') + _write!(io, v) + end + print(io, ']') +end + +# RFC 8785 §3.2.2.3: ECMAScript's Number::toString. Julia's own repr gives +# the shortest round-trip digits (Ryu, the same family ES specifies), so +# the work is reformatting those digits under ES's notation rules. +function _write_float!(io, x::Float64) + isnan(x) && throw(ArgumentError( + "canonical JSON refuses NaN: no consumer reads it back as the same value")) + isinf(x) && throw(ArgumentError( + "canonical JSON refuses Inf: no consumer reads it back as the same value")) + if x == 0 + print(io, "0") # +0.0 and -0.0 both collapse to 0, as ECMAScript does. + return + end + if signbit(x) + print(io, '-') + x = -x + end + digits, n = _shortest_digits(x) + k = length(digits) + if n > 21 || n <= -6 + # Exponential: one digit, then the fraction, then e with an + # explicit sign (ES prints "1e+30", not "1e30"). + print(io, digits[1]) + k > 1 && (print(io, '.'); print(io, digits[2:end])) + e = n - 1 + print(io, 'e', e < 0 ? '-' : '+', string(abs(e))) + elseif n >= k + # An integer too big for the fraction: digits followed by zeros. + print(io, digits, "0"^(n - k)) + elseif n > 0 + # Digits with the point inside them. + print(io, digits[1:n], '.', digits[n+1:end]) + else + # Below one, positional as far as ES goes: 0.000001, not 1e-6. + print(io, "0.", "0"^(-n), digits) + end +end + +# The shortest round-trip decimal form of x as (digits, n) with +# x = 0.digits * 10^n: repr yields either "123", "4.5" or "1.5e-10". +function _shortest_digits(x::Float64) + s = repr(x) + mantissa, exponent = occursin('e', s) ? (split(s, 'e') .|> String) : (s, "") + e = isempty(exponent) ? 0 : parse(Int, exponent) + if occursin('.', mantissa) + intpart, fracpart = split(mantissa, '.') .|> String + else + intpart, fracpart = mantissa, "" + end + digits = string(intpart, fracpart) + n = length(intpart) + e + # Trailing zeros are notation, not value: 1.0e30 is digits "1", n 31. + while length(digits) > 1 && endswith(digits, '0') + digits = digits[1:end-1] + end + digits, n +end + +end # module diff --git a/test/fixtures/jcs_oracle.jsonl b/test/fixtures/jcs_oracle.jsonl new file mode 100644 index 00000000..dd446d8b --- /dev/null +++ b/test/fixtures/jcs_oracle.jsonl @@ -0,0 +1,11 @@ +{"input":{"b":1,"a":2,"c":[]},"expected":"{\"a\":2,\"b\":1,\"c\":[]}"} +{"input":{"€":"euro","Ａ":"fullwidth A","a":"first"},"expected":"{\"a\":\"first\",\"€\":\"euro\",\"Ａ\":\"fullwidth A\"}"} +{"input":{"😀":"astral first?","Ａ":"high bmp"},"expected":"{\"😀\":\"astral first?\",\"Ａ\":\"high bmp\"}"} +{"input":{"d":[3,1,2],"e":{"z":1,"y":{"x":[true,false,null]}}},"expected":"{\"d\":[3,1,2],\"e\":{\"y\":{\"x\":[true,false,null]},\"z\":1}}"} +{"input":[4.50,2.4499999999999998e16,1E+30,0.000001,1E-7,1E+21,1E+20,1E-33,-0.0,0.0,9.9999999999999997E+22],"expected":"[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,1e-33,0,0,1e+23]"} +{"input":[0.000001,1E-7,1.5E-10,9007199254740991,-42],"expected":"[0.000001,1e-7,1.5e-10,9007199254740991,-42]"} +{"input":{"s\":with\"quotes":"tab\there\nnewline","esc\\":"\\backslash","ctl":"\u0001\u001f"},"expected":"{\"ctl\":\"\\u0001\\u001f\",\"esc\\\\\":\"\\\\backslash\",\"s\\\":with\\\"quotes\":\"tab\\there\\nnewline\"}"} +{"input":{"nested":{"arr":[{"n":1.0},{"m":2}],"deep":{"deeper":{"deepest":"value"}}}},"expected":"{\"nested\":{\"arr\":[{\"n\":1},{\"m\":2}],\"deep\":{\"deeper\":{\"deepest\":\"value\"}}}}"} +{"input":{"literals":true,"n":null,"f":false,"empty_obj":{},"empty_arr":[]},"expected":"{\"empty_arr\":[],\"empty_obj\":{},\"f\":false,\"literals\":true,\"n\":null}"} +{"input":[5E-324,1.7976931348623157E+308,2.2250738585072014E-308,0.3,0.1,0.30000000000000004],"expected":"[5e-324,1.7976931348623157e+308,2.2250738585072014e-308,0.3,0.1,0.30000000000000004]"} +{"input":{"Plotly":{"layout":{"title":{"text":"CO₂"}},"data":[{"x":[1,2,3],"y":[4.5,5.5,6.5],"type":"scatter"}]}},"expected":"{\"Plotly\":{\"data\":[{\"type\":\"scatter\",\"x\":[1,2,3],\"y\":[4.5,5.5,6.5]}],\"layout\":{\"title\":{\"text\":\"CO₂\"}}}}"} diff --git a/test/runtests.jl b/test/runtests.jl index d107e559..77e15bcf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -54,6 +54,7 @@ const UNIT_FILES = [ "test_permanova.jl", "test_write_gates.jl", "test_server_security.jl", + "test_canonical.jl", ] # The files named on the command line ("test_routes.jl", "routes" or diff --git a/test/unit/test_canonical.jl b/test/unit/test_canonical.jl new file mode 100644 index 00000000..22194b92 --- /dev/null +++ b/test/unit/test_canonical.jl @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Tests against stored canonical strings independently checked using +# ECMAScript JSON.stringify and UTF-16 key order. Rust fixture generation +# provenance has not been reproduced in this audit. +using MetaManifold +using MetaManifold.Canonical: jcs_bytes, jcs_sha256 +using JSON3, SHA, Test + +@testset "canonical JSON (JCS)" begin + # Byte-for-byte against the stored expected strings, over the fixture + # corpus: key sorting (including the astral-before-high-BMP UTF-16 + # case), ES number formatting, string escaping, nesting, literals. + fixtures = joinpath(@__DIR__, "..", "fixtures", "jcs_oracle.jsonl") + for (i, line) in enumerate(eachline(fixtures)) + case = JSON3.read(line) + @testset "oracle case $i" begin + value = case.input + # The input arrives parsed; the expected is the stored canonical text. + @test String(jcs_bytes(value)) == case.expected + end + end + + @testset "key order is irrelevant, bytes are not" begin + a = Dict("b" => 1, "a" => Dict("y" => [1.0, 2], "x" => "s")) + b = Dict("a" => Dict("x" => "s", "y" => [1.0, 2]), "b" => 1) + @test jcs_bytes(a) == jcs_bytes(b) + # ...but array order is preserved, as JCS requires. + c = Dict("a" => [2, 1]) + @test jcs_bytes(a) != jcs_bytes(c) + end + + @testset "the astral key sorts before the high BMP key" begin + # U+1F600 arrives as the surrogate pair D83D DE00; UTF-16 code-unit + # order puts it before U+FF21. Naive code-point and UTF-8 byte order would put the BMP key + # first instead -- this case distinguishes the UTF-16 rule. + d = Dict("\uff21" => "high bmp", "\U0001f600" => "astral") + @test String(jcs_bytes(d)) == "{\"😀\":\"astral\",\"Ａ\":\"high bmp\"}" + end + + @testset "float notation follows ECMAScript" begin + @test String(jcs_bytes([4.50, 2.4499999999999998e16, 1e30, 1e-6, 1e-7, + 1e21, 1e20, -0.0, 5e-324])) == + "[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,0,5e-324]" + end + + @testset "refusals" begin + @test_throws ArgumentError jcs_bytes(2^53) + @test_throws ArgumentError jcs_bytes(-(2^53)) + @test_throws ArgumentError jcs_bytes(Inf) + @test_throws ArgumentError jcs_bytes(NaN) + @test jcs_bytes(2^53 - 1) == b"9007199254740991" + end + + @testset "a content address is stable" begin + # The use this exists for: a chart-like spec built in any key + # order hashes identically, and the hash can be recorded where the + # bytes cannot. + spec = Dict( + "layout" => Dict("title" => Dict("text" => "CO₂")), + "data" => [Dict("x" => [1, 2, 3], "y" => [4.5, 5.5, 6.5], + "type" => "scatter")]) + same = Dict( + "data" => [Dict("type" => "scatter", "y" => [4.5, 5.5, 6.5], + "x" => [1, 2, 3])], + "layout" => Dict("title" => Dict("text" => "CO₂"))) + @test jcs_sha256(spec) == jcs_sha256(same) + @test jcs_sha256(spec) == bytes2hex(sha256(jcs_bytes(spec))) + end +end + +@testset "canonical input validation regressions" begin + @test String(jcs_bytes(Dict(:b => 2, :a => 1))) == "{\"a\":1,\"b\":2}" + @test String(jcs_bytes(copy(JSON3.read("{\"a\":1}")))) == "{\"a\":1}" + @test_throws ArgumentError jcs_bytes(Dict{Any,Int}(:a => 1, "a" => 2)) + @test_throws ArgumentError jcs_bytes(JSON3.read("{\"a\":1,\"a\":2}")) + @test_throws ArgumentError jcs_bytes(Dict(1 => "not a JSON key")) + invalid = String(UInt8[0xff]) + @test_throws ArgumentError jcs_bytes(invalid) + @test_throws ArgumentError jcs_bytes(Dict(invalid => 1)) + @test_throws ArgumentError jcs_bytes(BigFloat(1) + eps(BigFloat)) + @test String(jcs_bytes([Float16(1.5), Float32(1.5), Float64(1.5)])) == "[1.5,1.5,1.5]" + for T in (UInt8, UInt16, UInt32, UInt64, UInt128) + @test String(jcs_bytes(T(1))) == "1" + end + @test String(jcs_bytes(UInt8[0, 1, 255])) == "[0,1,255]" +end + +@testset "rounding fixture is explicitly binary64 input" begin + cases = readlines(joinpath(@__DIR__, "..", "fixtures", "jcs_oracle.jsonl")) + @test JSON3.read(cases[5]).input[2] isa AbstractFloat +end From a84b24f867152c0daac9fc390a126f527c325e86 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:05:53 +0100 Subject: [PATCH 07/35] feat(differential): add reproducible glmGamPoi dispersion-trend fits (#181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds optional `dispersion: glmGamPoi` for negative-binomial differential abundance. It fits the shared dispersion trend across tested taxa, then fits MASS NB GLMs with fixed scale 1. ML remains the default. This is plug-in NB Wald inference: it does not implement glmGamPoi’s quasi-likelihood test or propagate trend-estimation uncertainty. Zero dispersion withholds NB inference for that taxon; invalid trends affect their own taxa. Missing glmGamPoi returns HTTP 503. Configuration PATCHes save canonical dispersion values and publish one complete YAML document. PATCH and DELETE validate the prospective edited configuration and affected descendant cascades under the same lock before publication. Invalid combinations return HTTP 400 with the affected descendant identified and leave files unchanged. Runs without override files retain inheritance. The frontend enum matches the saved values. The R environment adds twelve dependency records, including glmGamPoi 1.22.0, for R 4.5.0 / Bioconductor 3.22; existing package versions are retained. Dependency discovery and both CI verification lists include glmGamPoi. Repository-maintained Julia installation/coverage steps satisfy the Actions policy, and restored relevance/occupancy jobs provide all six required branch checks. Julia’s archive URL/version/checksum come from the committed pins. CoverageTools is pinned to registered release 1.4.2. The redundant documentation child #185 is closed after its command-line rebase left no unique changes. Validation: - Real glmGamPoi/MASS reference fits, Poisson boundaries, malformed trends and missing-package HTTP 503 were validated against the declared restored R dependency closure. - Current configuration regression selection: 182 assertions pass, including the real MetaManifold router, covering PATCH/GET/YAML round trips and all four DELETE scopes. - Current running-application smoke test: 18 assertions pass through the HTTP listener, including canonical configuration persistence. This verifies the API; it does not claim an interactive browser test. - Isolated actual mutation helpers: 53 checks pass. Runtime/pin/relevance suite: 242 checks pass, including changes limited to either root installer. Frontend: 43 tests and TypeScript checking pass. Lockfile validation checks 160 dependency requirements. - [Final-commit CI](https://github.com/hyperpolymath/MetaManifold-WebUI/actions/runs/39089064276) passes all six required checks, including the registered Julia scientific/integration/server suite and fresh empirical occupancy-grade validation. Julia reports 3,187 passing assertions and five skipped optional-tool probes; the existing CI configuration bypasses DADA2 taxonomy classification (`CI_SKIP_TAXONOMY=1`). Reproduce the complete registered differential and pin tests with `julia --project=. -t 2 test/runtests.jl test_differential.jl test_install_pins.jl` after restoring `renv.lock`; full CI runs `julia --project=. -t 2 --code-coverage=user --compiled-modules=no test/runtests.jl --integration --server`. --------- Signed-off-by: Jonathan D.A. Jewell <6759885+hyperpolymath@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 195 +++++- R/_renv_dependencies.R | 1 + config/defaults/pipeline.yml | 8 + frontend/src/api/types.ts | 2 + .../src/components/PipelineStages.test.ts | 9 + frontend/src/components/PipelineStages.tsx | 2 + renv.lock | 559 ++++++++++++++++++ scripts/check-occupancy.sh | 6 +- scripts/install-ci-julia.sh | 33 ++ scripts/process-ci-coverage.jl | 7 + src/analysis/differential.jl | 240 ++++++-- src/server/routes/analysis.jl | 9 +- src/server/routes/config.jl | 218 +++++-- test/integration/test_server.jl | 16 + test/unit/test_differential.jl | 349 ++++++++++- test/unit/test_install_pins.jl | 95 ++- 16 files changed, 1634 insertions(+), 115 deletions(-) create mode 100644 frontend/src/components/PipelineStages.test.ts create mode 100644 scripts/install-ci-julia.sh create mode 100644 scripts/process-ci-coverage.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df48fab7..ef150857 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,138 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'push' }} jobs: + # Restore the required relevance contexts removed by the baseline sync (#178). + # A failed comparison fails its gate; a docs-only diff can skip expensive jobs. + gate: + name: Code relevance + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} + steps: + # A push only starts this workflow on main (the branch filter above), so + # the gate has nothing to decide: every push is relevant. + - name: A push is already branch-filtered + id: push + if: github.event_name == 'push' + run: echo 'relevant=true' >> "$GITHUB_OUTPUT" + + # Full history, because the three-dot diff names the merge base of the + # PR's base tip and its merge commit, and a stacked PR's merge base sits + # further back still. + - name: Checkout the change + if: github.event_name == 'pull_request' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false # nothing here pushes + fetch-depth: 0 + + # --no-renames, so a file renamed out of a listed path is reported as a + # deletion under its original name and still counts. -z with a + # NUL-delimited read, so a pathname Git would quote or that contains a + # newline arrives raw and matches the pattern. The base SHA travels + # through env, not template interpolation into the script. + - name: Does the change touch anything the suite exercises? + id: pr + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + relevant=false + # Every pattern is quoted, so the glob survives array assignment and + # fires at `case` time, against the diff -- an unquoted `src/*` would + # expand against the checkout the runner happens to be standing in. + patterns=('src/*' 'test/*' 'scripts/*' 'frontend/*' 'config/*' + 'bench/*' 'data/*' 'proofs/*' 'R/*' 'renv/*' 'Project.toml' + 'Manifest.toml' 'renv.lock' '.Rprofile' 'install.sh' 'install.jl' + 'start.sh' 'docs/occupancy/*' 'docs/occupancy-2026-10.adoc' + '.github/workflows/*') + # The diff goes to a file first because a process substitution hides + # git diff's exit status: an unavailable comparison SHA (a force-push + # on the base branch, for instance) or a repository error would read + # as "no changes", and the gate would post relevant=false -- skipping + # the Julia suite and the proof checker, whose `skipped` conclusion + # satisfies a required check -- so a relevant PR would pass silently. + # A failed diff must fail the gate itself: that is the one failure + # the required-check design cannot afford to skip. As in the + # occupancy gate, which this mirrors. + changed="$RUNNER_TEMP/ci-changed-files" + if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then + echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 + exit 1 + fi + while IFS= read -r -d '' file; do + for pattern in "${patterns[@]}"; do + case "$file" in + $pattern) relevant=true; break 2 ;; + esac + done + done < "$changed" + echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + + occupancy_gate: + name: Occupancy relevance + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + relevant: ${{ steps.push.outputs.relevant || steps.pr.outputs.relevant }} + steps: + # All pushes to main are relevant; PRs use the path classification below. + - name: A push to main is relevant + id: push + if: github.event_name == 'push' + run: echo 'relevant=true' >> "$GITHUB_OUTPUT" + + # Full history, because the three-dot diff names the merge base of the + # PR's base tip and its merge commit -- a shallow checkout does not have + # both, and a stacked PR's merge base sits further back still. + - name: Checkout the change + if: github.event_name == 'pull_request' + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false # nothing here pushes + fetch-depth: 0 + + - name: Does the change touch the occupancy reading? + id: pr + if: github.event_name == 'pull_request' + env: + # zizmor flags template expressions interpolated into run scripts + # (template-injection), and the gate script is easier to run locally + # with the value in an environment variable in any case. The base SHA + # is GitHub-supplied, but the indirection costs nothing. + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + relevant=false + patterns=('src/core/memory_probe.jl' 'src/core/provenance.jl' 'src/pipeline/tools.jl' + 'src/server/routes/pipeline.jl' 'test/occupancy_disagreement_test.jl' + 'test/unit/test_occupancy_grade.jl' 'scripts/check-occupancy-grades.jl' + 'scripts/lint-occupancy-readings.sh' 'docs/occupancy/*' + 'docs/occupancy-2026-10.adoc' 'scripts/check-occupancy.sh' + 'scripts/fetch-siblings.sh' 'scripts/install-ci-julia.sh' + 'Project.toml' 'Manifest.toml' 'renv.lock' 'config/defaults/tool_versions.yml' + '.github/workflows/*') + # The diff goes to a file first because a process substitution hides + # git diff's exit status: an unavailable comparison SHA or a repository + # error would read as "no changes", and the gate would post + # relevant=false -- skipping the two occupancy jobs, whose `skipped` + # conclusion satisfies a required check -- so a relevant PR would + # pass silently. A failed diff must fail the gate itself: that is + # the one failure the required-check design cannot afford to skip. + changed="$RUNNER_TEMP/occupancy-changed-files" + if ! git diff --no-renames --name-only -z "$BASE_SHA...HEAD" > "$changed"; then + echo "ERROR: git diff $BASE_SHA...HEAD failed" >&2 + exit 1 + fi + while IFS= read -r -d '' file; do + for pattern in "${patterns[@]}"; do + case "$file" in + $pattern) relevant=true; break 2 ;; + esac + done + done < "$changed" + echo "relevant=$relevant" >> "$GITHUB_OUTPUT" + test: # The name is a fixed string, it carries no version, and the job has NO matrix. # Both are required, and the first without the second is a trap that has already @@ -40,6 +172,8 @@ jobs: # silently. If a matrix is ever genuinely wanted here, the required context must move # to a matrix-free aggregator job first. name: Julia tests + needs: gate + if: needs.gate.outputs.relevant == 'true' # Explicit, not ubuntu-latest: the R pin below names an apt package revision built # for 24.04, and a runner that silently rolled to the next LTS would take that pin # with it. `runs-on` cannot read `env`, so this is a literal. @@ -58,17 +192,16 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Julia - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 # v2.7.0 - with: - # Pinned rather than tracking latest stable, because a pinned Manifest.toml - # resolved by an unpinned Julia is not a reproducible build. This must equal - # julia_version in Manifest.toml; it cannot be read from the pin file, since - # nothing can be read before Julia exists. test/unit/test_install_pins.jl - # fails if the three ever disagree. - version: "1.12.5" + run: bash scripts/install-ci-julia.sh - name: Cache Julia packages - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.julia/artifacts + ~/.julia/packages + ~/.julia/registries + key: julia-${{ runner.os }}-${{ hashFiles('Manifest.toml', 'config/defaults/tool_versions.yml') }} # Every external version CI installs is read from the committed pin file, so CI # and a developer's machine cannot drift apart. A temporary environment is used @@ -156,7 +289,7 @@ jobs: sudo chmod 777 "$RENV_PATHS_CACHE" sudo env R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'install.packages(c("renv", "BiocManager"), repos="https://cloud.r-project.org", lib=.Library)' sudo env RENV_PATHS_CACHE="$RENV_PATHS_CACHE" R_CLI_DYNAMIC=false TERM=dumb Rscript --no-init-file -e 'renv::restore(project=".", library=.Library, prompt=FALSE)' - Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' + Rscript --no-init-file -e 'for (pkg in c("dada2","Biostrings","ShortRead","vegan","MASS","zCompositions","glmGamPoi")) if (!require(pkg,character.only=TRUE,quietly=TRUE)) stop(pkg, " failed to install")' - name: Check DADA2 CSV assembly run: Rscript --vanilla test/unit/test_dada2_tables.R @@ -250,13 +383,14 @@ jobs: julia --project=. -e ' using RCall R"cat(.libPaths(), sep=\"\n\")" - for pkg in ["dada2", "vegan", "tibble", "MASS", "zCompositions"] + for pkg in ["dada2", "vegan", "tibble", "MASS", "zCompositions", "glmGamPoi"] R"if (!require($pkg, character.only=TRUE, quietly=TRUE)) stop($pkg, \" not found\")" end ' - name: Run tests env: + OCCUPANCY_RESULTS: ${{ runner.temp }}/occupancy-results.yml CI_SKIP_TAXONOMY: "1" R_LIBS_SITE: /usr/local/lib/R/site-library:/usr/lib/R/site-library:/usr/lib/R/library # A failure also posts annotations, which the checks API serves without the full @@ -264,6 +398,7 @@ jobs: # (up to 8), followed by the summary rows that report a failure or an error. shell: bash run: | + command -v vsearch # The full suite must run the occupancy experiment, not skip it. set +e julia --project=. -t 2 --code-coverage=user --compiled-modules=no test/runtests.jl --integration --server 2>&1 | tee /tmp/julia-tests.log status="${PIPESTATUS[0]}" @@ -292,8 +427,13 @@ jobs: fi exit "$status" + - name: Validate measured occupancy grades + run: | + julia --project=. scripts/check-occupancy-grades.jl "$RUNNER_TEMP/occupancy-results.yml" + bash scripts/lint-occupancy-readings.sh + - name: Process coverage - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38 # v1.2.2 + run: julia scripts/process-ci-coverage.jl - name: Upload coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2 @@ -314,6 +454,8 @@ jobs: # reads them. agda: name: Agda proofs + needs: gate + if: needs.gate.outputs.relevant == 'true' runs-on: ubuntu-24.04 container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb timeout-minutes: 30 @@ -346,7 +488,9 @@ jobs: agda MetaManifold/All.agda occupancy: - name: Occupancy proof gate + name: Occupancy typed peak + needs: occupancy_gate + if: needs.occupancy_gate.outputs.relevant == 'true' runs-on: ubuntu-24.04 container: debian:trixie-20250929-slim@sha256:1caf1c703c8f7e15dcf2e7769b35000c764e6f50e4d7401c355fb0248f3ddfdb timeout-minutes: 30 @@ -356,8 +500,27 @@ jobs: run: | apt-get update -qq apt-get install -y --no-install-recommends ca-certificates curl git agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 - - uses: julia-actions/setup-julia@4c0cb0fce8556fdb04a90347310e5db8b1f98fb9 - with: - version: "1.12.5" + - name: Set up Julia + run: bash scripts/install-ci-julia.sh - name: Check Session IR and occupancy proofs run: bash scripts/check-occupancy.sh + + # The full suite already runs the actual two-thread vsearch experiment. This + # required context reports that result, including the fresh grade validation, + # instead of paying for a second Julia/R installation and repeating the run. + disagreement: + name: Occupancy disagreement test + needs: [occupancy_gate, test] + if: always() && needs.occupancy_gate.result == 'success' && needs.occupancy_gate.outputs.relevant == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require the executed experiment and grade validation to pass + env: + TEST_RESULT: ${{ needs.test.result }} + run: | + if [ "$TEST_RESULT" != success ]; then + echo "ERROR: full suite, occupancy experiment or measured-grade validation did not pass ($TEST_RESULT)" >&2 + exit 1 + fi + echo 'The full Julia suite passed, including the two-thread vsearch experiment and fresh empirical grade validation.' >> "$GITHUB_STEP_SUMMARY" diff --git a/R/_renv_dependencies.R b/R/_renv_dependencies.R index 20394ab6..9f4cf2fb 100644 --- a/R/_renv_dependencies.R +++ b/R/_renv_dependencies.R @@ -6,4 +6,5 @@ if (FALSE) { # Retain while installer and CI explicitly verify this package. library(tibble) library(MASS) + library(glmGamPoi) } diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 8181ae6d..44b2c882 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -358,6 +358,14 @@ analysis: # rle is refused when no taxon is present in every sample. # Used by nb_glm only; clr_welch ignores it, since log-ratios carry no library size. offset: tss + # nb_glm only: where each taxon's negative-binomial theta comes from. ml + # (the default) takes the maximum-likelihood estimate MASS::glm.nb fits per + # taxon. glmGamPoi takes the shrunken dispersion glmGamPoi::glm_gp fits + # across all taxa in one call -- sharing information across taxa, which + # steadies low-count taxa -- and fits each taxon through + # MASS::negative.binomial(theta = 1/dispersion), so the Wald test is + # unchanged. Requires the glmGamPoi R package. + dispersion: ml # Fraction of samples, 0 to 1, in which a taxon must have reads to be # tested. Taxa below it are listed as filtered, without a p-value. For # clr_welch it is applied before zero replacement, so on a sparse table it also diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 87457dad..4ea20add 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -439,6 +439,8 @@ export interface DifferentialResult { min_prevalence: number /** nb_glm only. */ offset?: 'tss' | 'rle' + /** nb_glm only: where each taxon's theta comes from. */ + dispersion?: 'ml' | 'glmGamPoi' /** clr_welch only. */ replacement_delta?: number } diff --git a/frontend/src/components/PipelineStages.test.ts b/frontend/src/components/PipelineStages.test.ts new file mode 100644 index 00000000..37fa1f24 --- /dev/null +++ b/frontend/src/components/PipelineStages.test.ts @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: AGPL-3.0-only +import { expect, test } from 'bun:test' +import { CONFIG_TYPES } from './PipelineStages' + +test('dispersion uses the enum editor with both supported models', () => { + expect(CONFIG_TYPES['analysis.differential.dispersion']).toEqual({ + kind: 'enum', options: ['ml', 'glmGamPoi'], + }) +}) diff --git a/frontend/src/components/PipelineStages.tsx b/frontend/src/components/PipelineStages.tsx index 85129267..f4cc3fbe 100644 --- a/frontend/src/components/PipelineStages.tsx +++ b/frontend/src/components/PipelineStages.tsx @@ -152,6 +152,7 @@ export const CONFIG_DESCRIPTIONS: Record = { 'analysis.beta.depth': 'Reads per sample for rarefy or srs; 0 uses the smallest non-empty sample.', 'analysis.differential.method': 'Model fitted to each taxon: nb_glm (negative-binomial GLM on read counts; effect is a log2 fold change) or clr_welch (Welch\'s t-test on centred log-ratios over the taxa passing min_prevalence, after zCompositions::cmultRepl zero replacement; effect is a CLR difference, not a fold change).', 'analysis.differential.offset': 'Library-size offset in each negative-binomial model: tss (total reads per sample) or rle (median-of-ratios over taxa present in every sample; refused when there are none). Used by nb_glm only.', + 'analysis.differential.dispersion': 'nb_glm only: ml estimates theta separately with MASS::glm.nb; glmGamPoi uses a cross-taxon dispersion trend in fixed-scale negative-binomial Wald fits.', 'analysis.differential.min_prevalence': 'Fraction of samples, 0 to 1, in which a taxon must have reads to be tested; taxa below it are listed as filtered.', 'analysis.differential.replacement_delta': 'clr_welch only: cmultRepl\'s frac, between 0 and 1; an imputed proportion above its taxon\'s smallest observed proportion is lowered to this fraction of it. Replaced zeros are not measurements.', 'analysis.alpha.show_points': 'Show individual samples overlaid on alpha comparison boxplots.', @@ -252,6 +253,7 @@ export const CONFIG_TYPES: Record = { 'analysis.beta.depth': { kind: 'int' }, 'analysis.differential.method': { kind: 'enum', options: ['nb_glm', 'clr_welch'] }, 'analysis.differential.offset': { kind: 'enum', options: ['tss', 'rle'] }, + 'analysis.differential.dispersion': { kind: 'enum', options: ['ml', 'glmGamPoi'] }, 'analysis.differential.min_prevalence': { kind: 'float', min: 0, max: 1, step: 0.05 }, 'analysis.differential.replacement_delta': { kind: 'float', min: 0.01, max: 0.99, step: 0.05 }, 'analysis.alpha.show_points': { kind: 'boolean' }, diff --git a/renv.lock b/renv.lock index a487543c..7df10ebb 100644 --- a/renv.lock +++ b/renv.lock @@ -3231,6 +3231,565 @@ "BugReports": "https://github.com/Japal/zCompositions/issues", "NeedsCompilation": "no", "Author": "Javier Palarea-Albaladejo [cre, aut] (ORCID: ), Josep Antoni Martin-Fernandez [aut] (ORCID: )" + }, + "DelayedMatrixStats": { + "Package": "DelayedMatrixStats", + "Version": "1.32.0", + "Source": "Bioconductor", + "Type": "Package", + "Title": "Functions that Apply to Rows and Columns of 'DelayedMatrix' Objects", + "Date": "2025-01-09", + "Authors@R": "c(person(\"Peter\", \"Hickey\", role = c(\"aut\", \"cre\"), email = \"peter.hickey@gmail.com\", comment = c(ORCID = \"0000-0002-8153-6258\")), person(\"Hervé\", \"Pagès\", role = \"ctb\", email = \"hpages.on.github@gmail.com\"), person(\"Aaron\", \"Lun\", role = \"ctb\", email = \"infinite.monkeys.with.keyboards@gmail.com\"))", + "Description": "A port of the 'matrixStats' API for use with DelayedMatrix objects from the 'DelayedArray' package. High-performing functions operating on rows and columns of DelayedMatrix objects, e.g. col / rowMedians(), col / rowRanks(), and col / rowSds(). Functions optimized per data type and for subsetted calculations such that both memory usage and processing time is minimized.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "Roxygen": "list(markdown = TRUE)", + "RoxygenNote": "7.3.2", + "Depends": [ + "MatrixGenerics (>= 1.15.1)", + "DelayedArray (>= 0.31.7)" + ], + "Imports": [ + "methods", + "sparseMatrixStats (>= 1.13.2)", + "Matrix (>= 1.5-0)", + "S4Vectors (>= 0.17.5)", + "IRanges (>= 2.25.10)", + "SparseArray (>= 1.5.19)" + ], + "Suggests": [ + "testthat", + "knitr", + "rmarkdown", + "BiocStyle", + "microbenchmark", + "profmem", + "HDF5Array", + "matrixStats (>= 1.0.0)" + ], + "VignetteBuilder": "knitr", + "URL": "https://github.com/PeteHaitch/DelayedMatrixStats", + "BugReports": "https://github.com/PeteHaitch/DelayedMatrixStats/issues", + "biocViews": "Infrastructure, DataRepresentation, Software", + "git_url": "https://git.bioconductor.org/packages/DelayedMatrixStats", + "git_branch": "RELEASE_3_22", + "git_last_commit": "cbf7d75", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "no", + "Author": "Peter Hickey [aut, cre] (ORCID: ), Hervé Pagès [ctb], Aaron Lun [ctb]", + "Maintainer": "Peter Hickey " + }, + "HDF5Array": { + "Package": "HDF5Array", + "Version": "1.38.0", + "Source": "Bioconductor", + "Title": "HDF5 datasets as array-like objects in R", + "Description": "The HDF5Array package is an HDF5 backend for DelayedArray objects. It implements the HDF5Array, H5SparseMatrix, H5ADMatrix, and TENxMatrix classes, 4 convenient and memory-efficient array-like containers for representing and manipulating either: (1) a conventional (a.k.a. dense) HDF5 dataset, (2) an HDF5 sparse matrix (stored in CSR/CSC/Yale format), (3) the central matrix of an h5ad file (or any matrix in the /layers group), or (4) a 10x Genomics sparse matrix. All these containers are DelayedArray extensions and thus support all operations (delayed or block-processed) supported by DelayedArray objects.", + "biocViews": "Infrastructure, DataRepresentation, DataImport, Sequencing, RNASeq, Coverage, Annotation, GenomeAnnotation, SingleCell, ImmunoOncology", + "URL": "https://bioconductor.org/packages/HDF5Array", + "BugReports": "https://github.com/Bioconductor/HDF5Array/issues", + "License": "Artistic-2.0", + "Encoding": "UTF-8", + "Authors@R": "person(\"Hervé\", \"Pagès\", role=c(\"aut\", \"cre\"), email=\"hpages.on.github@gmail.com\", comment=c(ORCID=\"0009-0002-8272-4522\"))", + "Depends": [ + "R (>= 3.4)", + "methods", + "SparseArray (>= 1.7.5)", + "DelayedArray (>= 0.33.5)", + "h5mread (>= 0.99.4)" + ], + "Imports": [ + "utils", + "stats", + "tools", + "Matrix", + "BiocGenerics (>= 0.51.2)", + "S4Vectors", + "IRanges", + "S4Arrays (>= 1.1.1)", + "rhdf5" + ], + "Suggests": [ + "BiocParallel", + "GenomicRanges", + "SummarizedExperiment (>= 1.15.1)", + "h5vcData", + "ExperimentHub", + "TENxBrainData", + "zellkonverter", + "GenomicFeatures", + "SingleCellExperiment", + "DelayedMatrixStats", + "genefilter", + "RSpectra", + "RUnit", + "knitr", + "rmarkdown", + "BiocStyle" + ], + "VignetteBuilder": "knitr", + "Collate": "utils.R h5utils.R HDF5ArraySeed-class.R HDF5Array-class.R ReshapedHDF5ArraySeed-class.R ReshapedHDF5Array-class.R dump-management.R writeHDF5Array.R saveHDF5SummarizedExperiment.R H5SparseMatrixSeed-class.R H5SparseMatrix-class.R H5ADMatrixSeed-class.R H5ADMatrix-class.R TENxMatrixSeed-class.R TENxMatrix-class.R writeTENxMatrix.R zzz.R", + "git_url": "https://git.bioconductor.org/packages/HDF5Array", + "git_branch": "RELEASE_3_22", + "git_last_commit": "9bca08f", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "no", + "Author": "Hervé Pagès [aut, cre] (ORCID: )", + "Maintainer": "Hervé Pagès " + }, + "RcppArmadillo": { + "Package": "RcppArmadillo", + "Version": "15.6.2-1", + "Source": "Repository", + "Type": "Package", + "Title": "'Rcpp' Integration for the 'Armadillo' Templated Linear Algebra Library", + "Date": "2026-10-16", + "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Romain\", \"Francois\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Doug\", \"Bates\", role = \"aut\", comment = c(ORCID = \"0000-0001-8316-9503\")), person(\"Binxiang\", \"Ni\", role = \"aut\"), person(\"Conrad\", \"Sanderson\", role = \"aut\", comment = c(ORCID = \"0000-0002-0049-4501\")))", + "Description": "'Armadillo' is a templated C++ linear algebra library aiming towards a good balance between speed and ease of use. It provides high-level syntax and functionality deliberately similar to Matlab. It is useful for algorithm development directly in C++, or quick conversion of research code into production environments. It provides efficient classes for vectors, matrices and cubes where dense and sparse matrices are supported. Integer, floating point and complex numbers are supported. A sophisticated expression evaluator (based on template meta-programming) automatically combines several operations to increase speed and efficiency. Dynamic evaluation automatically chooses optimal code paths based on detected matrix structures. Matrix decompositions are provided through integration with LAPACK, or one of its high performance drop-in replacements (such as 'MKL' or 'OpenBLAS'). It can automatically use 'OpenMP' multi-threading (parallelisation) to speed up computationally expensive operations. . The 'RcppArmadillo' package includes the header files from the 'Armadillo' library; users do not need to install 'Armadillo' itself in order to use 'RcppArmadillo'. Starting from release 15.0.0, the minimum compilation standard is C++14. . Since release 7.800.0, 'Armadillo' is licensed under Apache License 2; previous releases were under licensed as MPL 2.0 from version 3.800.0 onwards and LGPL-3 prior to that; 'RcppArmadillo' (the 'Rcpp' bindings/bridge to Armadillo) is licensed under the GNU GPL version 2 or later, as is the rest of 'Rcpp'.", + "License": "GPL (>= 2)", + "LazyLoad": "yes", + "Depends": [ + "R (>= 3.3.0)" + ], + "LinkingTo": [ + "Rcpp" + ], + "Imports": [ + "Rcpp (>= 1.1.1)", + "stats", + "utils", + "methods" + ], + "Suggests": [ + "tinytest", + "Matrix (>= 1.3.0)", + "pkgKitten", + "reticulate", + "slam" + ], + "URL": "https://github.com/RcppCore/RcppArmadillo, https://dirk.eddelbuettel.com/code/rcpp.armadillo.html", + "BugReports": "https://github.com/RcppCore/RcppArmadillo/issues", + "RoxygenNote": "6.0.1", + "Encoding": "UTF-8", + "VignetteBuilder": "Rcpp", + "NeedsCompilation": "yes", + "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), Romain Francois [aut] (ORCID: ), Doug Bates [aut] (ORCID: ), Binxiang Ni [aut], Conrad Sanderson [aut] (ORCID: )", + "Maintainer": "Dirk Eddelbuettel ", + "Repository": "CRAN" + }, + "Rhdf5lib": { + "Package": "Rhdf5lib", + "Version": "1.32.0", + "Source": "Bioconductor", + "Type": "Package", + "Title": "hdf5 library as an R package", + "Authors@R": "c( person( \"Mike\", \"Smith\", role = \"ctb\", comment = c(ORCID = \"0000-0002-7800-3848\") ), person( \"Hugo\", \"Gruson\", role = \"cre\", email = \"hugo.gruson@embl.de\", comment = c(ORCID = \"0000-0002-4094-1476\")) , person( given = \"The HDF Group\", role = \"cph\" ) )", + "Description": "Provides C and C++ hdf5 libraries.", + "License": "Artistic-2.0", + "Copyright": "src/hdf5/COPYING", + "LazyLoad": "true", + "VignetteBuilder": "knitr", + "Depends": [ + "R (>= 4.2.0)" + ], + "Suggests": [ + "BiocStyle", + "knitr", + "rmarkdown", + "tinytest", + "mockery" + ], + "URL": "https://github.com/Huber-group-EMBL/Rhdf5lib", + "BugReports": "https://github.com/Huber-group-EMBL/Rhdf5lib/issues", + "SystemRequirements": "GNU make", + "Encoding": "UTF-8", + "biocViews": "Infrastructure", + "RoxygenNote": "7.1.2", + "git_url": "https://git.bioconductor.org/packages/Rhdf5lib", + "git_branch": "RELEASE_3_22", + "git_last_commit": "f62ae28", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Mike Smith [ctb] (ORCID: ), Hugo Gruson [cre] (ORCID: ), The HDF Group [cph]", + "Maintainer": "Hugo Gruson " + }, + "SingleCellExperiment": { + "Package": "SingleCellExperiment", + "Version": "1.32.0", + "Source": "Bioconductor", + "Date": "2025-06-27", + "Title": "S4 Classes for Single Cell Data", + "Authors@R": "c( person(\"Aaron\", \"Lun\", role=c(\"aut\", \"cph\"), email=\"infinite.monkeys.with.keyboards@gmail.com\"), person(\"Davide\",\"Risso\", role=c(\"aut\",\"cre\", \"cph\"), email=\"risso.davide@gmail.com\"), person(\"Keegan\", \"Korthauer\", role=\"ctb\"), person(\"Kevin\", \"Rue-Albrecht\", role=\"ctb\"), person(\"Luke\", \"Zappia\", role = \"ctb\", comment = c(ORCID = \"0000-0001-7744-8565\", github = \"lazappi\")))", + "Depends": [ + "SummarizedExperiment" + ], + "Imports": [ + "methods", + "utils", + "stats", + "S4Vectors", + "BiocGenerics", + "GenomicRanges", + "DelayedArray" + ], + "Suggests": [ + "testthat", + "BiocStyle", + "knitr", + "rmarkdown", + "Matrix", + "scRNAseq (>= 2.9.1)", + "Rtsne" + ], + "biocViews": "ImmunoOncology, DataRepresentation, DataImport, Infrastructure, SingleCell", + "Description": "Defines a S4 class for storing data from single-cell experiments. This includes specialized methods to store and retrieve spike-in information, dimensionality reduction coordinates and size factors for each cell, along with the usual metadata for genes and libraries.", + "License": "GPL-3", + "VignetteBuilder": "knitr", + "RoxygenNote": "7.3.2", + "git_url": "https://git.bioconductor.org/packages/SingleCellExperiment", + "git_branch": "RELEASE_3_22", + "git_last_commit": "db7133e", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "no", + "Author": "Aaron Lun [aut, cph], Davide Risso [aut, cre, cph], Keegan Korthauer [ctb], Kevin Rue-Albrecht [ctb], Luke Zappia [ctb] (ORCID: , github: lazappi)", + "Maintainer": "Davide Risso " + }, + "assorthead": { + "Package": "assorthead", + "Version": "1.4.0", + "Source": "Bioconductor", + "Date": "2025-10-27", + "Title": "Assorted Header-Only C++ Libraries", + "Authors@R": "person(\"Aaron\", \"Lun\", role=c(\"cre\", \"aut\"), email=\"infinite.monkeys.with.keyboards@gmail.com\")", + "Description": "Vendors an assortment of useful header-only C++ libraries. Bioconductor packages can use these libraries in their own C++ code by LinkingTo this package without introducing any additional dependencies. The use of a central repository avoids duplicate vendoring of libraries across multiple R packages, and enables better coordination of version updates across cohorts of interdependent C++ libraries.", + "License": "MIT + file LICENSE", + "Suggests": [ + "knitr", + "rmarkdown", + "BiocStyle" + ], + "VignetteBuilder": "knitr", + "URL": "https://github.com/LTLA/assorthead", + "BugReports": "https://github.com/LTLA/assorthead/issues", + "Encoding": "UTF-8", + "biocViews": "SingleCell, QualityControl, Normalization, DataRepresentation, DataImport, DifferentialExpression, Alignment", + "git_url": "https://git.bioconductor.org/packages/assorthead", + "git_branch": "RELEASE_3_22", + "git_last_commit": "9255f2f", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "no", + "Author": "Aaron Lun [cre, aut]", + "Maintainer": "Aaron Lun " + }, + "beachmat": { + "Package": "beachmat", + "Version": "2.26.0", + "Source": "Bioconductor", + "Date": "2025-08-19", + "Title": "Compiling Bioconductor to Handle Each Matrix Type", + "Authors@R": "c(person(\"Aaron\", \"Lun\", role = c(\"aut\", \"cre\"), email = \"infinite.monkeys.with.keyboards@gmail.com\"), person(\"Hervé\", \"Pagès\", role=\"aut\"), person(\"Mike\", \"Smith\", role=\"aut\"))", + "Imports": [ + "methods", + "DelayedArray (>= 0.27.2)", + "SparseArray", + "BiocGenerics", + "Matrix", + "Rcpp" + ], + "Suggests": [ + "testthat", + "BiocStyle", + "knitr", + "rmarkdown", + "rcmdcheck", + "BiocParallel", + "HDF5Array", + "beachmat.hdf5" + ], + "LinkingTo": [ + "Rcpp", + "assorthead (>= 1.3.3)" + ], + "biocViews": "DataRepresentation, DataImport, Infrastructure", + "Description": "Provides a consistent C++ class interface for reading from a variety of commonly used matrix types. Ordinary matrices and several sparse/dense Matrix classes are directly supported, along with a subset of the delayed operations implemented in the DelayedArray package. All other matrix-like objects are supported by calling back into R.", + "License": "GPL-3", + "NeedsCompilation": "yes", + "VignetteBuilder": "knitr", + "SystemRequirements": "C++17", + "URL": "https://github.com/tatami-inc/beachmat", + "BugReports": "https://github.com/tatami-inc/beachmat/issues", + "RoxygenNote": "7.3.2", + "Encoding": "UTF-8", + "git_url": "https://git.bioconductor.org/packages/beachmat", + "git_branch": "RELEASE_3_22", + "git_last_commit": "016c55e", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "Author": "Aaron Lun [aut, cre], Hervé Pagès [aut], Mike Smith [aut]", + "Maintainer": "Aaron Lun " + }, + "glmGamPoi": { + "Package": "glmGamPoi", + "Version": "1.22.0", + "Source": "Bioconductor", + "Type": "Package", + "Title": "Fit a Gamma-Poisson Generalized Linear Model", + "Authors@R": "c(person(\"Constantin\", \"Ahlmann-Eltze\", email = \"artjom31415@googlemail.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-3762-068X\")), person(\"Nathan\", \"Lubock\", email=\"lubock.nathan@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0001-8064-2465\")), person(\"Michael\", \"Love\", email=\"michaelisaiahlove@gmail.com\", role = \"ctb\"))", + "Description": "Fit linear models to overdispersed count data. The package can estimate the overdispersion and fit repeated models for matrix input. It is designed to handle large input datasets as they typically occur in single cell RNA-seq experiments.", + "License": "GPL-3", + "Encoding": "UTF-8", + "SystemRequirements": "C++17", + "Suggests": [ + "testthat (>= 2.1.0)", + "zoo", + "DESeq2", + "edgeR", + "limma", + "MASS", + "statmod", + "ggplot2", + "bench", + "BiocParallel", + "knitr", + "rmarkdown", + "BiocStyle", + "TENxPBMCData", + "muscData", + "scran", + "dplyr" + ], + "LinkingTo": [ + "Rcpp", + "RcppArmadillo", + "beachmat", + "assorthead" + ], + "Imports": [ + "Rcpp", + "beachmat", + "DelayedMatrixStats", + "matrixStats", + "MatrixGenerics", + "SparseArray (>= 1.5.21)", + "S4Vectors", + "DelayedArray", + "HDF5Array", + "Matrix", + "SummarizedExperiment", + "SingleCellExperiment", + "BiocGenerics", + "methods", + "stats", + "utils", + "splines", + "rlang", + "vctrs" + ], + "Roxygen": "list(markdown = TRUE)", + "RoxygenNote": "7.3.2", + "URL": "https://github.com/const-ae/glmGamPoi", + "BugReports": "https://github.com/const-ae/glmGamPoi/issues", + "biocViews": "Regression, RNASeq, Software, SingleCell", + "VignetteBuilder": "knitr", + "git_url": "https://git.bioconductor.org/packages/glmGamPoi", + "git_branch": "RELEASE_3_22", + "git_last_commit": "908cde9", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Constantin Ahlmann-Eltze [aut, cre] (ORCID: ), Nathan Lubock [ctb] (ORCID: ), Michael Love [ctb]", + "Maintainer": "Constantin Ahlmann-Eltze ", + "Depends": [ + "R (>= 4.1.0)" + ] + }, + "h5mread": { + "Package": "h5mread", + "Version": "1.2.1", + "Source": "Bioconductor", + "Title": "A fast HDF5 reader", + "Description": "The main function in the h5mread package is h5mread(), which allows reading arbitrary data from an HDF5 dataset into R, similarly to what the h5read() function from the rhdf5 package does. In the case of h5mread(), the implementation has been optimized to make it as fast and memory-efficient as possible.", + "biocViews": "Infrastructure, DataRepresentation, DataImport", + "URL": "https://bioconductor.org/packages/h5mread", + "BugReports": "https://github.com/Bioconductor/h5mread/issues", + "License": "Artistic-2.0", + "Encoding": "UTF-8", + "Authors@R": "person(\"Hervé\", \"Pagès\", role=c(\"aut\", \"cre\"), email=\"hpages.on.github@gmail.com\", comment=c(ORCID=\"0009-0002-8272-4522\"))", + "Depends": [ + "R (>= 4.5)", + "methods", + "rhdf5", + "BiocGenerics", + "SparseArray" + ], + "Imports": [ + "stats", + "tools", + "rhdf5filters", + "S4Vectors", + "IRanges", + "S4Arrays" + ], + "LinkingTo": [ + "Rhdf5lib", + "S4Vectors" + ], + "SystemRequirements": "GNU make", + "Suggests": [ + "BiocParallel", + "ExperimentHub", + "TENxBrainData", + "HDF5Array", + "testthat", + "knitr", + "rmarkdown", + "BiocStyle" + ], + "VignetteBuilder": "knitr", + "Collate": "utils.R h5dim.R H5File-class.R h5ls.R H5DSetDescriptor-class.R uaselection.R h5mread.R h5summarize.R h5mread_from_reshaped.R h5dimscales.R h5writeDimnames.R zzz.R", + "git_url": "https://git.bioconductor.org/packages/h5mread", + "git_branch": "RELEASE_3_22", + "git_last_commit": "b377076", + "git_last_commit_date": "2025-11-21", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Hervé Pagès [aut, cre] (ORCID: )", + "Maintainer": "Hervé Pagès " + }, + "rhdf5": { + "Package": "rhdf5", + "Version": "2.54.1", + "Source": "Bioconductor", + "Type": "Package", + "Title": "R Interface to HDF5", + "Authors@R": "c( person(\"Bernd\", \"Fischer\", role = \"aut\"), person( \"Mike\", \"Smith\", role= \"aut\", comment = c(ORCID = \"0000-0002-7800-3848\", \"Maintainer from 2017 to 2025\") ), person(\"Gregoire\", \"Pau\", role=\"aut\"), person(\"Martin\", \"Morgan\", role = \"ctb\"), person(\"Daniel\", \"van Twisk\", role = \"ctb\"), person( \"Hugo\", \"Gruson\", role = \"cre\", email = \"hugo.gruson@embl.de\", comment = c(ORCID = \"0000-0002-4094-1476\") ) )", + "Description": "This package provides an interface between HDF5 and R. HDF5's main features are the ability to store and access very large and/or complex datasets and a wide variety of metadata on mass storage (disk) through a completely portable file format. The rhdf5 package is thus suited for the exchange of large and/or complex datasets between R and other software package, and for letting R applications work on datasets that are larger than the available RAM.", + "License": "Artistic-2.0", + "URL": "https://github.com/Huber-group-EMBL/rhdf5", + "BugReports": "https://github.com/Huber-group-EMBL/rhdf5/issues", + "LazyLoad": "true", + "VignetteBuilder": "knitr", + "Imports": [ + "Rhdf5lib (>= 1.13.4)", + "rhdf5filters (>= 1.15.5)" + ], + "Depends": [ + "R (>= 4.0.0)", + "methods" + ], + "Suggests": [ + "bit64", + "BiocStyle", + "knitr", + "rmarkdown", + "testthat", + "bench", + "dplyr", + "ggplot2", + "mockery", + "BiocParallel" + ], + "LinkingTo": [ + "Rhdf5lib" + ], + "SystemRequirements": "GNU make", + "biocViews": "Infrastructure, DataImport", + "Encoding": "UTF-8", + "Roxygen": "list(markdown = TRUE)", + "RoxygenNote": "7.3.3", + "git_url": "https://git.bioconductor.org/packages/rhdf5", + "git_branch": "RELEASE_3_22", + "git_last_commit": "7f691e4", + "git_last_commit_date": "2025-12-02", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Bernd Fischer [aut], Mike Smith [aut] (ORCID: , Maintainer from 2017 to 2025), Gregoire Pau [aut], Martin Morgan [ctb], Daniel van Twisk [ctb], Hugo Gruson [cre] (ORCID: )", + "Maintainer": "Hugo Gruson " + }, + "rhdf5filters": { + "Package": "rhdf5filters", + "Version": "1.22.0", + "Source": "Bioconductor", + "Type": "Package", + "Title": "HDF5 Compression Filters", + "Authors@R": "c( person(\"Mike\", \"Smith\", role = c(\"aut\", \"ccp\"), comment = c(ORCID = \"0000-0002-7800-3848\") ), person(\"Hugo\", \"Gruson\", role = \"cre\", email = \"hugo.gruson@embl.de\", comment = c(ORCID = \"0000-0002-4094-1476\") ) )", + "Description": "Provides a collection of additional compression filters for HDF5 datasets. The package is intended to provide seemless integration with rhdf5, however the compiled filters can also be used with external applications.", + "License": "BSD_2_clause + file LICENSE", + "LazyLoad": "true", + "VignetteBuilder": "knitr", + "Suggests": [ + "BiocStyle", + "knitr", + "rmarkdown", + "tinytest", + "rhdf5 (>= 2.47.7)" + ], + "SystemRequirements": "GNU make", + "URL": "https://github.com/Huber-group-EMBL/rhdf5filters", + "BugReports": "https://github.com/Huber-group-EMBL/rhdf5filters/issues", + "LinkingTo": [ + "Rhdf5lib" + ], + "RoxygenNote": "7.3.2", + "Encoding": "UTF-8", + "biocViews": "Infrastructure, DataImport", + "git_url": "https://git.bioconductor.org/packages/rhdf5filters", + "git_branch": "RELEASE_3_22", + "git_last_commit": "3465c24", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Mike Smith [aut, ccp] (ORCID: ), Hugo Gruson [cre] (ORCID: )", + "Maintainer": "Hugo Gruson " + }, + "sparseMatrixStats": { + "Package": "sparseMatrixStats", + "Version": "1.22.0", + "Source": "Bioconductor", + "Type": "Package", + "Title": "Summary Statistics for Rows and Columns of Sparse Matrices", + "Authors@R": "person(\"Constantin\", \"Ahlmann-Eltze\", email = \"artjom31415@googlemail.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-3762-068X\"))", + "Description": "High performance functions for row and column operations on sparse matrices. For example: col / rowMeans2, col / rowMedians, col / rowVars etc. Currently, the optimizations are limited to data in the column sparse format. This package is inspired by the matrixStats package by Henrik Bengtsson.", + "License": "MIT + file LICENSE", + "Encoding": "UTF-8", + "RoxygenNote": "7.3.1", + "LinkingTo": [ + "Rcpp" + ], + "Imports": [ + "Rcpp", + "Matrix", + "matrixStats (>= 0.60.0)", + "methods" + ], + "Depends": [ + "MatrixGenerics (>= 1.5.3)" + ], + "Suggests": [ + "testthat (>= 2.1.0)", + "knitr", + "bench", + "rmarkdown", + "BiocStyle" + ], + "SystemRequirements": "C++11", + "VignetteBuilder": "knitr", + "URL": "https://github.com/const-ae/sparseMatrixStats", + "BugReports": "https://github.com/const-ae/sparseMatrixStats/issues", + "biocViews": "Infrastructure, Software, DataRepresentation", + "git_url": "https://git.bioconductor.org/packages/sparseMatrixStats", + "git_branch": "RELEASE_3_22", + "git_last_commit": "3acf348", + "git_last_commit_date": "2025-10-29", + "Repository": "Bioconductor 3.22", + "NeedsCompilation": "yes", + "Author": "Constantin Ahlmann-Eltze [aut, cre] (ORCID: )", + "Maintainer": "Constantin Ahlmann-Eltze " } } } diff --git a/scripts/check-occupancy.sh b/scripts/check-occupancy.sh index 185934a6..e7d45c85 100755 --- a/scripts/check-occupancy.sh +++ b/scripts/check-occupancy.sh @@ -17,8 +17,12 @@ bash scripts/fetch-siblings.sh deps=_build/dependencies sir="$deps/occupancy-types/src/julia" agda --version | head -1 +# Debian's stdlib 2.x stores modules under src/; older packages used the root. +stdlib=/usr/share/agda-stdlib +if [ -f "$stdlib/src/Data/Integer.agda" ]; then stdlib="$stdlib/src"; fi +[ -f "$stdlib/Data/Integer.agda" ] || { echo "ERROR: Agda standard library is unavailable" >&2; exit 1; } flags=(--safe --without-K --no-libraries --ignore-interfaces --double-check - -i docs/occupancy -i /usr/share/agda-stdlib -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") + -i docs/occupancy -i "$stdlib" -i "$deps/occupancy-types/src" -i "$deps/resource-algebra/src") logs=_build/check-logs mkdir -p "$logs" failures=0 diff --git a/scripts/install-ci-julia.sh b/scripts/install-ci-julia.sh new file mode 100644 index 00000000..ec5da9e0 --- /dev/null +++ b/scripts/install-ci-julia.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Install the committed Linux x86-64 runtime after verifying its archive checksum. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +pins=config/defaults/tool_versions.yml +version=$(awk '/^runtimes:/ {r=1;next} r && /^ julia:/ {j=1;next} j && /^ version:/ {gsub(/"/,"",$2);print $2;exit}' "$pins") +# Print the requested field from the Julia linux-x86_64 pin to stdout. +# Arguments: $1 is the field name (url or sha256); missing fields print nothing. +field() { + awk -v key="$1" ' + /^runtimes:/ {r=1;next} + r && /^[^ #]/ {r=0} + r && /^ julia:/ {j=1;next} + j && /^ [^ ]/ {j=0} + j && $1 == "linux-x86_64:" {p=1;next} + p && /^ [^ ]/ {p=0} + p && $1 == key ":" {gsub(/"/,"",$2);print $2;exit} + ' "$pins" +} +url=$(field url) +sha=$(field sha256) +[[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ && "$sha" =~ ^[0-9a-f]{64}$ && "$url" == https://* ]] || { echo 'Invalid Julia runtime pin' >&2; exit 1; } +if [[ "${1:-}" == --print-pin ]]; then printf '%s\n' "$version" "$url" "$sha"; exit; fi +[[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || { echo 'This CI installer requires Linux x86-64' >&2; exit 1; } +: "${RUNNER_TEMP:?}" "${GITHUB_PATH:?}" +archive=$(mktemp "$RUNNER_TEMP/julia.XXXXXX.tar.gz") +trap 'rm -f "$archive"' EXIT +curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --retry 3 "$url" -o "$archive" +printf '%s %s\n' "$sha" "$archive" | sha256sum -c - +tar --no-same-owner -xzf "$archive" -C "$RUNNER_TEMP" +"$RUNNER_TEMP/julia-$version/bin/julia" --version +printf '%s\n' "$RUNNER_TEMP/julia-$version/bin" >> "$GITHUB_PATH" diff --git a/scripts/process-ci-coverage.jl b/scripts/process-ci-coverage.jl new file mode 100644 index 00000000..2e4b69c3 --- /dev/null +++ b/scripts/process-ci-coverage.jl @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# CI tooling lives in a temporary environment, separate from the app lockfile. +using Pkg +Pkg.activate(; temp=true) +Pkg.add(PackageSpec(name="CoverageTools", version="1.4.2")) +using CoverageTools +LCOV.writefile("lcov.info", process_folder("src")) diff --git a/src/analysis/differential.jl b/src/analysis/differential.jl index d2c78de0..66cbf471 100644 --- a/src/analysis/differential.jl +++ b/src/analysis/differential.jl @@ -22,13 +22,18 @@ using DataFrames, RCall using ..RRuntime: with_r_lock using ..Analysis: _palette_hex, R_WAIT_SECONDS export DifferentialConfig, ScalingRefusal, MASSUnavailable, ZCompositionsUnavailable, - OFFSET_METHODS, METHODS, tss_factors, rle_factors, size_factors, bh_adjust, - validate_counts, replace_zeros, clr_transform, fit_nb, fit_welch, - differential_abundance, volcano_chart + GlmGamPoiUnavailable, OFFSET_METHODS, METHODS, tss_factors, rle_factors, + size_factors, bh_adjust, validate_counts, replace_zeros, clr_transform, + fit_nb, fit_welch, differential_abundance, volcano_chart # Methods accepted under `analysis.differential.method`. const METHODS = ("nb_glm", "clr_welch") +# The per-taxon dispersion `nb_glm` uses: maximum likelihood (MASS::glm.nb's +# own theta) or glmGamPoi's shrinkage estimator, which shares information +# across taxa. `"ml"` is the default and keeps #24's results unchanged. +const DISPERSIONS = ("ml", "glmGamPoi") + # Offset (size factor) methods accepted under `analysis.differential.offset`. const OFFSET_METHODS = ("tss", "rle") @@ -43,7 +48,7 @@ const DEFAULT_DELTA = 0.65 """ DifferentialConfig(offset="tss", min_prevalence=0.0; method="nb_glm", - replacement_delta=0.65) + replacement_delta=0.65, dispersion="ml") Settings for one differential abundance analysis, read from `analysis.differential` in the pipeline config. `method` is `"nb_glm"` or @@ -54,26 +59,50 @@ log-ratios, which are already free of library size. have at least one read to be tested. `replacement_delta`, in (0, 1), is the `frac` passed to `zCompositions::cmultRepl` by `clr_welch`: an imputed proportion above its taxon's smallest observed proportion is lowered to this fraction of it. +`dispersion` names the per-taxon dispersion `nb_glm` uses (`"ml"` or +`"glmGamPoi"`): `"ml"` takes the maximum-likelihood theta from `MASS::glm.nb`, +`"glmGamPoi"` uses `overdispersion_shrinkage_list.dispersion_trend` from +`glmGamPoi::glm_gp` for +the taxa passing the prevalence filter, with `theta = 1 / dispersion` fixed +in each negative-binomial GLM. Both options use a Wald test of the group +coefficient with fixed scale 1; `"glmGamPoi"` is refused for `clr_welch`. +The trend is plugged into an NB Wald model; its estimation uncertainty is not +propagated. This option does not implement glmGamPoi's quasi-likelihood test. +Method and offset names are stripped and lowercased; dispersion names are +stripped and matched without regard to case, then stored as `"ml"` or +`"glmGamPoi"`. Throws `ArgumentError` for unsupported names, incompatible +method and dispersion choices, or non-finite or out-of-range fractions, +including settings unused by the selected method. """ struct DifferentialConfig method :: String offset :: String min_prevalence :: Float64 replacement_delta :: Float64 + dispersion :: String function DifferentialConfig(offset::AbstractString="tss", min_prevalence::Real=0.0; method::AbstractString="nb_glm", - replacement_delta::Real=DEFAULT_DELTA) + replacement_delta::Real=DEFAULT_DELTA, + dispersion::AbstractString="ml") m = lowercase(strip(String(method))) m in METHODS || throw(ArgumentError( "analysis.differential.method must be one of $(join(METHODS, ", ")), not '$method'")) o = lowercase(strip(String(offset))) o in OFFSET_METHODS || throw(ArgumentError( "analysis.differential.offset must be one of $(join(OFFSET_METHODS, ", ")), not '$offset'")) + d = strip(String(dispersion)) + di = findfirst(s -> lowercase(s) == lowercase(d), DISPERSIONS) + di === nothing && throw(ArgumentError( + "analysis.differential.dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) + d = DISPERSIONS[di] + (d == "ml" || m == "nb_glm") || throw(ArgumentError( + "analysis.differential.dispersion is a negative-binomial setting, but method is " * + "'$m'; only nb_glm has a dispersion to choose")) (isfinite(min_prevalence) && 0 <= min_prevalence <= 1) || throw(ArgumentError( "analysis.differential.min_prevalence must be a fraction in [0, 1], not $min_prevalence")) (isfinite(replacement_delta) && 0 < replacement_delta < 1) || throw(ArgumentError( "analysis.differential.replacement_delta must be in (0, 1), not $replacement_delta")) - new(m, o, Float64(min_prevalence), Float64(replacement_delta)) + new(m, o, Float64(min_prevalence), Float64(replacement_delta), d) end end @@ -115,6 +144,25 @@ Base.showerror(io::IO, ::ZCompositionsUnavailable) = print(io, "the R package zCompositions is not available, so zeros cannot be replaced for " * "clr_welch; restore the R library from renv.lock (Rscript -e 'renv::restore()')") +""" + GlmGamPoiUnavailable() + +Raised when the R package glmGamPoi, which provides the shrunken dispersion +`analysis.differential.dispersion: glmGamPoi` asks for, cannot be loaded. +""" +struct GlmGamPoiUnavailable <: Exception end + +""" + Base.showerror(io::IO, ::GlmGamPoiUnavailable) + +Write the missing-package error and guidance to restore the pinned R library or select +`dispersion: ml` to `io`, without a trailing newline. +""" +Base.showerror(io::IO, ::GlmGamPoiUnavailable) = print(io, + "the R package glmGamPoi is not available, so the shrunken dispersion cannot be " * + "fitted; restore the pinned R library (Rscript -e 'renv::restore()') " * + "or use analysis.differential.dispersion: ml") + """ _geomean(x) -> Float64 @@ -243,7 +291,37 @@ da_result <- data.frame(status = rep("ok", da_n), note = rep("", da_n), estimate = rep(NA_real_, da_n), se = rep(NA_real_, da_n), statistic = rep(NA_real_, da_n), pvalue = rep(NA_real_, da_n), theta = rep(NA_real_, da_n), stringsAsFactors = FALSE) +# glmGamPoi's shrunken dispersion, fitted across all taxa in one call -- its +# genes x samples convention, so the counts are transposed, and size_factors +# is the linear size factor (the per-taxon fits below take its log as the +# offset). A global fit error or missing trend fails the batch. A zero trend +# is a per-taxon Poisson boundary; malformed values fail only their own taxa. +# No fallback to a different dispersion estimator is permitted. +da_phi <- rep(NA_real_, da_n) +da_gp_failed <- "" +if (da_use_gp) { + da_gp <- tryCatch( + glmGamPoi::glm_gp(t(da_counts), design = model.matrix(~ da_g), + size_factors = exp(da_offset), overdispersion_shrinkage = TRUE), + error = function(e) e) + if (inherits(da_gp, "error")) { + da_gp_failed <- paste("glm_gp stopped with an error:", conditionMessage(da_gp)) + } else { + # glm_gp's overdispersions retain the unshrunk MLEs. Its NB refit uses + # dispersion_trend; ql_disp_shrunken is a distinct quasi-likelihood scale. + da_phi <- as.numeric(da_gp[["overdispersion_shrinkage_list"]][["dispersion_trend"]]) + if (length(da_phi) != da_n) { + da_gp_failed <- "glm_gp returned no dispersion trend for each taxon" + da_phi <- rep(NA_real_, da_n) + } + } +} for (da_j in seq_len(da_n)) { + if (nzchar(da_gp_failed)) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- da_gp_failed + next + } da_y <- da_counts[, da_j] if (length(unique(da_y)) < 2L) { da_result[da_j, "status"] <- "failed" @@ -263,18 +341,58 @@ for (da_j in seq_len(da_n)) { next } da_warns <- character(0) - da_fit <- tryCatch( - withCallingHandlers( - MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), - warning = function(w) { - da_warns <<- c(da_warns, conditionMessage(w)) - invokeRestart("muffleWarning") - }), - error = function(e) e) - if (inherits(da_fit, "error")) { - da_result[da_j, "status"] <- "failed" - da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) - next + # "ml": theta is estimated per taxon by glm.nb's own alternating iterations. + # "glmGamPoi": theta is TAKEN from the shrunken dispersion fitted above + # (theta = 1 / phi) and the per-taxon fit goes through glm() with MASS's + # negative.binomial family, so the Wald test and every check below are the + # same machinery on a given dispersion instead of an estimated one. + if (da_use_gp) { + if (!is.finite(da_phi[da_j]) || da_phi[da_j] < 0) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- "the dispersion trend for this taxon is invalid" + next + } + if (da_phi[da_j] == 0) { + da_result[da_j, "status"] <- "boundary" + da_result[da_j, "note"] <- "zero dispersion: Poisson boundary; NB Wald inference withheld" + next + } + da_theta <- 1 / da_phi[da_j] + if (!is.finite(da_theta) || da_theta <= 0) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- "the shrunken dispersion for this taxon is not usable" + next + } + da_fit <- tryCatch( + withCallingHandlers( + glm(da_y ~ da_g + offset(da_offset), + family = MASS::negative.binomial(theta = da_theta), + control = glm.control(maxit = 100)), + warning = function(w) { + da_warns <<- c(da_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + }), + error = function(e) e) + if (inherits(da_fit, "error")) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- paste("the negative-binomial fit stopped with an error:", + conditionMessage(da_fit)) + next + } + } else { + da_fit <- tryCatch( + withCallingHandlers( + MASS::glm.nb(da_y ~ da_g + offset(da_offset), control = glm.control(maxit = 100)), + warning = function(w) { + da_warns <<- c(da_warns, conditionMessage(w)) + invokeRestart("muffleWarning") + }), + error = function(e) e) + if (inherits(da_fit, "error")) { + da_result[da_j, "status"] <- "failed" + da_result[da_j, "note"] <- paste("glm.nb stopped with an error:", conditionMessage(da_fit)) + next + } } da_warns <- unique(da_warns) if (!isTRUE(da_fit[["converged"]]) || !is.null(da_fit[["th.warn"]]) || @@ -284,7 +402,7 @@ for (da_j in seq_len(da_n)) { paste(c(da_fit[["th.warn"]], da_warns), collapse = "; ")) next } - da_co <- summary(da_fit)[["coefficients"]] + da_co <- summary(da_fit, dispersion = 1)[["coefficients"]] if (!(da_term %in% rownames(da_co))) { da_result[da_j, "status"] <- "failed" da_result[da_j, "note"] <- "the group coefficient is not estimable (aliased)" @@ -296,15 +414,23 @@ for (da_j in seq_len(da_n)) { da_result[da_j, "note"] <- "the fit returned a non-finite estimate, standard error or p-value" next } - da_theta <- da_fit[["theta"]] + # "glmGamPoi" set da_theta before the fit; "ml" takes it from the fit. + if (!da_use_gp) da_theta <- da_fit[["theta"]] da_result[da_j, c("estimate", "se", "statistic", "pvalue")] <- unname(da_row[1:4]) da_result[da_j, "theta"] <- da_theta if (!is.finite(da_theta) || da_theta >= da_theta_upper) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" + da_result[da_j, "note"] <- if (da_use_gp) + paste("the shrunken dispersion puts these counts at the Poisson end:", + "no overdispersion is left, so the fit is effectively Poisson") + else + "the dispersion parameter theta reached its upper bound: these counts show no overdispersion, so the fit is effectively Poisson" } else if (da_theta <= da_theta_lower) { da_result[da_j, "status"] <- "boundary" - da_result[da_j, "note"] <- "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" + da_result[da_j, "note"] <- if (da_use_gp) + "the shrunken dispersion puts these counts at the extreme-variance end" + else + "the dispersion parameter theta reached its lower bound: the variance is extreme relative to the mean" } else if (length(da_warns) > 0L) { da_result[da_j, "note"] <- paste("R warned:", paste(da_warns, collapse = "; ")) } @@ -319,36 +445,64 @@ A finite number as `Float64`, or `nothing` for R's NA, NaN or an infinity. _num(x) = (ismissing(x) || isnothing(x) || !isfinite(x)) ? nothing : Float64(x) """ - fit_nb(counts, groups, offset; levels) -> DataFrame + fit_nb(counts, groups, offset; levels, dispersion="ml") -> DataFrame + +Fit a negative-binomial GLM with a log link to each column of the samples x +taxa integer matrix `counts`. `offset` contains the natural logarithm of each +sample's size factor. `levels` is `(reference, contrast)`, so the estimate is +the natural-log fold change of `contrast` over `reference`. +`dispersion` names where each taxon's theta comes from: `"ml"` (the default) +takes `MASS::glm.nb`'s maximum-likelihood estimate; `"glmGamPoi"` uses the +`overdispersion_shrinkage_list.dispersion_trend` from `glmGamPoi::glm_gp` on +all supplied taxa in one call, then fits each taxon with `theta = 1 / dispersion` fixed. Both options +use a Wald test of the group coefficient. Names are stripped and matched +without regard to case. -Fit `MASS::glm.nb(y ~ group + offset(offset))` to each column of the samples x -taxa integer matrix `counts`. `levels` is `(reference, contrast)`, so the -estimate is the natural-log fold change of `contrast` over `reference`. Returns one row per taxon with `status` (`ok`, `boundary` or `failed`), `note`, `estimate`, `se`, `statistic`, `pvalue` and `theta`; a failed taxon has -`nothing` for every statistic. Throws `MASSUnavailable` when MASS cannot be -loaded and `RBusyError` when the R runtime stays busy. +`nothing` for every statistic. Boundary fits retain their statistics: theta +at or above 1e7 marks the Poisson end, and theta at or below 1e-8 marks extreme +variance. Constant counts, complete separation, non-convergence and unusable +coefficients produce failed rows. Errors caught from the model-fitting calls +also produce failed rows; a caught `glm_gp` error or an invalid dispersion +vector fails every taxon, with no fallback to `"ml"`. A zero trend marks only +its taxon as a Poisson boundary and withholds NB Wald statistics. Invalid +negative/non-finite trends fail only the affected taxa. + +Throws `ArgumentError` for an unsupported dispersion name or mismatched sample, +group and offset lengths, `MASSUnavailable` when MASS cannot be loaded, +`GlmGamPoiUnavailable` when `"glmGamPoi"` is selected and its package cannot be +loaded, and `RBusyError` when the R runtime stays busy. R errors outside the +caught model-fitting calls propagate. """ function fit_nb(counts::AbstractMatrix{<:Integer}, groups::AbstractVector{<:AbstractString}, - offset::AbstractVector{<:Real}; levels::NTuple{2,String}) + offset::AbstractVector{<:Real}; levels::NTuple{2,String}, + dispersion::AbstractString="ml") + d = strip(String(dispersion)) + lowercase(d) in lowercase.(DISPERSIONS) || throw(ArgumentError( + "dispersion must be one of $(join(DISPERSIONS, ", ")), not '$dispersion'")) + use_gp = lowercase(d) == "glmgampoi" size(counts, 1) == length(groups) == length(offset) || throw(ArgumentError( "counts have $(size(counts, 1)) samples, groups $(length(groups)), offset $(length(offset))")) raw = with_r_lock(; timeout=R_WAIT_SECONDS[]) do RCall.rcopy(RCall.reval("requireNamespace('MASS', quietly = TRUE)")) || throw(MASSUnavailable()) + use_gp && !RCall.rcopy(RCall.reval("requireNamespace('glmGamPoi', quietly = TRUE)")) && + throw(GlmGamPoiUnavailable()) RCall.globalEnv[:da_counts] = Matrix{Int}(counts) RCall.globalEnv[:da_group] = String.(groups) RCall.globalEnv[:da_levels] = collect(levels) RCall.globalEnv[:da_offset] = Float64.(offset) RCall.globalEnv[:da_theta_upper] = THETA_UPPER RCall.globalEnv[:da_theta_lower] = THETA_LOWER + RCall.globalEnv[:da_use_gp] = use_gp try RCall.reval(_FIT_R) DataFrame(RCall.rcopy(RCall.reval("da_result"))) finally RCall.reval("rm(list = intersect(ls(), c('da_counts', 'da_group', 'da_levels', " * - "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_g', 'da_term', " * - "'da_n', 'da_result', 'da_j', 'da_y', 'da_warns', 'da_fit', 'da_co', " * - "'da_row', 'da_theta', 'da_absent'))); invisible(gc())") + "'da_offset', 'da_theta_upper', 'da_theta_lower', 'da_use_gp', 'da_g', 'da_term', " * + "'da_n', 'da_result', 'da_phi', 'da_gp_failed', 'da_gp', 'da_j', 'da_y', " * + "'da_warns', 'da_fit', 'da_co', 'da_row', 'da_theta', 'da_absent'))); invisible(gc())") end end DataFrame(status = String.(raw.status), @@ -589,7 +743,10 @@ labelled `reference` in `groups`, by `config.method`. `nb_glm`: size factors are computed on the full matrix by `config.offset` and enter each negative-binomial model as `log(factor)`; the estimate is a -natural-log fold change. +natural-log fold change. `config.dispersion` selects per-taxon maximum-likelihood +theta (`"ml"`) or fixed theta from `glmGamPoi::glm_gp`'s cross-taxon dispersion trend +(`"glmGamPoi"`), fitted in one call on the taxa passing the prevalence filter. +The result's `config` includes the selected dispersion name. `clr_welch`, in this order: (1) `min_prevalence` chooses the taxa, and taxa with no reads in any sample are always left out; (2) the zeros of those taxa only @@ -612,8 +769,11 @@ effect size and its label. Throws `ArgumentError` for unusable input (including a zero replacement that cannot be made), `ScalingRefusal` when the offset cannot be computed, -`MASSUnavailable` or `ZCompositionsUnavailable` when the R package a method -needs is missing, and an `ErrorException` when no taxon could be fitted at all. +`MASSUnavailable`, `GlmGamPoiUnavailable` or `ZCompositionsUnavailable` when a +required R package is missing, `RBusyError` when the R runtime stays busy, and +an `ErrorException` when no taxon could be fitted at all, including when the +`glm_gp` fit fails. Uncaught R errors from the fitting and zero-replacement +helpers propagate. """ function differential_abundance(counts::AbstractMatrix{<:Real}, samples::AbstractVector{<:AbstractString}, @@ -651,7 +811,8 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, factors = size_factors(counts, samples, config.offset) tested = findall(>=(config.min_prevalence), prevalence) fits = isempty(tested) ? nothing : - fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels) + fit_nb(Int.(counts[:, tested]), groups, log.(factors); levels, + dispersion=config.dispersion) if !isnothing(fits) for (k, j) in enumerate(tested) f = fits[k, :] @@ -662,14 +823,17 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, "pvalue" => f.pvalue, "dispersion_theta" => f.theta)) end end - method_text = "Negative-binomial GLM per taxon (MASS::glm.nb, log link) with a " * + nb_fit = config.dispersion == "ml" ? "MASS::glm.nb" : + "MASS::negative.binomial with dispersion trend from glmGamPoi::glm_gp" + method_text = "Negative-binomial GLM per taxon ($nb_fit, log link) with a " * "$(uppercase(config.offset)) size-factor offset; Wald test of the group " * "coefficient; Benjamini-Hochberg adjustment over the fitted taxa" effect = Dict("key" => "log2_fold_change", "label" => "log2 fold change") size_factor_rows = [Dict("sample" => String(samples[i]), "group" => String(groups[i]), "factor" => factors[i]) for i in eachindex(samples)] config_echo = Dict{String,Any}("method" => config.method, "offset" => config.offset, - "min_prevalence" => config.min_prevalence) + "min_prevalence" => config.min_prevalence, + "dispersion" => config.dispersion) else (n_ref >= 2 && n_con >= 2) || throw(ArgumentError( "Welch's t-test needs at least 2 samples in each group to estimate its variance; " * diff --git a/src/server/routes/analysis.jl b/src/server/routes/analysis.jl index 0dd506a3..2d3af55f 100644 --- a/src/server/routes/analysis.jl +++ b/src/server/routes/analysis.jl @@ -1195,7 +1195,10 @@ end _differential_config(study) -> DifferentialConfig The study's `analysis.differential` settings, resolved through the config -cascade. +cascade. Missing or `nothing` values use the defaults in `DifferentialConfig`, +including `dispersion="ml"`. Invalid settings propagate the constructor's +`ArgumentError`; values that cannot be converted to strings or numbers also +raise their conversion errors. """ function _differential_config(study::String) resolved = _resolved_run_config(study, nothing, nothing, nothing) @@ -1203,7 +1206,8 @@ function _differential_config(study::String) DifferentialConfig(String(val("analysis.differential.offset", "tss")), Float64(val("analysis.differential.min_prevalence", 0.0)); method=String(val("analysis.differential.method", "nb_glm")), - replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65))) + replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65)), + dispersion=String(val("analysis.differential.dispersion", "ml"))) end """ @@ -1282,6 +1286,7 @@ function _differential(study::String, body) e isa ScalingRefusal && return json_error(422, "scaling_refused", sprint(showerror, e)) e isa MASSUnavailable && return json_error(503, "r_unavailable", sprint(showerror, e)) e isa ZCompositionsUnavailable && return json_error(503, "r_unavailable", sprint(showerror, e)) + e isa GlmGamPoiUnavailable && return json_error(503, "r_unavailable", sprint(showerror, e)) e isa ErrorException && return json_error(422, "no_taxon_fitted", e.msg) rethrow() end diff --git a/src/server/routes/config.jl b/src/server/routes/config.jl index 2f927ba1..f4f2ba0b 100644 --- a/src/server/routes/config.jl +++ b/src/server/routes/config.jl @@ -108,6 +108,7 @@ const _ENUM_VALUES = Dict( "analysis.beta.normalisation" => ("hellinger", "none", "rarefy", "srs"), "analysis.differential.method" => Differential.METHODS, "analysis.differential.offset" => Differential.OFFSET_METHODS, + "analysis.differential.dispersion" => Differential.DISPERSIONS, "phylogeny.reference.align.strategy" => Validation.PHYLOGENY_ALIGN_STRATEGIES, "phylogeny.placement.align.strategy" => Validation.PHYLOGENY_ALIGN_STRATEGIES, "phylogeny.reference.trim.method" => Validation.PHYLOGENY_TRIM_METHODS, @@ -130,6 +131,11 @@ factory default; a key whose default is ~ accepts a string, number or boolean. """ function _config_value_error(key::String, value) v = value isa JSON3.Array ? collect(value) : value + if key == "analysis.differential.dispersion" && v isa AbstractString + v = lowercase(strip(v)) + return v in lowercase.(Differential.DISPERSIONS) ? nothing : + "$key must be one of $(join(Differential.DISPERSIONS, ", "))" + end if haskey(_ENUM_VALUES, key) allowed = _ENUM_VALUES[key] v in allowed || return "$key must be one of $(join(filter(!isnothing, collect(allowed)), ", "))" @@ -177,50 +183,172 @@ function _config_value_error(key::String, value) nothing end +# Validate the effective pair after merging a PATCH, before writing any key. +# Resolved entries carry source annotations; global defaults are raw values. +function _differential_pair_error(resolved::AbstractDict, changes::AbstractDict) + keys_ = ("analysis.differential.method", "analysis.differential.dispersion") + any(k -> haskey(changes, k), keys_) || return nothing + value(k, default) = if haskey(changes, k) + changes[k] + else + current = get(resolved, k, default) + current isa NamedTuple ? current.value : current + end + method = value(keys_[1], "nb_glm") + dispersion = value(keys_[2], "ml") + if method isa AbstractString && dispersion isa AbstractString && + lowercase(strip(method)) == "clr_welch" && lowercase(strip(dispersion)) == "glmgampoi" + return "analysis.differential.dispersion: glmGamPoi requires method: nb_glm; use dispersion: ml with clr_welch" + end + nothing +end + +# Resolve every affected study/group/run against the prospective document. +# Enumerate runnable descendants too: a run need not own a pipeline.yml file. +# This is validation evidence, so an unreadable cascade must refuse the save. +function _differential_cascade_error(path::String, document) + data = ServerState.data_dir() + root = dirname(data) + global_path = joinpath(root, "config", "pipeline.yml") + prefix = path == global_path ? String[] : splitpath(relpath(dirname(path), data)) + cache = Dict{String,Any}(path => _flatten(document)) + flat(file) = get!(cache, file) do + _flatten(_load_yml(file)) + end + pair = ("analysis.differential.method", "analysis.differential.dispersion") + function check(values, label) + effective = Dict(pair[1]=>get(values, pair[1], "nb_glm"), + pair[2]=>get(values, pair[2], "ml")) + bad = _differential_pair_error(effective, effective) + isnothing(bad) ? nothing : "$label: $bad" + end + try + base = merge(flat(joinpath(root, "config", "defaults", "pipeline.yml")), flat(global_path)) + if isempty(prefix) + bad = check(base, "global config") + isnothing(bad) || return bad + end + targets = Set{String}() + isempty(prefix) || push!(targets, joinpath(prefix...)) + if length(prefix) >= 2 + # A run edit has no descendants; a group edit can affect its runs. + # Inspect only the edited subtree, so malformed siblings cannot block it. + if length(prefix) == 2 && !get(document, "pool_children", false) + for run in _group_run_names(prefix[1], prefix[2]) + push!(targets, joinpath(prefix..., run)) + end + end + else + for study in _study_names() + isempty(prefix) || first(prefix) == study || continue + push!(targets, study) + for run in _run_names(study) + push!(targets, joinpath(study, run)) + end + for group in _group_names(study) + push!(targets, joinpath(study, group)) + for run in _group_run_names(study, group) + push!(targets, joinpath(study, group, run)) + end + end + end + end + for label in sort!(collect(targets)) + parts = splitpath(label) + length(parts) >= length(prefix) && parts[1:length(prefix)] == prefix || continue + effective = copy(base) + for i in eachindex(parts) + merge!(effective, flat(joinpath(data, parts[1:i]..., "pipeline.yml"))) + end + bad = check(effective, label) + isnothing(bad) || return bad + end + nothing + catch e + "Cannot validate the affected config cascade: $(sprint(showerror, e))" + end +end + +# Validate, prepare the complete document, and publish once under one lock. +# Readers do not take this lock, so per-key renames would expose partial PATCHes. +function _write_config_updates(path::String, changes::AbstractDict; remove=()) + lock(_config_file_lock) do + native = Dict{String,Any}(string(k) => _to_plain(v) for (k, v) in changes) + key = "analysis.differential.dispersion" + if haskey(native, key) && native[key] isa AbstractString + i = findfirst(v -> lowercase(v) == lowercase(strip(native[key])), Differential.DISPERSIONS) + isnothing(i) || (native[key] = Differential.DISPERSIONS[i]) + end + isempty(native) && isempty(remove) && return nothing + cfg = _load_yml(path) + for (k, v) in native + _set_config_value!(cfg, k, v) + end + removed = false + for k in remove + removed |= _remove_config_value!(cfg, k) + end + isempty(native) && !removed && return nothing + pair = ("analysis.differential.method", "analysis.differential.dispersion") + if any(k -> haskey(native, k) || k in remove, pair) + bad = _differential_cascade_error(path, cfg) + isnothing(bad) || return json_error(400, "invalid_config_value", bad) + end + _atomic_write_yaml(path, cfg) + nothing + end +end + function _validate_config_key(dotted_key::String) dotted_key in _ALLOWED_CONFIG_KEYS || error( "Config key '$dotted_key' is not a recognised pipeline option") end -function _write_override(path::String, dotted_key::String, value) +# Shared in-memory setter. A bad later key must not publish earlier changes. +function _set_config_value!(cfg::AbstractDict, dotted_key::String, value) _validate_config_key(dotted_key) - lock(_config_file_lock) do - cfg = _load_yml(path) - parts = split(dotted_key, ".") - d = cfg - for p in parts[1:end-1] - child = get(d, p, nothing) - if child isa Dict - d = child - elseif isnothing(child) - new_dict = Dict() - d[p] = new_dict - d = new_dict - else - error("Config key segment '$p' is not a section; cannot set sub-key") - end + parts = split(dotted_key, ".") + d = cfg + for p in parts[1:end-1] + child = get(d, p, nothing) + if child isa Dict + d = child + elseif isnothing(child) + new_dict = Dict() + d[p] = new_dict + d = new_dict + else + error("Config key segment '$p' is not a section; cannot set sub-key") end - d[parts[end]] = value - _atomic_write_yaml(path, cfg) end + d[parts[end]] = value + cfg end -function _delete_override(path::String, dotted_key::String) - isfile(path) || return +function _write_override(path::String, dotted_key::String, value) lock(_config_file_lock) do - cfg = _load_yml(path) - parts = split(dotted_key, ".") - d = cfg - for p in parts[1:end-1] - haskey(d, p) || return - d = d[p] - d isa Dict || return - end - delete!(d, parts[end]) + cfg = _load_yml(path) + _set_config_value!(cfg, dotted_key, value) _atomic_write_yaml(path, cfg) end end +function _remove_config_value!(cfg::AbstractDict, dotted_key::String) + parts = split(dotted_key, ".") + d = cfg + for p in parts[1:end-1] + haskey(d, p) || return false + d = d[p] + d isa AbstractDict || return false + end + haskey(d, parts[end]) || return false + delete!(d, parts[end]) + true +end + +_delete_override(path::String, dotted_key::String) = + _write_config_updates(path, Dict(); remove=(dotted_key,)) + ## Routes # Default (global) config endpoints @get "/api/v1/config" function(req) @@ -238,16 +366,16 @@ end bad = _config_value_error(string(k), v) isnothing(bad) || return json_error(400, "invalid_config_value", bad) end - for (k, v) in body - _write_override(user_path, string(k), v) - end + bad = _write_config_updates(user_path, body) + isnothing(bad) || return bad json(Dict(k => (; value=v, source="default") for (k, v) in _default_cfg(root))) end @delete "/api/v1/config/{key}" function(req, key::String) root = dirname(ServerState.data_dir()) user_path = joinpath(root, "config", "pipeline.yml") - _delete_override(user_path, key) + bad = _delete_override(user_path, key) + isnothing(bad) || return bad json(Dict(k => (; value=v, source="default") for (k, v) in _default_cfg(root))) end @@ -502,9 +630,8 @@ end isnothing(bad) || return json_error(400, "invalid_config_value", bad) end path = joinpath(ServerState.data_dir(), study, "pipeline.yml") - for (k, v) in body - _write_override(path, string(k), v) - end + bad = _write_config_updates(path, body) + isnothing(bad) || return bad json(_resolve_config(study)) end @@ -512,7 +639,8 @@ end study in _study_names() || return json_error(404, "study_not_found", "Study '$study' not found") path = joinpath(ServerState.data_dir(), study, "pipeline.yml") - _delete_override(path, key) + bad = _delete_override(path, key) + isnothing(bad) || return bad json(_resolve_config(study)) end @@ -586,9 +714,8 @@ end isnothing(bad) || return json_error(400, "invalid_config_value", bad) end path = joinpath(ServerState.data_dir(), study, group, "pipeline.yml") - for (k, v) in body - _write_override(path, string(k), v) - end + bad = _write_config_updates(path, body) + isnothing(bad) || return bad json(_resolve_config(study, nothing, group)) end @@ -600,7 +727,8 @@ end group in _group_names(study) || return json_error(404, "group_not_found", "Group '$group' not found") path = joinpath(ServerState.data_dir(), study, group, "pipeline.yml") - _delete_override(path, key) + bad = _delete_override(path, key) + isnothing(bad) || return bad json(_resolve_config(study, nothing, group)) end @@ -634,9 +762,8 @@ end isnothing(bad) || return json_error(400, "invalid_config_value", bad) end path = joinpath(ServerState.data_dir(), study, run_path, "pipeline.yml") - for (k, v) in body - _write_override(path, string(k), v) - end + bad = _write_config_updates(path, body) + isnothing(bad) || return bad json(_resolve_config(study, run, group)) end @@ -648,6 +775,7 @@ end group = let g = _req_group(req); isnothing(g) ? _run_group(study, run) : g end run_path = isnothing(group) ? run : joinpath(group, run) path = joinpath(ServerState.data_dir(), study, run_path, "pipeline.yml") - _delete_override(path, key) + bad = _delete_override(path, key) + isnothing(bad) || return bad json(_resolve_config(study, run, group)) end diff --git a/test/integration/test_server.jl b/test/integration/test_server.jl index 184e70fa..4cc02029 100644 --- a/test/integration/test_server.jl +++ b/test/integration/test_server.jl @@ -65,6 +65,22 @@ r4 = HTTP.get("http://localhost:$port/api/v1/databases"; status_exception=false) @test r4.status == 200 + ## Configuration persistence through the real listener/middleware. + config_url = "http://localhost:$port/api/v1/config" + headers = ["Content-Type"=>"application/json", "Origin"=>"http://localhost:$port"] + for (submitted, canonical) in ((" ML ", "ml"), (" GLMGAMPOI ", "glmGamPoi")) + response = HTTP.patch(config_url, headers, + JSON3.write(Dict("analysis.differential.method"=>"nb_glm", + "analysis.differential.dispersion"=>submitted)); status_exception=false) + @test response.status == 200 + @test JSON3.read(String(response.body))["analysis.differential.dispersion"].value == canonical + response = HTTP.get(config_url; status_exception=false) + @test response.status == 200 + @test JSON3.read(String(response.body))["analysis.differential.dispersion"].value == canonical + saved = YAML.load_file(joinpath(tmp_root, "config", "pipeline.yml")) + @test saved["analysis"]["differential"]["dispersion"] == canonical + end + ## Path traversal guard -> 403 r5 = HTTP.get("http://localhost:$port/files/smoke_study/runs/run_A/../../../etc/passwd"; status_exception=false) diff --git a/test/unit/test_differential.jl b/test/unit/test_differential.jl index 3b034896..dcab0549 100644 --- a/test/unit/test_differential.jl +++ b/test/unit/test_differential.jl @@ -10,7 +10,8 @@ using MetaManifold using MetaManifold.Differential using RCall using Test -using DuckDB, DBInterface, JSON3 +using DuckDB, DBInterface, JSON3, HTTP, YAML +using OrderedCollections: OrderedDict """ _r_has(pkg) -> Bool @@ -29,6 +30,8 @@ end const HAVE_R = _r_has("stats") const HAVE_MASS = HAVE_R && _r_has("MASS") HAVE_MASS || @info "R or MASS unavailable - the negative-binomial fit tests are SKIPPED" +const HAVE_GLMGAMPOI = HAVE_MASS && _r_has("glmGamPoi") +HAVE_GLMGAMPOI || @info "R, MASS or glmGamPoi unavailable - the shrunken-dispersion fit tests are SKIPPED" const HAVE_ZCOMP = HAVE_R && _r_has("zCompositions") HAVE_ZCOMP || @info "R or zCompositions unavailable - the clr_welch tests are SKIPPED" @@ -172,6 +175,14 @@ end @test_throws ArgumentError DifferentialConfig(; method="ancombc") # The pre-review name, never released, is not accepted. @test_throws ArgumentError DifferentialConfig(; method="clr_lm") + # The dispersion option: ml is the default, glmGamPoi is the alternative, + # and it is a negative-binomial setting only. + @test DifferentialConfig().dispersion == "ml" + @test DifferentialConfig(dispersion="glmGamPoi").dispersion == "glmGamPoi" + @test DifferentialConfig(dispersion=" GLMGAMPOI ").dispersion == "glmGamPoi" + @test_throws ArgumentError DifferentialConfig(dispersion="shrinkage") + @test_throws ArgumentError DifferentialConfig(method="clr_welch", dispersion="glmGamPoi") + @test DifferentialConfig(method="clr_welch").dispersion == "ml" for d in (0, 1, -0.5, NaN) @test_throws ArgumentError DifferentialConfig(; method="clr_welch", replacement_delta=d) end @@ -211,6 +222,9 @@ end reference="A", contrast="B") byt = Dict(r["taxon"] => r for r in res["rows"]) @test res["status"] == "ok" + @test res["method"] == "Negative-binomial GLM per taxon (MASS::glm.nb, log link) with a " * + "TSS size-factor offset; Wald test of the group coefficient; " * + "Benjamini-Hochberg adjustment over the fitted taxa" @test res["diagnostics"]["n_tested"] == 3 @test all(r -> r["status"] == "ok", res["rows"]) @test isapprox(byt["t_up"]["estimate"], log(4); atol=0.5) @@ -249,6 +263,85 @@ end end end + @testset "glmGamPoi matches a direct negative-binomial GLM with fixed dispersion" begin + if !HAVE_GLMGAMPOI + @test get(ENV, "CI", "false") != "true" # the pinned dependency is required in CI + @test_skip false + else + res = differential_abundance(effect, effect_samples, effect_taxa, effect_groups; + reference="A", contrast="B", + config=DifferentialConfig(dispersion="glmGamPoi")) + @test res["status"] == "ok" + @test res["method"] == "Negative-binomial GLM per taxon (MASS::negative.binomial " * + "with dispersion trend from glmGamPoi::glm_gp, log link) with a " * + "TSS size-factor offset; Wald test of the group coefficient; " * + "Benjamini-Hochberg adjustment over the fitted taxa" + byt = Dict(r["taxon"] => r for r in res["rows"]) + off = log.(tss_factors(effect, effect_samples)) + g = effect_groups + phi = rcopy(R""" + local({ + gp <- glmGamPoi::glm_gp(t($effect), + design = model.matrix(~ factor($g, levels = c("A", "B"))), + size_factors = exp($off), overdispersion_shrinkage = TRUE) + as.numeric(gp$overdispersion_shrinkage_list$dispersion_trend) + }) + """) + for (j, t) in enumerate(effect_taxa) + y = effect[:, j] + theta = 1 / phi[j] + direct = rcopy(R""" + local({ + fit <- glm($y ~ factor($g, levels = c("A", "B")) + offset($off), + family = MASS::negative.binomial(theta = $theta), + control = glm.control(maxit = 100)) + unname(summary(fit, dispersion = 1)$coefficients[2, ]) + }) + """) + r = byt[t] + @test r["status"] == "ok" + @test r["estimate"] ≈ direct[1] rtol=1e-6 + @test r["standard_error"] ≈ direct[2] rtol=1e-6 + @test r["statistic"] ≈ direct[3] rtol=1e-6 + @test r["pvalue"] ≈ direct[4] rtol=1e-6 + @test r["dispersion_theta"] ≈ theta rtol=1e-6 + end + end + end + + @testset "glmGamPoi invalid trends are isolated per taxon" begin + if !HAVE_GLMGAMPOI + @test_skip false + else + MetaManifold.RRuntime.with_r_lock() do + reval(raw""" + test_original_gp <- get("glm_gp", asNamespace("glmGamPoi")) + test_phi <- 0 + assignInNamespace("glm_gp", function(...) { + fit <- test_original_gp(...) + fit$overdispersion_shrinkage_list$dispersion_trend[1] <- test_phi + fit + }, ns="glmGamPoi") + """) + try + off = log.(tss_factors(effect, effect_samples)) + for (phi, status) in ((0.0, "boundary"), (-1.0, "failed"), (NaN, "failed")) + RCall.globalEnv[:test_phi] = phi + rows = fit_nb(effect, effect_groups, off; levels=("A", "B"), dispersion="glmGamPoi") + @test rows.status[1] == status + @test isnothing(rows.pvalue[1]) + @test all(rows.status[2:end] .== "ok") + end + finally + reval(raw""" + assignInNamespace("glm_gp", test_original_gp, ns="glmGamPoi") + rm(test_original_gp, test_phi) + """) + end + end + end + end + @testset "Degenerate and filtered taxa are reported, not dropped" begin if !HAVE_MASS @test_skip false @@ -353,6 +446,205 @@ end @test isnothing(SV._config_value_error("analysis.differential.min_prevalence", 0.1)) @test isnothing(SV._config_value_error("analysis.differential.method", "clr_welch")) @test !isnothing(SV._config_value_error("analysis.differential.method", "ancombc")) + for dispersion in ("ml", "glmGamPoi", " ML ", " GLMGAMPOI ") + @test isnothing(SV._config_value_error("analysis.differential.dispersion", dispersion)) + end + for dispersion in ("shrinkage", "", nothing, 1, true, ["ml"]) + @test !isnothing(SV._config_value_error("analysis.differential.dispersion", dispersion)) + end + end + + @testset "Config rejects incompatible effective dispersion pairs" begin + SV = MetaManifold.Server + keym, keyd = "analysis.differential.method", "analysis.differential.dispersion" + @test SV._differential_pair_error(Dict(keym=>"clr_welch"), Dict(keyd=>"glmGamPoi")) !== nothing + @test SV._differential_pair_error(Dict(keyd=>"glmGamPoi"), Dict(keym=>"clr_welch")) !== nothing + @test SV._differential_pair_error(Dict(keym=>(; value="clr_welch",source="study")), Dict(keyd=>" GLMGAMPOI ")) !== nothing + @test SV._differential_pair_error(Dict(keym=>"clr_welch"), Dict(keym=>"nb_glm",keyd=>"glmGamPoi")) === nothing + @test SV._differential_pair_error(Dict(keyd=>"glmGamPoi"), Dict(keym=>"clr_welch",keyd=>"ml")) === nothing + _da_fixture() do + root = SV.ServerState._root[] + path = joinpath(root,"data","studyD","pipeline.yml") + write(path,"analysis:\n differential:\n method: clr_welch\n") + before = read(path,String) + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", "/api/v1/studies/studyD/config", ["Content-Type"=>"application/json"], JSON3.write(Dict(keyd=>"glmGamPoi")))) + @test response.status == 400 + @test read(path,String) == before + # A batch can change both options together, irrespective of key order. + batch = Dict(keyd=>"glmGamPoi", keym=>"nb_glm") + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", "/api/v1/studies/studyD/config", ["Content-Type"=>"application/json"], JSON3.write(batch))) + @test response.status == 200 + before = read(path,String) + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", "/api/v1/studies/studyD/config", ["Content-Type"=>"application/json"], JSON3.write(Dict(keym=>"clr_welch")))) + @test response.status == 400 + @test read(path,String) == before + end + end + + @testset "Config PATCH publishes canonical, complete documents" begin + SV = MetaManifold.Server + keym, keyd = "analysis.differential.method", "analysis.differential.dispersion" + _da_fixture() do + root = SV.ServerState._root[] + group_dir = joinpath(root, "data", "studyD", "groupG") + run_dir = joinpath(group_dir, "runG") + mkpath(run_dir) + touch(joinpath(group_dir, ".group")) + touch(joinpath(run_dir, "sample_R1.fastq.gz")) + touch(joinpath(run_dir, "sample_R2.fastq.gz")) + scopes = [("/api/v1/config", joinpath(root, "config", "pipeline.yml")), + ("/api/v1/studies/studyD/config", joinpath(root, "data", "studyD", "pipeline.yml")), + ("/api/v1/studies/studyD/groups/groupG/config", joinpath(group_dir, "pipeline.yml")), + ("/api/v1/studies/studyD/runs/runG/config?group=groupG", joinpath(run_dir, "pipeline.yml"))] + for (url, path) in scopes + for (submitted, expected) in ((" ML ", "ml"), (" GLMGAMPOI ", "glmGamPoi")) + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", url, + ["Content-Type"=>"application/json"], JSON3.write(Dict(keym=>"nb_glm", keyd=>submitted, "seed"=>456)))) + @test response.status == 200 + @test JSON3.read(String(response.body))[keyd].value == expected + @test YAML.load_file(path)["analysis"]["differential"]["dispersion"] == expected + response = SV.Oxygen.internalrequest(HTTP.Request("GET", url)) + @test response.status == 200 + @test JSON3.read(String(response.body))[keyd].value == expected + @test YAML.load_file(path)["seed"] == 456 + end + end + # Both transitions must preserve unrelated fields. Explicit ordering + # exercises the formerly invalid intermediate clr_welch/glmGamPoi pair. + path = scopes[2][2] + for changes in (OrderedDict(keyd=>"ml", keym=>"clr_welch"), + OrderedDict(keyd=>"glmGamPoi", keym=>"nb_glm"), + OrderedDict(keym=>"clr_welch", keyd=>"ml")) + @test SV._write_config_updates(path, changes) === nothing + saved = YAML.load_file(path) + @test saved["seed"] == 456 + @test saved["analysis"]["differential"]["method"] == changes[keym] + @test saved["analysis"]["differential"]["dispersion"] == changes[keyd] + end + before = read(path, String) + bad = SV._write_config_updates(path, Dict(keyd=>"glmGamPoi")) + @test bad.status == 400 + @test read(path, String) == before + + run_url, run_path = scopes[4] + before_run = read(run_path, String) + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", run_url, + ["Content-Type"=>"application/json"], JSON3.write(Dict(keym=>"clr_welch")))) + @test response.status == 400 + @test read(run_path, String) == before_run + + # The first valid key must stay unpublished when a later setter fails. + YAML.write_file(path, Dict("phylogeny"=>"not-a-section", "seed"=>456)) + before = read(path, String) + @test_throws ErrorException SV._write_config_updates(path, + OrderedDict("seed"=>789, "phylogeny.reference.trim.method"=>"none")) + @test read(path, String) == before + end + end + + @testset "Parent PATCH validates affected descendants before publication" begin + SV = MetaManifold.Server + keym, keyd = "analysis.differential.method", "analysis.differential.dispersion" + _da_fixture() do + root = SV.ServerState._root[] + global_path = joinpath(root, "config", "pipeline.yml") + study_path = joinpath(root, "data", "studyD", "pipeline.yml") + group_dir = joinpath(dirname(study_path), "groupG") + run_dir = joinpath(group_dir, "runG") + mkpath(run_dir); touch(joinpath(group_dir, ".group")) + touch(joinpath(run_dir, "sample_R1.fastq.gz")) + group_path, run_path = joinpath(group_dir, "pipeline.yml"), joinpath(run_dir, "pipeline.yml") + save(path, values) = YAML.write_file(path, Dict("analysis"=>Dict("differential"=>values))) + patch(url, changes) = SV.Oxygen.internalrequest(HTTP.Request("PATCH", url, + ["Content-Type"=>"application/json"], JSON3.write(changes))) + save(global_path, Dict("method"=>"nb_glm", "dispersion"=>"ml")) + save(study_path, Dict("dispersion"=>"glmGamPoi")) + before = read(global_path, String) + response = patch("/api/v1/config", Dict(keym=>"clr_welch", "seed"=>789)) + @test response.status == 400 + @test occursin("studyD", JSON3.read(String(response.body)).message) + @test read(global_path, String) == before + # A deeper complete pair shields that subtree from a parent edit. + save(study_path, Dict("method"=>"nb_glm", "dispersion"=>"glmGamPoi")) + @test patch("/api/v1/config", Dict(keym=>"clr_welch", keyd=>"ml")).status == 200 + + save(global_path, Dict("method"=>"nb_glm", "dispersion"=>"ml")) + save(study_path, Dict("method"=>"nb_glm", "dispersion"=>"ml")) + save(group_path, Dict("dispersion"=>"glmGamPoi")) + before = read(study_path, String) + @test patch("/api/v1/studies/studyD/config", Dict(keym=>"clr_welch")).status == 400 + @test read(study_path, String) == before + save(group_path, Dict("method"=>"nb_glm", "dispersion"=>"ml")) + save(run_path, Dict("dispersion"=>"glmGamPoi")) + before = read(group_path, String) + group_url = "/api/v1/studies/studyD/groups/groupG/config" + response = patch(group_url, Dict(keym=>"clr_welch")) + @test response.status == 400 + @test occursin("runG", JSON3.read(String(response.body)).message) + @test read(group_path, String) == before + save(run_path, Dict("method"=>"nb_glm", "dispersion"=>"glmGamPoi")) + @test patch(group_url, Dict(keym=>"clr_welch")).status == 200 + + # A run without an override file still inherits a validated cascade. + run_a = joinpath(root, "data", "studyD", "runA", "pipeline.yml") + rm(run_a) + @test patch("/api/v1/studies/studyD/config", Dict(keyd=>"glmGamPoi")).status == 200 + response = SV.Oxygen.internalrequest(HTTP.Request("GET", "/api/v1/studies/studyD/runs/runA/config")) + @test response.status == 200 + @test JSON3.read(String(response.body))[keyd].value == "glmGamPoi" + + # A malformed sibling is outside a run edit's affected subtree. + write(joinpath(root,"data","studyD","runB","pipeline.yml"), "analysis: [broken") + @test SV._write_config_updates(run_a, Dict(keyd=>"ml")) === nothing + end + end + + @testset "DELETE validates inherited values at every config scope" begin + SV = MetaManifold.Server + keym, keyd = "analysis.differential.method", "analysis.differential.dispersion" + for scope in 1:4, deleted in (keym, keyd, "valid") + _da_fixture() do + root = SV.ServerState._root[] + group_dir = joinpath(root, "data", "studyD", "groupG") + run_dir = joinpath(group_dir, "runG") + mkpath(run_dir); touch(joinpath(group_dir, ".group")) + touch(joinpath(run_dir, "sample_R1.fastq.gz")) + factory = joinpath(root, "config", "defaults", "pipeline.yml") + mkpath(dirname(factory)) + paths = [joinpath(root,"config","pipeline.yml"), + joinpath(root,"data","studyD","pipeline.yml"), + joinpath(group_dir,"pipeline.yml"), joinpath(run_dir,"pipeline.yml")] + urls = ["/api/v1/config", "/api/v1/studies/studyD/config", + "/api/v1/studies/studyD/groups/groupG/config", + "/api/v1/studies/studyD/runs/runG/config"] + query = scope == 4 ? "?group=groupG" : "" + method, dispersion = deleted == keym ? ("clr_welch", "ml") : + deleted == keyd ? ("nb_glm", "glmGamPoi") : ("nb_glm", "ml") + doc(m, d, seed) = Dict("seed"=>seed, "analysis"=>Dict("differential"=>Dict("method"=>m, "dispersion"=>d))) + for path in [factory; paths] + YAML.write_file(path, doc(method, dispersion, 123)) + end + target_method, target_dispersion = deleted == keym ? ("nb_glm", "glmGamPoi") : + deleted == keyd ? ("clr_welch", "ml") : ("nb_glm", "ml") + YAML.write_file(paths[scope], doc(target_method, target_dispersion, 456)) + before = Dict(p=>read(p,String) for p in [factory; paths]) + if deleted != "valid" + response = SV.Oxygen.internalrequest(HTTP.Request("DELETE", "$(urls[scope])/$deleted$query")) + @test response.status == 400 + @test JSON3.read(String(response.body)).error == "invalid_config_value" + @test all(read(p,String) == text for (p,text) in before) + else + for (key, expected) in ((keym,"nb_glm"), (keyd,"ml"), ("seed",123)) + response = SV.Oxygen.internalrequest(HTTP.Request("DELETE", "$(urls[scope])/$key$query")) + @test response.status == 200 + @test JSON3.read(String(response.body))[key].value == expected + end + unchanged = read(paths[scope], String) + @test SV.Oxygen.internalrequest(HTTP.Request("DELETE", "$(urls[scope])/seed$query")).status == 200 + @test read(paths[scope], String) == unchanged + end + end + end end @testset "CLR transform" begin @@ -720,4 +1012,59 @@ end @test rcopy(Bool, reval("requireNamespace('zCompositions', quietly = TRUE)")) == HAVE_ZCOMP end end + + ## Without glmGamPoi, dispersion: glmGamPoi refuses with + # GlmGamPoiUnavailable and the route answers 503. As above, a + # `requireNamespace` bound in R's global environment reports glmGamPoi + # missing and defers to base for every other package, so the production + # probe sees the package as absent on any machine. MASS is probed first + # and must be real, so this testset needs it. + @testset "nb_glm without glmGamPoi refuses with GlmGamPoiUnavailable and a 503" begin + if !HAVE_MASS + @test_skip false # no MASS means the ml path's own guard fires first + else + reval("""requireNamespace <- function(package, ...) { + if (identical(package, "glmGamPoi")) FALSE + else base::requireNamespace(package, ...) + }""") + try + @test !rcopy(Bool, reval("requireNamespace('glmGamPoi', quietly = TRUE)")) + e = try + fit_nb([5 6; 7 3; 9 4; 4 8], ["A", "A", "B", "B"], [0.0, 0.0, 0.0, 0.0]; + levels=("A", "B"), dispersion="glmGamPoi") + nothing + catch err + err + end + @test e isa GlmGamPoiUnavailable + @test occursin("renv::restore()", sprint(showerror, e)) + _da_fixture() do + root = MetaManifold.Server.ServerState._root[] + write(joinpath(root, "data", "studyD", "pipeline.yml"), + "analysis:\n differential:\n dispersion: glmGamPoi\n") + response = _da_post("studyD", Dict("runs" => [Dict("run" => "runA"), Dict("run" => "runB")], + "table" => "merged", "rank" => "Genus")) + body = JSON3.read(String(response.body)) + @test response.status == 503 + @test body.error == "r_unavailable" + @test occursin("glmGamPoi", body.message) + end + # The override defers to base for everything else, so the default + # dispersion must still fit normally with it in place. The + # injected-effect dataset, not the minimal matrix above: + # MEASURED with MASS::glm.nb, with 4 samples of small counts the + # alternating fit runs theta to its upper bound (status boundary) + # and one taxon's fit even produced NaNs (status failed), so that + # data never reaches status ok on any machine; the dataset + # "Negative-binomial fit recovers an injected effect" already + # proves fits. + r = fit_nb(effect, effect_groups, log.(tss_factors(effect, effect_samples)); + levels=("A", "B")) + @test all(r.status .== "ok") + finally + reval("rm(requireNamespace)") + end + @test !rcopy(Bool, reval("exists('requireNamespace', envir = globalenv(), inherits = FALSE)")) + end + end end diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index a3053b97..b0b2f8b7 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -187,18 +187,13 @@ end ci = YAML.load_file(CI_PATH) job = ci["jobs"]["test"] - # Julia is the one version CI cannot read from the pin file, because nothing can - # be read before Julia exists. It is therefore duplicated, and this is the test - # that makes the duplication safe. - # - # Read from the `Set up Julia` step rather than from a matrix: the matrix was - # removed because GitHub appends a matrix combination to the posted check name - # (see "a required check name is a stable identifier" below). The step is located - # by its `uses:` rather than by index, so reordering the steps cannot make this - # assertion quietly vanish. - setup = ci_step_using(job, "julia-actions/setup-julia") - @test setup !== nothing - @test setup["with"]["version"] == pins["runtimes"]["julia"]["version"] + # The repo-owned installer reads the same committed version, URL and SHA. + setup = only(filter(step -> get(step, "name", "") == "Set up Julia", job["steps"])) + @test setup["run"] == "bash scripts/install-ci-julia.sh" + script = joinpath(REPO_ROOT, "scripts", "install-ci-julia.sh") + declared = split(strip(read(`bash $script --print-pin`, String)), '\n') + archive = pins["runtimes"]["julia"]["archives"]["linux-x86_64"] + @test declared == [pins["runtimes"]["julia"]["version"], archive["url"], archive["sha256"]] # A floating runner would carry the R apt pin, which names a 24.04 build, off to # whatever the next LTS ships. @@ -253,6 +248,82 @@ end end end + @testset "required relevance gates classify changes and fail closed" begin + jobs = YAML.load_file(CI_PATH)["jobs"] + required = ["Code relevance", "Julia tests", "Agda proofs", "Occupancy relevance", + "Occupancy disagreement test", "Occupancy typed peak"] + @test all(name -> count(j -> get(j, "name", "") == name, values(jobs)) == 1, required) + @test jobs["disagreement"]["needs"] == ["occupancy_gate", "test"] + @test startswith(jobs["disagreement"]["if"], "always() &&") + steps = jobs["test"]["steps"] + run_tests = only(filter(s -> get(s, "name", "") == "Run tests", steps)) + @test haskey(run_tests["env"], "OCCUPANCY_RESULTS") + @test occursin("command -v vsearch", run_tests["run"]) + @test any(s -> get(s, "name", "") == "Validate measured occupancy grades", steps) + + # Execute the actual YAML scripts against real diffs, including a rename + # out of a watched path and a pathname containing a newline. A missing + # comparison commit must fail rather than authorise skipped tests. + mktempdir() do dir + git(args...) = run(pipeline(Cmd(["git", "-C", dir, args...]); stdout=devnull, stderr=devnull)) + git("init") + git("config", "user.name", "CI gate test") + git("config", "user.email", "ci@example.invalid") + mkpath(joinpath(dir, "docs", "occupancy")) + write(joinpath(dir, "docs", "occupancy", "watched.agda"), "fixture") + git("add", "."); git("commit", "-m", "base") + base = strip(read(`git -C $dir rev-parse HEAD`, String)) + function verdict(id, comparison=base) + script = only(filter(s -> get(s, "id", "") == "pr", jobs[id]["steps"]))["run"] + out = joinpath(dir, "gate-output") + write(out, "") + cmd = addenv(Cmd(Cmd(["bash", "-e", "-o", "pipefail", "-c", script]); dir=dir), + "BASE_SHA" => comparison, "RUNNER_TEMP" => dir, "GITHUB_OUTPUT" => out) + ok = success(pipeline(cmd; stdout=devnull, stderr=devnull)) + (ok, strip(read(out, String))) + end + for id in ("gate", "occupancy_gate") + @test verdict(id) == (true, "relevant=false") + @test verdict(id, "not-a-commit") == (false, "") + end + write(joinpath(dir, "README.md"), "docs only") + git("add", "README.md"); git("commit", "-m", "docs") + for id in ("gate", "occupancy_gate") + @test verdict(id) == (true, "relevant=false") + end + git("mv", "docs/occupancy/watched.agda", "elsewhere.agda") + git("commit", "-m", "rename away") + for id in ("gate", "occupancy_gate") + @test verdict(id) == (true, "relevant=true") + end + git("reset", "--hard", base) + write(joinpath(dir, "docs", "occupancy", "unusual\nname.agda"), "fixture") + git("add", "docs"); git("commit", "-m", "unusual path") + for id in ("gate", "occupancy_gate") + @test verdict(id) == (true, "relevant=true") + end + git("reset", "--hard", base) + mkpath(joinpath(dir, "src", "analysis")) + write(joinpath(dir, "src", "analysis", "differential.jl"), "fixture") + git("add", "src"); git("commit", "-m", "analysis only") + @test verdict("gate") == (true, "relevant=true") + @test verdict("occupancy_gate") == (true, "relevant=false") + for driver in ("install.jl", "install.sh") + git("reset", "--hard", base) + write(joinpath(dir, driver), "fixture") + git("add", driver); git("commit", "-m", "installer only") + @test verdict("gate") == (true, "relevant=true") + @test verdict("occupancy_gate") == (true, "relevant=false") + end + report = only(jobs["disagreement"]["steps"])["run"] + for result in ("success", "failure", "cancelled", "skipped") + cmd = addenv(`bash -e -c $report`, "TEST_RESULT" => result, + "GITHUB_STEP_SUMMARY" => joinpath(dir, "summary")) + @test success(pipeline(cmd; stdout=devnull, stderr=devnull)) == (result == "success") + end + end + end + @testset "a required check name is a stable identifier" begin ci = YAML.load_file(CI_PATH) From 01055c8cc56993d02e0dc1b27f173fd4b1b6afd8 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:48:19 +0100 Subject: [PATCH 08/35] =?UTF-8?q?ci(fork):=20governance=20lane=20=E2=80=94?= =?UTF-8?q?=20DEED=20syntax=20and=20the=20Hypatia=20verdict=20grid=20(#190?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Demonstration of the fork governance lane, as designed in hyperpolymath/standards `docs/VERDICT-GRID-SPEC.adoc` (standards branch `feat/verdict-grid`). Nothing here is proposed upstream; upstream `ci.yml` is untouched. **What this adds to the fork** - `.github/workflows/governance.yml`: two reusable lanes from standards, pinned, plus the two caller-side jobs the reusable cannot carry itself (posting the grid comment on a PR, publishing badge endpoints to an orphan `governance-badges` branch on pushes to `main`). - `MetaManifold-WebUI_chora.deed`: the fork's repo deed, linted by the DEED lane (`bun deed_lint.js` says OK locally). - `.gitbot-fleet/enrolment.json`: which fleet members apply here, so a member without a report-back is orange and one not enrolled is grey with its reason. - README: three fork badges (workflow status, DEED syntax endpoint, Hypatia grid endpoint) and one sentence on the grading. **What the grid shows on this PR** Hypatia, panic-attack and pons asinorum rows graded by their worst finding; seven enrolled fleet members in orange until each bot writes `.gitbot-fleet/report-backs/.json` (the envelope is in the spec); cipherbot and sustainabot grey. Expected overall state on this tree: purple, from Hypatia's `cicd_rules/banned_language_file` on the upstream TypeScript frontend, which is upstream's settled decision and stays visible rather than baselined away. **Pinning** The Hypatia lane is pinned to the standards branch for this demonstration; it becomes a commit SHA once hyperpolymath/standards#1295 merges. --- .gitbot-fleet/enrolment.json | 14 +++ .github/workflows/governance.yml | 154 +++++++++++++++++++++++++++++++ .gitignore | 4 + MetaManifold-WebUI_chora.deed | 29 ++++++ README.md | 6 ++ 5 files changed, 207 insertions(+) create mode 100644 .gitbot-fleet/enrolment.json create mode 100644 .github/workflows/governance.yml create mode 100644 MetaManifold-WebUI_chora.deed diff --git a/.gitbot-fleet/enrolment.json b/.gitbot-fleet/enrolment.json new file mode 100644 index 00000000..4f8c6841 --- /dev/null +++ b/.gitbot-fleet/enrolment.json @@ -0,0 +1,14 @@ +{ + "schema": 1, + "reporters": { + "rhodibot": { "title": "rhodibot (RSR compliance)", "enrolled": true }, + "panicbot": { "title": "panicbot (panic recovery, error handling)", "enrolled": true }, + "echidnabot": { "title": "echidnabot (formal verification: proofs/)", "enrolled": true }, + "seambot": { "title": "seambot (integration health: API routes)", "enrolled": true }, + "finishingbot": { "title": "finishingbot (release readiness)", "enrolled": true }, + "accessibilitybot": { "title": "accessibilitybot (WCAG on the React UI)", "enrolled": true }, + "glambot": { "title": "glambot (presentation quality)", "enrolled": true }, + "cipherbot": { "enrolled": false, "reason": "no cryptography is implemented in this repository" }, + "sustainabot": { "enrolled": false, "reason": "fleet adapter slot reserved; no adapter exists yet (gitbot-fleet README)" } + } +} diff --git a/.github/workflows/governance.yml b/.github/workflows/governance.yml new file mode 100644 index 00000000..20910d5b --- /dev/null +++ b/.github/workflows/governance.yml @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Fork governance lane. Two reusable lanes from hyperpolymath/standards, each +# pinned to a commit: DEED syntax for the repo deed, and the Hypatia verdict +# grid (Hypatia, panic-attack, pons asinorum and the gitbot-fleet report-backs +# under .gitbot-fleet/). This workflow is the fork's; nothing in it is +# proposed upstream, and upstream's ci.yml is untouched. +# +# The reusable renders the grid; the two jobs below post it on the pull +# request and publish the badges, because a called workflow may not request +# permissions its caller does not grant. +name: Governance (fork) + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + deed: + name: DEED syntax + uses: hyperpolymath/standards/.github/workflows/deed-conformance.yml@75bd5994470661a099df562d6d5ad2fe744dbd66 + permissions: + contents: read + with: + require-files: true + + hypatia: + name: Hypatia verdict grid + uses: hyperpolymath/standards/.github/workflows/hypatia-scan-reusable.yml@feat/verdict-grid + permissions: + actions: read + contents: read + security-events: write + secrets: + HYPATIA_SCAN_PAT: ${{ secrets.HYPATIA_SCAN_PAT }} + + verdict: + name: Post the verdict grid + needs: [deed, hypatia] + if: ${{ always() && github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + pull-requests: write + steps: + - name: Download the verdict grid + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: verdict-grid + path: verdict-grid + + - name: Update the compact PR verdict + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + DEED_RESULT: ${{ needs.deed.result }} + HYPATIA_RESULT: ${{ needs.hypatia.result }} + HYPATIA_VERDICT: ${{ needs.hypatia.outputs.verdict }} + run: | + set -euo pipefail + # The DEED lane is a conformance row: pass is green, failure red, + # anything else (skipped, cancelled, no result) orange. The grid + # itself comes from the artifact; a missing artifact is orange. + case "$DEED_RESULT" in + success) deed='🟢 conformant' ;; + failure) deed='🔴 failing' ;; + *) deed="🟠 indeterminate ($DEED_RESULT)" ;; + esac + { + echo '' + echo '# Fork governance' + echo + echo "| Lane | Status | Run |" + echo "|---|---|---|" + echo "| DEED syntax (repo deed) | $deed | [run $GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) |" + echo "| Hypatia verdict grid | ${HYPATIA_VERDICT:-orange} (job: $HYPATIA_RESULT) | same run |" + echo + if [ -s verdict-grid/report.md ]; then + cat verdict-grid/report.md + else + echo '🟠 indeterminate · no verdict-grid artifact was produced by the Hypatia lane; see the run.' + fi + } > "$RUNNER_TEMP/body.md" + # One comment per PR, updated in place; newest run wins. + comment_id=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + | jq -r '[.[][] | select(.user.login == "github-actions[bot]" and (.body | contains(" + + +# Advanced zero-handling benchmark + +This opt-in benchmark measures the reference-backed replacement methods on +100, 1,000 and 10,000 taxa with 24 samples. An additional `--dispersion` flag +measures per-taxon ML and glmGamPoi fixed-dispersion NB fits. The dispersion +flag is separate because 10,000 R GLM fits can take substantially longer than +zero replacement. + +Restore `renv.lock` first: zCompositions is required for replacement and both +MASS and glmGamPoi are required for `--dispersion`. + +```sh +# Replacement only; JSON goes to stdout. +julia --project=. bench/advanced_zero_handling/benchmark.jl + +# Include NB dispersion fits and save the measurements outside Git. +julia --project=. bench/advanced_zero_handling/benchmark.jl \ + --dispersion --output=/tmp/advanced-zero-current.json + +# Quick smoke run (not a substitute for the full three-size benchmark). +julia --project=. bench/advanced_zero_handling/benchmark.jl --sizes=100 + +# Fail if an existing method takes more than 110% of its baseline runtime. +julia --project=. bench/advanced_zero_handling/benchmark.jl \ + --dispersion --baseline=/tmp/advanced-zero-baseline.json +``` + +Each record reports elapsed seconds, GC seconds, input-matrix size and Julia +allocated bytes. The allocation number includes RCall result copies but cannot +include allocations inside the embedded R runtime; it is not presented as total +RSS. Any individual operation taking more than five minutes emits a warning. +When a baseline is supplied, regressions over 10% in the pre-existing default +Bayesian replacement or ML dispersion path make the script exit unsuccessfully, +so a CI performance job can use the same gate. Baselines must use the same +runner, R package versions, data sizes and dispersion options to be comparable. diff --git a/bench/advanced_zero_handling/benchmark.jl b/bench/advanced_zero_handling/benchmark.jl new file mode 100644 index 00000000..ebeb206d --- /dev/null +++ b/bench/advanced_zero_handling/benchmark.jl @@ -0,0 +1,110 @@ +#!/usr/bin/env julia +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) + +# Opt-in scaling benchmark for issue #21. No result file is written unless +# --output=PATH is supplied. See README.md in this directory. + +using MetaManifold +using MetaManifold.Differential +using JSON3 + +function arg_value(prefix::AbstractString) + for arg in ARGS + startswith(arg, prefix) && return arg[length(prefix) + 1:end] + end + nothing +end + +"""A deterministic 24-sample table with common and sparse taxa.""" +function fixture(n_taxa::Int; n_samples::Int=24) + n_taxa >= 6 || throw(ArgumentError("the benchmark fixture needs at least 6 taxa")) + counts = zeros(Int, n_samples, n_taxa) + for j in 1:n_taxa, i in 1:n_samples + guaranteed = i == (j % n_samples + 1) || i == ((j + 7) % n_samples + 1) + present = j <= 6 || guaranteed || (i + 3j) % 8 == 0 + present || continue + counts[i, j] = 1 + (17i + 31j) % 50 + j == 1 && i > n_samples ÷ 2 && (counts[i, j] *= 4) + end + counts +end + +function replacement_benchmark(counts, method::String, alpha) + out = replace_zeros(counts; method, alpha) + # Consume the result so a future implementation cannot win by doing no work. + (sum(out.proportions), out.zeros_replaced) +end + +function dispersion_benchmark(counts, samples, groups, dispersion::String) + factors = tss_factors(counts, samples) + fits = fit_nb(counts, groups, log.(factors); levels=("A", "B"), dispersion) + size(fits, 1) +end + +function main() + sizes = parse.(Int, split(something(arg_value("--sizes="), "100,1000,10000"), ',')) + all(>=(6), sizes) || throw(ArgumentError("all --sizes must be at least 6")) + include_dispersion = "--dispersion" in ARGS + output = arg_value("--output=") + baseline_path = arg_value("--baseline=") + results = Dict{String,Any}[] + samples = ["sample $i" for i in 1:24] + groups = vcat(fill("A", 12), fill("B", 12)) + + for n_taxa in sizes + counts = fixture(n_taxa) + scenarios = Any[ + ("replacement/bayesian_multiplicative", () -> replacement_benchmark( + counts, "bayesian_multiplicative", nothing)), + ("replacement/bayesian_alpha_2.5", () -> replacement_benchmark( + counts, "bayesian_multiplicative", 2.5)), + ("replacement/multiplicative_replacement", () -> replacement_benchmark( + counts, "multiplicative_replacement", nothing)), + ] + if include_dispersion + push!(scenarios, ("dispersion/ml", () -> dispersion_benchmark(counts, samples, groups, "ml"))) + push!(scenarios, ("dispersion/glmGamPoi", () -> dispersion_benchmark(counts, samples, groups, "glmGamPoi"))) + end + for (operation, f) in scenarios + n_taxa == first(sizes) && f() # warm up each method before timing it + GC.gc() + timed = @timed f() + timed.time > 300 && @warn "Benchmark operation exceeded five minutes" operation n_taxa seconds=timed.time + push!(results, Dict{String,Any}( + "operation" => operation, + "n_taxa" => n_taxa, + "n_samples" => 24, + "seconds" => timed.time, + # @timed measures Julia allocations, including RCall copies. It + # cannot measure allocations inside the embedded R runtime. + "julia_allocated_bytes" => timed.bytes, + "input_matrix_bytes" => Base.summarysize(counts), + "gc_seconds" => timed.gctime, + )) + println(stderr, "$operation, $n_taxa taxa: $(round(timed.time; digits=3)) s, " * + "$(timed.bytes) Julia bytes allocated") + end + end + + document = Dict("kind" => "advanced_zero_handling_benchmark", "results" => results) + rendered = JSON3.write(document) + isnothing(output) ? println(rendered) : write(output, rendered * "\n") + + if !isnothing(baseline_path) + baseline = JSON3.read(read(baseline_path, String)) + regressions = String[] + for old in baseline.results + old.operation in ("replacement/bayesian_multiplicative", "dispersion/ml") || continue + new = findfirst(r -> r["operation"] == old.operation && r["n_taxa"] == old.n_taxa, results) + isnothing(new) && error("baseline operation $(old.operation) at $(old.n_taxa) taxa was not measured") + ratio = results[new]["seconds"] / old.seconds + ratio > 1.10 && push!(regressions, + "$(old.operation) at $(old.n_taxa) taxa: $(round(100 * ratio; digits=1))% of baseline") + end + isempty(regressions) || error("Existing-method runtime regressed by more than 10%: " * join(regressions, "; ")) + end + nothing +end + +abspath(PROGRAM_FILE) == @__FILE__ && main() diff --git a/config/defaults/pipeline.yml b/config/defaults/pipeline.yml index 44b2c882..a3a5f6ff 100644 --- a/config/defaults/pipeline.yml +++ b/config/defaults/pipeline.yml @@ -369,13 +369,24 @@ analysis: # Fraction of samples, 0 to 1, in which a taxon must have reads to be # tested. Taxa below it are listed as filtered, without a p-value. For # clr_welch it is applied before zero replacement, so on a sparse table it also - # keeps imputed values for rare taxa out of every sample; clr_welch refuses a - # taxon seen in fewer than 2 samples. + # keeps imputed values for rare taxa out of every sample. Bayesian replacement + # refuses a taxon seen in fewer than 2 samples; CZM requires at least one. min_prevalence: 0.0 - # clr_welch only: cmultRepl's frac, between 0 and 1. An imputed proportion above - # its taxon's smallest observed proportion is lowered to this fraction of it. - # A replaced zero is not a measurement. + # clr_welch only: how cmultRepl replaces zeros before the centred log-ratio. + # bayesian_multiplicative uses the geometric Bayesian-multiplicative (GBM) + # posterior replacement. multiplicative_replacement uses count zero + # multiplicative (CZM) replacement. Both preserve observed-part ratios; + # neither turns an imputed zero into a measurement. + replacement_method: bayesian_multiplicative + # clr_welch only: cmultRepl's frac, strictly between 0 and 1. For CZM this + # is the fraction of the 0.5/sample-total threshold; for either method an + # imputed proportion above its taxon's smallest observed proportion is + # lowered to this fraction of it. Values below 0.01 or at least 0.9 warn. replacement_delta: 0.65 + # clr_welch with bayesian_multiplicative only: null uses GBM's estimated + # Dirichlet concentration. A finite value greater than 0 overrides it with + # cmultRepl(method = "user") and the same leave-one-out prior mean. + bayesian_alpha: ~ ## Occupancy probe (MMW-01) # Measures each attested stage's memory: Julia's allocation total, and the diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 4ea20add..9e86753c 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -441,16 +441,25 @@ export interface DifferentialResult { offset?: 'tss' | 'rle' /** nb_glm only: where each taxon's theta comes from. */ dispersion?: 'ml' | 'glmGamPoi' - /** clr_welch only. */ + /** clr_welch only: the reference cmultRepl method and its parameters. */ + replacement_method?: 'bayesian_multiplicative' | 'multiplicative_replacement' replacement_delta?: number + /** Null uses GBM's estimated concentration; positive values override it. */ + bayesian_alpha?: number | null } diagnostics: { n_taxa: number; n_tested: number; n_failed: number; n_boundary: number; n_filtered: number /** clr_welch only. */ zero_replacement?: { - method: string; delta: number; zeros_replaced: number; n_taxa_in_composition: number + method: string + replacement_method: 'bayesian_multiplicative' | 'multiplicative_replacement' + delta: number; alpha: number | null; zeros_replaced: number; n_taxa_in_composition: number n_taxa_unobserved: number; median_imputed_fraction: number; max_imputed_fraction: number warnings: string[] + posterior?: { + family: 'Dirichlet'; prior_mean: string; shape: string; adjustments: string + concentration: { sample: string; value: number }[] + } } } rows: DifferentialRow[] diff --git a/frontend/src/components/DifferentialPanel.tsx b/frontend/src/components/DifferentialPanel.tsx index 825fb812..a29fcd06 100644 --- a/frontend/src/components/DifferentialPanel.tsx +++ b/frontend/src/components/DifferentialPanel.tsx @@ -22,7 +22,7 @@ function csvBlob(result: DifferentialResult): Blob { /** * Differential abundance between exactly two runs or sub-groups: a model per - * taxon at the chosen rank (negative-binomial GLM or CLR linear model, as + * taxon at the chosen rank (negative-binomial GLM or CLR Welch test, as * configured), with BH-adjusted p-values, shown as a volcano plot and a table * that can be downloaded or added to the report. The first selected condition is the reference. */ @@ -136,6 +136,22 @@ export function DifferentialPanel({ study, runs, option, aggregate }: { })()}

+ {result.diagnostics.zero_replacement && result.diagnostics.zero_replacement.zeros_replaced > 0 && (() => { + const z = result.diagnostics.zero_replacement! + return ( +
+ DANGER — zero replacement is biased. Replaced zeros are model-based imputations, + not measurements. Do not try multiple delta or alpha values and report only the most significant result; + the selected values are recorded in this result and its CSV provenance. + {z.posterior && ' Dirichlet posterior parameters are recorded; the replacements are adjusted posterior estimates, not posterior draws.'} + {z.warnings.length > 0 &&
    + {z.warnings.map(w =>
  • {w}
    • )} +
} +
+ ) + })()} +
diff --git a/frontend/src/components/PipelineStages.test.ts b/frontend/src/components/PipelineStages.test.ts index 37fa1f24..dc1b8af5 100644 --- a/frontend/src/components/PipelineStages.test.ts +++ b/frontend/src/components/PipelineStages.test.ts @@ -1,9 +1,19 @@ // SPDX-License-Identifier: AGPL-3.0-only import { expect, test } from 'bun:test' -import { CONFIG_TYPES } from './PipelineStages' +import { CONFIG_DESCRIPTIONS, CONFIG_TYPES } from './PipelineStages' test('dispersion uses the enum editor with both supported models', () => { expect(CONFIG_TYPES['analysis.differential.dispersion']).toEqual({ kind: 'enum', options: ['ml', 'glmGamPoi'], }) }) + +test('advanced zero handling exposes both methods, bounded delta and nullable alpha', () => { + expect(CONFIG_TYPES['analysis.differential.replacement_method']).toEqual({ + kind: 'enum', options: ['bayesian_multiplicative', 'multiplicative_replacement'], + }) + expect(CONFIG_TYPES['analysis.differential.replacement_delta']).toMatchObject({ kind: 'float', step: 0.05 }) + expect(CONFIG_TYPES['analysis.differential.bayesian_alpha']).toMatchObject({ kind: 'float', nullable: true }) + expect(CONFIG_DESCRIPTIONS['analysis.differential.replacement_method']).toContain('every replacement is biased') + expect(CONFIG_DESCRIPTIONS['analysis.differential.bayesian_alpha']).toContain('no posterior draws') +}) diff --git a/frontend/src/components/PipelineStages.tsx b/frontend/src/components/PipelineStages.tsx index f4cc3fbe..ae9849aa 100644 --- a/frontend/src/components/PipelineStages.tsx +++ b/frontend/src/components/PipelineStages.tsx @@ -152,9 +152,11 @@ export const CONFIG_DESCRIPTIONS: Record = { 'analysis.beta.depth': 'Reads per sample for rarefy or srs; 0 uses the smallest non-empty sample.', 'analysis.differential.method': 'Model fitted to each taxon: nb_glm (negative-binomial GLM on read counts; effect is a log2 fold change) or clr_welch (Welch\'s t-test on centred log-ratios over the taxa passing min_prevalence, after zCompositions::cmultRepl zero replacement; effect is a CLR difference, not a fold change).', 'analysis.differential.offset': 'Library-size offset in each negative-binomial model: tss (total reads per sample) or rle (median-of-ratios over taxa present in every sample; refused when there are none). Used by nb_glm only.', - 'analysis.differential.dispersion': 'nb_glm only: ml estimates theta separately with MASS::glm.nb; glmGamPoi uses a cross-taxon dispersion trend in fixed-scale negative-binomial Wald fits.', - 'analysis.differential.min_prevalence': 'Fraction of samples, 0 to 1, in which a taxon must have reads to be tested; taxa below it are listed as filtered.', - 'analysis.differential.replacement_delta': 'clr_welch only: cmultRepl\'s frac, between 0 and 1; an imputed proportion above its taxon\'s smallest observed proportion is lowered to this fraction of it. Replaced zeros are not measurements.', + 'analysis.differential.dispersion': 'nb_glm only: ml estimates theta separately with MASS::glm.nb; glmGamPoi (Ahlmann-Eltze & Huber 2020) uses a cross-taxon dispersion trend in fixed-scale negative-binomial Wald fits. This is not glmGamPoi\'s quasi-likelihood test.', + 'analysis.differential.min_prevalence': 'Fraction of samples, 0 to 1, in which a taxon must have reads to be tested; taxa below it are listed as filtered. Filtering before replacement limits imputed mass on sparse tables.', + 'analysis.differential.replacement_method': 'clr_welch only: bayesian_multiplicative uses cmultRepl GBM posterior estimates (Martín-Fernández et al. 2015); multiplicative_replacement uses CZM (Martín-Fernández et al. 2003). Both preserve observed-part ratios, but every replacement is biased. For sparse data consider NB on raw counts, prevalence filtering, occupancy or zero-inflated models instead.', + 'analysis.differential.replacement_delta': 'clr_welch only: cmultRepl\'s frac, strictly between 0 and 1. CZM starts at delta × 0.5/sample total; either method caps high imputations at delta × the taxon\'s smallest observed proportion. Below 0.01 or at least 0.9 warns. Do not tune delta for significance; every replacement is biased.', + 'analysis.differential.bayesian_alpha': 'Bayesian replacement only: null uses GBM\'s estimated Dirichlet concentration; a positive number uses that concentration with the same leave-one-out prior mean. The posterior parameters are recorded; no posterior draws are generated. Larger alpha gives the prior more weight.', 'analysis.alpha.show_points': 'Show individual samples overlaid on alpha comparison boxplots.', 'analysis.alpha.annotate_significance': 'Annotate alpha comparison panels with Kruskal-Wallis significance.', 'analysis.alpha.pairwise_brackets': 'Run BH-adjusted pairwise Wilcoxon rank-sum tests between every group pair and draw brackets, including n.s. results.', @@ -255,7 +257,9 @@ export const CONFIG_TYPES: Record = { 'analysis.differential.offset': { kind: 'enum', options: ['tss', 'rle'] }, 'analysis.differential.dispersion': { kind: 'enum', options: ['ml', 'glmGamPoi'] }, 'analysis.differential.min_prevalence': { kind: 'float', min: 0, max: 1, step: 0.05 }, - 'analysis.differential.replacement_delta': { kind: 'float', min: 0.01, max: 0.99, step: 0.05 }, + 'analysis.differential.replacement_method': { kind: 'enum', options: ['bayesian_multiplicative', 'multiplicative_replacement'] }, + 'analysis.differential.replacement_delta': { kind: 'float', min: 0, max: 1, step: 0.05 }, + 'analysis.differential.bayesian_alpha': { kind: 'float', nullable: true, min: 0, step: 0.1 }, 'analysis.alpha.show_points': { kind: 'boolean' }, 'analysis.alpha.annotate_significance': { kind: 'boolean' }, 'analysis.alpha.pairwise_brackets': { kind: 'boolean' }, diff --git a/frontend/src/components/differentialTable.test.ts b/frontend/src/components/differentialTable.test.ts index f2143e07..9e75b0ad 100644 --- a/frontend/src/components/differentialTable.test.ts +++ b/frontend/src/components/differentialTable.test.ts @@ -68,9 +68,22 @@ describe('clr_welch results', () => { expect(effectValue(result, result.rows[0])).toBe(2) }) - test('the settings line names the delta and says the estimate is not a fold change', () => { - expect(configLine(clr)).toBe('method clr_welch, replacement_delta 0.65, min_prevalence 0; estimate is a CLR difference, not a fold change') + test('the settings line records replacement provenance and says the estimate is not a fold change', () => { + expect(configLine(clr)).toBe('method clr_welch, replacement_method bayesian_multiplicative, replacement_delta 0.65, bayesian_alpha estimated, min_prevalence 0; estimate is a CLR difference, not a fold change') expect(configLine(result)).toBe('method nb_glm, offset tss, min_prevalence 0') expect(differentialCsv(clr).split('\r\n')[2]).toBe(`# ${configLine(clr)}`) + const custom: DifferentialResult = { + ...clr, + config: { method: 'clr_welch', min_prevalence: 0.1, + replacement_method: 'multiplicative_replacement', replacement_delta: 0.4, bayesian_alpha: null }, + } + expect(configLine(custom)).toContain('replacement_method multiplicative_replacement') + expect(configLine(custom)).toContain('replacement_delta 0.4') + const alpha: DifferentialResult = { + ...clr, + config: { method: 'clr_welch', min_prevalence: 0, + replacement_method: 'bayesian_multiplicative', replacement_delta: 0.65, bayesian_alpha: 2.5 }, + } + expect(configLine(alpha)).toContain('bayesian_alpha 2.5') }) }) diff --git a/frontend/src/components/differentialTable.ts b/frontend/src/components/differentialTable.ts index 90a547a8..90608da2 100644 --- a/frontend/src/components/differentialTable.ts +++ b/frontend/src/components/differentialTable.ts @@ -41,14 +41,18 @@ export function formatStat(value: number | null): string { /** * The settings line of the CSV: the method and the parameters it used. The - * offset belongs to nb_glm and the replacement delta to clr_welch, so each line - * names only its own method's parameters. + * offset belongs to nb_glm; replacement method, delta and alpha belong to + * clr_welch, so each line names only its own method's parameters. */ export function configLine(result: DifferentialResult): string { const c = result.config - return c.method === 'clr_welch' - ? `method clr_welch, replacement_delta ${c.replacement_delta}, min_prevalence ${c.min_prevalence}; estimate is a CLR difference, not a fold change` - : `method nb_glm, offset ${c.offset}, min_prevalence ${c.min_prevalence}` + if (c.method !== 'clr_welch') + return `method nb_glm, offset ${c.offset}, min_prevalence ${c.min_prevalence}` + const replacement = c.replacement_method ?? 'bayesian_multiplicative' + const alpha = c.bayesian_alpha === null || c.bayesian_alpha === undefined + ? 'estimated' : String(c.bayesian_alpha) + return `method clr_welch, replacement_method ${replacement}, replacement_delta ${c.replacement_delta}, ` + + `bayesian_alpha ${alpha}, min_prevalence ${c.min_prevalence}; estimate is a CLR difference, not a fold change` } /** The effect size of one row, from the field the result names in `effect`. */ diff --git a/src/analysis/differential.jl b/src/analysis/differential.jl index 66cbf471..32ee9aef 100644 --- a/src/analysis/differential.jl +++ b/src/analysis/differential.jl @@ -9,8 +9,8 @@ # integer read counts, with a library-size offset. There are no pseudocounts. # - clr_welch: Welch's two-sample t-test per taxon (stats::t.test) on centred # log-ratios, over the taxa that pass `min_prevalence`, after their zeros are -# replaced by zCompositions::cmultRepl (Bayesian-multiplicative, for counts). -# The estimate is a difference in CLR units, not a fold change. +# replaced by zCompositions::cmultRepl (Bayesian GBM/user or multiplicative +# CZM, for counts). The estimate is a difference in CLR units, not a fold change. # # Count matrices here are samples x taxa, as everywhere else in `Analysis`. # Anything that is not a non-negative integer count is refused rather than @@ -22,9 +22,9 @@ using DataFrames, RCall using ..RRuntime: with_r_lock using ..Analysis: _palette_hex, R_WAIT_SECONDS export DifferentialConfig, ScalingRefusal, MASSUnavailable, ZCompositionsUnavailable, - GlmGamPoiUnavailable, OFFSET_METHODS, METHODS, tss_factors, rle_factors, - size_factors, bh_adjust, validate_counts, replace_zeros, clr_transform, - fit_nb, fit_welch, differential_abundance, volcano_chart + GlmGamPoiUnavailable, OFFSET_METHODS, METHODS, DISPERSIONS, REPLACEMENT_METHODS, + tss_factors, rle_factors, size_factors, bh_adjust, validate_counts, replace_zeros, + clr_transform, fit_nb, fit_welch, differential_abundance, volcano_chart # Methods accepted under `analysis.differential.method`. const METHODS = ("nb_glm", "clr_welch") @@ -34,6 +34,11 @@ const METHODS = ("nb_glm", "clr_welch") # across taxa. `"ml"` is the default and keeps #24's results unchanged. const DISPERSIONS = ("ml", "glmGamPoi") +# Zero-replacement methods accepted under +# `analysis.differential.replacement_method`. Both are exact calls to +# zCompositions::cmultRepl; the Bayesian default is the existing GBM path. +const REPLACEMENT_METHODS = ("bayesian_multiplicative", "multiplicative_replacement") + # Offset (size factor) methods accepted under `analysis.differential.offset`. const OFFSET_METHODS = ("tss", "rle") @@ -46,9 +51,21 @@ const THETA_LOWER = 1e-8 # zCompositions::cmultRepl's default `frac`, as in Martín-Fernández et al. (2003). const DEFAULT_DELTA = 0.65 +# Canonicalise and validate a public replacement-method name in one place, so +# direct library calls and YAML configuration have identical semantics. +function _replacement_method(method::AbstractString) + m = lowercase(strip(String(method))) + m in REPLACEMENT_METHODS || throw(ArgumentError( + "analysis.differential.replacement_method must be one of " * + "$(join(REPLACEMENT_METHODS, ", ")), not '$method'")) + m +end + """ DifferentialConfig(offset="tss", min_prevalence=0.0; method="nb_glm", - replacement_delta=0.65, dispersion="ml") + replacement_delta=0.65, dispersion="ml", + replacement_method="bayesian_multiplicative", + bayesian_alpha=nothing) Settings for one differential abundance analysis, read from `analysis.differential` in the pipeline config. `method` is `"nb_glm"` or @@ -59,6 +76,12 @@ log-ratios, which are already free of library size. have at least one read to be tested. `replacement_delta`, in (0, 1), is the `frac` passed to `zCompositions::cmultRepl` by `clr_welch`: an imputed proportion above its taxon's smallest observed proportion is lowered to this fraction of it. +`replacement_method` selects `"bayesian_multiplicative"` (the default GBM +posterior replacement) or `"multiplicative_replacement"` (CZM). For the Bayesian +method, `bayesian_alpha=nothing` uses GBM's estimated concentration; a finite, +strictly positive value instead uses that Dirichlet concentration with +cmultRepl's `method="user"` and the same leave-one-out prior mean. Alpha is +recorded but not used by CZM. `dispersion` names the per-taxon dispersion `nb_glm` uses (`"ml"` or `"glmGamPoi"`): `"ml"` takes the maximum-likelihood theta from `MASS::glm.nb`, `"glmGamPoi"` uses `overdispersion_shrinkage_list.dispersion_trend` from @@ -75,15 +98,19 @@ method and dispersion choices, or non-finite or out-of-range fractions, including settings unused by the selected method. """ struct DifferentialConfig - method :: String - offset :: String - min_prevalence :: Float64 - replacement_delta :: Float64 - dispersion :: String + method :: String + offset :: String + min_prevalence :: Float64 + replacement_delta :: Float64 + dispersion :: String + replacement_method :: String + bayesian_alpha :: Union{Float64,Nothing} function DifferentialConfig(offset::AbstractString="tss", min_prevalence::Real=0.0; method::AbstractString="nb_glm", replacement_delta::Real=DEFAULT_DELTA, - dispersion::AbstractString="ml") + dispersion::AbstractString="ml", + replacement_method::AbstractString="bayesian_multiplicative", + bayesian_alpha::Union{Real,Nothing}=nothing) m = lowercase(strip(String(method))) m in METHODS || throw(ArgumentError( "analysis.differential.method must be one of $(join(METHODS, ", ")), not '$method'")) @@ -102,7 +129,13 @@ struct DifferentialConfig "analysis.differential.min_prevalence must be a fraction in [0, 1], not $min_prevalence")) (isfinite(replacement_delta) && 0 < replacement_delta < 1) || throw(ArgumentError( "analysis.differential.replacement_delta must be in (0, 1), not $replacement_delta")) - new(m, o, Float64(min_prevalence), Float64(replacement_delta), d) + r = _replacement_method(replacement_method) + (isnothing(bayesian_alpha) || + (!(bayesian_alpha isa Bool) && isfinite(bayesian_alpha) && bayesian_alpha > 0)) || + throw(ArgumentError("analysis.differential.bayesian_alpha must be empty or a " * + "finite number greater than 0, not $bayesian_alpha")) + a = isnothing(bayesian_alpha) ? nothing : Float64(bayesian_alpha) + new(m, o, Float64(min_prevalence), Float64(replacement_delta), d, r, a) end end @@ -535,16 +568,43 @@ end # `min_prevalence` before this runs. Errors and warnings are captured, not raised. const _REPLACE_R = raw""" da_warns <- character(0) -da_out <- tryCatch( +da_posterior_t <- NULL +da_posterior_s <- NULL +da_out <- tryCatch({ + da_args <- list(X = da_counts, label = 0, method = da_method, output = "prop", + frac = da_frac, z.warning = 1, z.delete = FALSE, + suppress.print = TRUE) + if (da_method != "CZM") { + # The Dirichlet posterior is parameterised by the observed counts plus + # s*t. GBM estimates s; method="user" keeps the same leave-one-out t but + # takes s from the explicitly configured alpha. Returning t and s lets a + # caller reconstruct that posterior without storing posterior draws. + da_x <- as.data.frame(da_counts) + da_x[da_x == 0] <- NA + da_n <- nrow(da_x) + da_d <- ncol(da_x) + da_prior <- matrix(0, nrow = da_n, ncol = da_d) + for (da_i in seq_len(da_n)) { + da_prior[da_i, ] <- apply(da_x, 2, function(da_col) sum(da_col[-da_i], na.rm = TRUE)) + } + da_posterior_t <- da_prior / rowSums(da_prior) + da_posterior_s <- if (da_method == "user") { + rep(da_alpha, da_n) + } else { + 1 / apply(da_posterior_t, 1, function(da_row) exp(mean(log(da_row)))) + } + if (da_method == "user") { + da_args$t <- da_posterior_t + da_args$s <- da_posterior_s + } + } withCallingHandlers( - as.matrix(zCompositions::cmultRepl(da_counts, label = 0, method = "GBM", output = "prop", - frac = da_frac, z.warning = 1, z.delete = FALSE, - suppress.print = TRUE)), + as.matrix(do.call(zCompositions::cmultRepl, da_args)), warning = function(w) { da_warns <<- c(da_warns, conditionMessage(w)) invokeRestart("muffleWarning") - }), - error = function(e) conditionMessage(e)) + }) +}, error = function(e) conditionMessage(e)) """ """ @@ -571,72 +631,122 @@ function _check_compositions(out::AbstractMatrix{<:Real}, zero_mask::AbstractMat out end -""" - replace_zeros(counts; frac=DEFAULT_DELTA, samples, taxa) -> NamedTuple +# Heuristic numerical cautions, not claims that a particular delta is +# biologically correct. They travel with the result so exported provenance does +# not separate the warning from the parameter that caused it. +function _delta_warnings(frac::Real) + warnings = String[] + frac < 0.01 && push!(warnings, + "replacement_delta < 0.01 can underflow or make imputed proportions numerically indistinguishable from zero") + frac >= 0.9 && push!(warnings, + "replacement_delta >= 0.9 can give replaced zeros substantial mass relative to observed parts") + warnings +end -Replace the zeros of the samples x taxa read-count matrix `counts` with -`zCompositions::cmultRepl(counts, label = 0, method = "GBM", output = "prop", -frac = frac)`, the geometric Bayesian-multiplicative replacement for count -data. Returns `(proportions, zeros_replaced, imputed_mass, warnings)`: -`proportions` has the same shape as `counts`, each row summing to 1 with no -zero left; `imputed_mass` is the share of each sample's total held by replaced -values; `warnings` is whatever R warned. +""" + replace_zeros(counts; method="bayesian_multiplicative", frac=DEFAULT_DELTA, + alpha=nothing, samples, taxa) -> NamedTuple + +Replace zeros in the samples x taxa read-count matrix `counts` through +`zCompositions::cmultRepl`. `method="multiplicative_replacement"` selects its +CZM replacement (`frac * 0.5 / sample_total`, followed by the reference's cap +and multiplicative closure). `method="bayesian_multiplicative"` selects GBM by +default; a finite, strictly positive `alpha` instead selects `method="user"` +with the same leave-one-out Dirichlet prior mean and `s=alpha`. + +Returns `(proportions, zeros_replaced, imputed_mass, warnings, posterior, +method, delta, alpha)`. Every output row is a positive composition summing to +1, and ratios among observed parts are preserved. `warnings` combines R's +warnings with delta's numerical cautions. For Bayesian replacement with at +least one zero, `posterior=(prior_mean, concentration)` parameterises each +sample's pre-adjustment Dirichlet posterior exactly: +`Dirichlet(counts[i, :] + concentration[i] * prior_mean[i, :])`. It is not a +posterior draw, and cmultRepl's subsequent `frac` cap and multiplicative closure +are not part of that Dirichlet distribution. Posterior is `nothing` for CZM or +when there are no zeros to replace. The caller chooses the taxa: everything passed in is replaced and kept, none is -dropped. Throws `ArgumentError` when there are fewer than 2 taxa, when a taxon -has reads in fewer than 2 samples (the GBM prior is then undefined), when a -sample has no reads, or when cmultRepl refuses or returns a value that is not a -positive proportion; the messages name the taxon or sample and point at +dropped. Throws `ArgumentError` for invalid delta/alpha/method values, fewer +than 2 taxa, an empty sample, a never-observed taxon, or a taxon seen in fewer +than 2 samples for Bayesian replacement (whose leave-one-out prior is then +undefined). Errors name the affected taxon or sample and point at `min_prevalence`. Throws `ZCompositionsUnavailable` when zCompositions cannot be loaded and `RBusyError` when the R runtime stays busy. """ function replace_zeros(counts::AbstractMatrix{<:Real}; + method::AbstractString="bayesian_multiplicative", frac::Real=DEFAULT_DELTA, + alpha::Union{Real,Nothing}=nothing, samples::AbstractVector{<:AbstractString}=["sample $i" for i in axes(counts, 1)], taxa::AbstractVector{<:AbstractString}=["taxon $j" for j in axes(counts, 2)]) + m = _replacement_method(method) + (isfinite(frac) && 0 < frac < 1) || throw(ArgumentError( + "replacement delta must be in (0, 1), not $frac")) + (isnothing(alpha) || (!(alpha isa Bool) && isfinite(alpha) && alpha > 0)) || + throw(ArgumentError("Bayesian alpha must be empty or a finite number greater than 0, not $alpha")) + a = isnothing(alpha) ? nothing : Float64(alpha) size(counts) == (length(samples), length(taxa)) || throw(ArgumentError( "the table is $(size(counts)) but there are $(length(samples)) samples and $(length(taxa)) taxa")) size(counts, 2) >= 2 || throw(ArgumentError( "the centred log-ratio needs at least 2 taxa; there are $(size(counts, 2))")) + validate_counts(counts, samples, taxa) n = size(counts, 1) - sparse = [j for j in axes(counts, 2) if count(>(0), view(counts, :, j)) < 2] - isempty(sparse) || throw(ArgumentError( - "$(length(sparse)) taxa have reads in fewer than 2 samples, so cmultRepl's " * - "Bayesian-multiplicative prior cannot be estimated for them (" * - join(("'$(taxa[j])'" for j in first(sparse, 5)), ", ") * - (length(sparse) > 5 ? ", ..." : "") * "); raise analysis.differential.min_prevalence " * - "to at least $(round(2 / n; digits=3)) so that only taxa seen in 2 or more samples are tested")) + zero_mask = counts .== 0 + n_zero = count(zero_mask) + n_zero > 0 && n < 2 && throw(ArgumentError( + "zCompositions::cmultRepl needs at least 2 samples for zero replacement")) + + minimum_present = m == "bayesian_multiplicative" ? 2 : 1 + sparse = [j for j in axes(counts, 2) if count(>(0), view(counts, :, j)) < minimum_present] + if !isempty(sparse) + reason = m == "bayesian_multiplicative" ? + "cmultRepl's Bayesian-multiplicative leave-one-out prior cannot be estimated for them" : + "cmultRepl's CZM method has no observed proportion for them" + throw(ArgumentError( + "$(length(sparse)) taxa have reads in fewer than $minimum_present sample" * + (minimum_present == 1 ? "" : "s") * ", so $reason (" * + join(("'$(taxa[j])'" for j in first(sparse, 5)), ", ") * + (length(sparse) > 5 ? ", ..." : "") * "); raise analysis.differential.min_prevalence " * + "to at least $(round(minimum_present / n; digits=3)) so that only adequately observed taxa are tested")) + end for i in axes(counts, 1) sum(view(counts, i, :)) > 0 || throw(ArgumentError( "sample '$(samples[i])' has no reads in the $(size(counts, 2)) taxa that pass " * "min_prevalence, so it has no composition to transform")) end - zero_mask = counts .== 0 - n_zero = count(zero_mask) - out, warns = with_r_lock(; timeout=R_WAIT_SECONDS[]) do + r_method = m == "multiplicative_replacement" ? "CZM" : isnothing(a) ? "GBM" : "user" + out, warns, posterior = with_r_lock(; timeout=R_WAIT_SECONDS[]) do RCall.rcopy(RCall.reval("requireNamespace('zCompositions', quietly = TRUE)")) || throw(ZCompositionsUnavailable()) # cmultRepl stops when there is no zero to replace; the closure of the - # data is then the whole answer. - n_zero == 0 && return (Float64.(counts) ./ sum(counts; dims=2), String[]) + # data is then the whole answer and no posterior replacement is needed. + n_zero == 0 && return (Float64.(counts) ./ sum(counts; dims=2), String[], nothing) RCall.globalEnv[:da_counts] = Matrix{Float64}(counts) RCall.globalEnv[:da_frac] = Float64(frac) + RCall.globalEnv[:da_method] = r_method + RCall.globalEnv[:da_alpha] = something(a, 0.0) try RCall.reval(_REPLACE_R) res = RCall.rcopy(RCall.reval("da_out")) res isa AbstractString && throw(ArgumentError( "zCompositions::cmultRepl refused the table: $res. Raising " * "analysis.differential.min_prevalence leaves fewer, better-observed taxa")) - (Matrix{Float64}(res), String.(RCall.rcopy(Vector{String}, RCall.reval("da_warns")))) + post = m == "bayesian_multiplicative" ? ( + prior_mean=Matrix{Float64}(RCall.rcopy(RCall.reval("da_posterior_t"))), + concentration=Float64.(RCall.rcopy(RCall.reval("da_posterior_s")))) : nothing + (Matrix{Float64}(res), String.(RCall.rcopy(Vector{String}, RCall.reval("da_warns"))), post) finally - RCall.reval("rm(list = intersect(ls(), c('da_counts', 'da_frac', 'da_out', 'da_warns'))); " * - "invisible(gc())") + RCall.reval("rm(list = intersect(ls(), c('da_counts', 'da_frac', 'da_method', 'da_alpha', " * + "'da_out', 'da_warns', 'da_posterior_t', 'da_posterior_s', 'da_args', 'da_x', " * + "'da_n', 'da_d', 'da_prior', 'da_i', 'da_col', 'da_row'))); invisible(gc())") end end _check_compositions(out, zero_mask, samples) mass = [sum(out[i, j] for j in axes(out, 2) if zero_mask[i, j]; init=0.0) for i in axes(out, 1)] - (proportions=out, zeros_replaced=n_zero, imputed_mass=mass, warnings=unique(warns)) + append!(warns, _delta_warnings(frac)) + (proportions=out, zeros_replaced=n_zero, imputed_mass=mass, warnings=unique(warns), + posterior=posterior, method=m, delta=Float64(frac), alpha=a) end # The per-taxon Welch t-test loop, with the same `da_` naming and statuses as @@ -750,10 +860,11 @@ The result's `config` includes the selected dispersion name. `clr_welch`, in this order: (1) `min_prevalence` chooses the taxa, and taxa with no reads in any sample are always left out; (2) the zeros of those taxa only -are replaced by `zCompositions::cmultRepl` (GBM, `frac = -config.replacement_delta`), so the rare taxa that are not tested contribute no -imputed values; (3) each sample is transformed to centred log-ratios over the -same taxa; (4) each taxon is tested by Welch's t-test. Replacing zeros across +are replaced by `zCompositions::cmultRepl` using `config.replacement_method`, +`frac=config.replacement_delta`, and optional `config.bayesian_alpha`, so rare +taxa that are not tested contribute no imputed values; (3) each sample is +transformed to centred log-ratios over the same taxa; (4) each taxon is tested +by Welch's t-test. Replacing zeros across every observed taxon first would let imputed values for rare taxa take over the samples of a sparse table. The estimate is the difference in mean CLR, contrast minus reference: a difference in centred-log-ratio units, not a fold @@ -845,7 +956,8 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, length(tested) >= 2 || throw(ArgumentError( "the centred log-ratio needs at least 2 taxa with reads at or above " * "analysis.differential.min_prevalence = $(config.min_prevalence); there are $(length(tested))")) - replaced = replace_zeros(counts[:, tested]; frac=config.replacement_delta, + replaced = replace_zeros(counts[:, tested]; method=config.replacement_method, + frac=config.replacement_delta, alpha=config.bayesian_alpha, samples, taxa=taxa[tested]) clr = clr_transform(replaced.proportions) fits = fit_welch(clr, groups; levels) @@ -856,25 +968,42 @@ function differential_abundance(counts::AbstractMatrix{<:Real}, "standard_error" => f.se, "statistic" => f.statistic, "pvalue" => f.pvalue)) end + r_fit = config.replacement_method == "multiplicative_replacement" ? "CZM" : + isnothing(config.bayesian_alpha) ? "GBM" : "user" + alpha_text = r_fit == "user" ? ", alpha = $(config.bayesian_alpha)" : "" method_text = "Welch's two-sample t-test per taxon (stats::t.test, var.equal = FALSE) on " * "centred log-ratios over the taxa passing min_prevalence, after their zeros " * - "were replaced by zCompositions::cmultRepl (GBM, frac = $(config.replacement_delta)); " * + "were replaced by zCompositions::cmultRepl ($r_fit, frac = $(config.replacement_delta)$alpha_text); " * "Benjamini-Hochberg adjustment over the fitted taxa. Estimates are " * "differences in CLR units, not fold changes, and replaced zeros are not measurements" effect = Dict("key" => "estimate", "label" => "CLR difference") size_factor_rows = Dict{String,Any}[] config_echo = Dict{String,Any}("method" => config.method, "min_prevalence" => config.min_prevalence, - "replacement_delta" => config.replacement_delta) + "replacement_method" => config.replacement_method, + "replacement_delta" => config.replacement_delta, + "bayesian_alpha" => config.bayesian_alpha) sorted_mass = sort(replaced.imputed_mass) - diagnostics["zero_replacement"] = Dict{String,Any}( - "method" => "zCompositions::cmultRepl (GBM)", - "delta" => config.replacement_delta, "zeros_replaced" => replaced.zeros_replaced, + zero_diagnostics = Dict{String,Any}( + "method" => "zCompositions::cmultRepl ($r_fit)", + "replacement_method" => config.replacement_method, + "delta" => config.replacement_delta, "alpha" => config.bayesian_alpha, + "zeros_replaced" => replaced.zeros_replaced, "n_taxa_in_composition" => length(tested), "n_taxa_unobserved" => count(==(0), present), "median_imputed_fraction" => _median(sorted_mass), "max_imputed_fraction" => last(sorted_mass), "warnings" => replaced.warnings) + if !isnothing(replaced.posterior) + zero_diagnostics["posterior"] = Dict{String,Any}( + "family" => "Dirichlet", + "prior_mean" => "leave-one-out taxon profile of the other samples", + "shape" => "observed counts + concentration * prior_mean", + "adjustments" => "The frac cap and multiplicative closure are applied after this posterior; no posterior draws are used", + "concentration" => [Dict("sample" => String(samples[i]), "value" => s) + for (i, s) in enumerate(replaced.posterior.concentration)]) + end + diagnostics["zero_replacement"] = zero_diagnostics end family = findall(r -> r["status"] in ("ok", "boundary") && !isnothing(r["pvalue"]), rows) diff --git a/src/core/doi.jl b/src/core/doi.jl index 4db53f69..8ab16e64 100644 --- a/src/core/doi.jl +++ b/src/core/doi.jl @@ -677,5 +677,3 @@ function publish_study(config::DOIConfig, study::AbstractString, run::AbstractSt )) return receipt end - -end diff --git a/src/server/routes/analysis.jl b/src/server/routes/analysis.jl index 2d3af55f..cb865d0d 100644 --- a/src/server/routes/analysis.jl +++ b/src/server/routes/analysis.jl @@ -1196,18 +1196,23 @@ end The study's `analysis.differential` settings, resolved through the config cascade. Missing or `nothing` values use the defaults in `DifferentialConfig`, -including `dispersion="ml"`. Invalid settings propagate the constructor's +including `dispersion="ml"`, GBM replacement and an estimated Bayesian alpha. +Invalid settings propagate the constructor's `ArgumentError`; values that cannot be converted to strings or numbers also raise their conversion errors. """ function _differential_config(study::String) resolved = _resolved_run_config(study, nothing, nothing, nothing) val(k, d) = something(get(get(resolved, k, (; value=d)), :value, d), d) + alpha = val("analysis.differential.bayesian_alpha", nothing) DifferentialConfig(String(val("analysis.differential.offset", "tss")), Float64(val("analysis.differential.min_prevalence", 0.0)); method=String(val("analysis.differential.method", "nb_glm")), replacement_delta=Float64(val("analysis.differential.replacement_delta", 0.65)), - dispersion=String(val("analysis.differential.dispersion", "ml"))) + dispersion=String(val("analysis.differential.dispersion", "ml")), + replacement_method=String(val("analysis.differential.replacement_method", + "bayesian_multiplicative")), + bayesian_alpha=isnothing(alpha) ? nothing : Float64(alpha)) end """ diff --git a/src/server/routes/config.jl b/src/server/routes/config.jl index f4f2ba0b..9cc12dec 100644 --- a/src/server/routes/config.jl +++ b/src/server/routes/config.jl @@ -109,6 +109,7 @@ const _ENUM_VALUES = Dict( "analysis.differential.method" => Differential.METHODS, "analysis.differential.offset" => Differential.OFFSET_METHODS, "analysis.differential.dispersion" => Differential.DISPERSIONS, + "analysis.differential.replacement_method" => Differential.REPLACEMENT_METHODS, "phylogeny.reference.align.strategy" => Validation.PHYLOGENY_ALIGN_STRATEGIES, "phylogeny.placement.align.strategy" => Validation.PHYLOGENY_ALIGN_STRATEGIES, "phylogeny.reference.trim.method" => Validation.PHYLOGENY_TRIM_METHODS, @@ -121,7 +122,8 @@ const _FILE_NAME_PARTS = ("dada2.output.fasta_prefix", "dada2.output.seq_table_p "dada2.output.taxa_prefix", "cutadapt.r1_suffix", "cutadapt.r2_suffix") # Numeric keys whose factory default is a number but which also take ~. -const _NULLABLE_NUMBERS = ("phylogeny.placement.place.heuristic",) +const _NULLABLE_NUMBERS = ("phylogeny.placement.place.heuristic", + "analysis.differential.bayesian_alpha") """ _config_value_error(key, value) -> Union{String,Nothing} @@ -136,6 +138,20 @@ function _config_value_error(key::String, value) return v in lowercase.(Differential.DISPERSIONS) ? nothing : "$key must be one of $(join(Differential.DISPERSIONS, ", "))" end + if key == "analysis.differential.replacement_method" && v isa AbstractString + v = lowercase(strip(v)) + return v in Differential.REPLACEMENT_METHODS ? nothing : + "$key must be one of $(join(Differential.REPLACEMENT_METHODS, ", "))" + end + if key == "analysis.differential.replacement_delta" + return (v isa Real && !(v isa Bool) && isfinite(v) && 0 < v < 1) ? nothing : + "$key must be a finite number in (0, 1)" + end + if key == "analysis.differential.bayesian_alpha" + isnothing(v) && return nothing + return (v isa Real && !(v isa Bool) && isfinite(v) && v > 0) ? nothing : + "$key must be empty or a finite number greater than 0" + end if haskey(_ENUM_VALUES, key) allowed = _ENUM_VALUES[key] v in allowed || return "$key must be one of $(join(filter(!isnothing, collect(allowed)), ", "))" @@ -274,10 +290,12 @@ end function _write_config_updates(path::String, changes::AbstractDict; remove=()) lock(_config_file_lock) do native = Dict{String,Any}(string(k) => _to_plain(v) for (k, v) in changes) - key = "analysis.differential.dispersion" - if haskey(native, key) && native[key] isa AbstractString - i = findfirst(v -> lowercase(v) == lowercase(strip(native[key])), Differential.DISPERSIONS) - isnothing(i) || (native[key] = Differential.DISPERSIONS[i]) + for (key, allowed) in (("analysis.differential.dispersion", Differential.DISPERSIONS), + ("analysis.differential.replacement_method", Differential.REPLACEMENT_METHODS)) + if haskey(native, key) && native[key] isa AbstractString + i = findfirst(v -> lowercase(v) == lowercase(strip(native[key])), allowed) + isnothing(i) || (native[key] = allowed[i]) + end end isempty(native) && isempty(remove) && return nothing cfg = _load_yml(path) diff --git a/test/unit/test_differential.jl b/test/unit/test_differential.jl index dcab0549..c423998e 100644 --- a/test/unit/test_differential.jl +++ b/test/unit/test_differential.jl @@ -171,7 +171,16 @@ end @test_throws ArgumentError DifferentialConfig("tss", NaN) @test DifferentialConfig().method == "nb_glm" @test DifferentialConfig().replacement_delta == 0.65 + @test DifferentialConfig().replacement_method == "bayesian_multiplicative" + @test isnothing(DifferentialConfig().bayesian_alpha) @test DifferentialConfig(; method="clr_welch", replacement_delta=0.3).replacement_delta == 0.3 + @test DifferentialConfig(method="clr_welch", + replacement_method=" MULTIPLICATIVE_REPLACEMENT ").replacement_method == "multiplicative_replacement" + @test DifferentialConfig(method="clr_welch", bayesian_alpha=2.5).bayesian_alpha == 2.5 + @test_throws ArgumentError DifferentialConfig(replacement_method="pseudocount") + for alpha in (0, -1, NaN, Inf, true) + @test_throws ArgumentError DifferentialConfig(method="clr_welch", bayesian_alpha=alpha) + end @test_throws ArgumentError DifferentialConfig(; method="ancombc") # The pre-review name, never released, is not accepted. @test_throws ArgumentError DifferentialConfig(; method="clr_lm") @@ -183,9 +192,16 @@ end @test_throws ArgumentError DifferentialConfig(dispersion="shrinkage") @test_throws ArgumentError DifferentialConfig(method="clr_welch", dispersion="glmGamPoi") @test DifferentialConfig(method="clr_welch").dispersion == "ml" - for d in (0, 1, -0.5, NaN) + for d in (0, 1, -0.5, NaN, Inf) @test_throws ArgumentError DifferentialConfig(; method="clr_welch", replacement_delta=d) end + @test occursin("underflow", only(MetaManifold.Differential._delta_warnings(0.005))) + @test occursin("substantial mass", only(MetaManifold.Differential._delta_warnings(0.9))) + @test isempty(MetaManifold.Differential._delta_warnings(0.65)) + @test_throws ArgumentError replace_zeros([0 1; 2 3]; frac=0) + @test_throws ArgumentError replace_zeros([0 1; 2 3]; frac=1) + @test_throws ArgumentError replace_zeros([0 1; 2 3]; alpha=0) + @test_throws ArgumentError replace_zeros([0 1; 2 3]; method="pseudocount") end @testset "Input validation" begin @@ -309,6 +325,46 @@ end end end + @testset "glmGamPoi dispersion parity on two additional count scales" begin + if !HAVE_GLMGAMPOI + @test_skip false + else + # Together with the unscaled fixture above, these are three + # independent end-to-end comparisons against the R reference. + for counts in (2 .* effect, 3 .* effect) + off = log.(tss_factors(counts, effect_samples)) + rows = fit_nb(counts, effect_groups, off; levels=("A", "B"), dispersion="glmGamPoi") + g = effect_groups + phi = rcopy(R""" + local({ + gp <- glmGamPoi::glm_gp(t($counts), + design = model.matrix(~ factor($g, levels = c("A", "B"))), + size_factors = exp($off), overdispersion_shrinkage = TRUE) + as.numeric(gp$overdispersion_shrinkage_list$dispersion_trend) + }) + """) + for j in axes(counts, 2) + y = counts[:, j] + theta = 1 / phi[j] + direct = rcopy(R""" + local({ + fit <- glm($y ~ factor($g, levels = c("A", "B")) + offset($off), + family = MASS::negative.binomial(theta = $theta), + control = glm.control(maxit = 100)) + unname(summary(fit, dispersion = 1)$coefficients[2, ]) + }) + """) + @test rows.status[j] == "ok" + @test rows.estimate[j] ≈ direct[1] rtol=1e-6 + @test rows.se[j] ≈ direct[2] rtol=1e-6 + @test rows.statistic[j] ≈ direct[3] rtol=1e-6 + @test rows.pvalue[j] ≈ direct[4] rtol=1e-6 + @test rows.theta[j] ≈ theta rtol=1e-6 + end + end + end + end + @testset "glmGamPoi invalid trends are isolated per taxon" begin if !HAVE_GLMGAMPOI @test_skip false @@ -452,6 +508,59 @@ end for dispersion in ("shrinkage", "", nothing, 1, true, ["ml"]) @test !isnothing(SV._config_value_error("analysis.differential.dispersion", dispersion)) end + for method in Differential.REPLACEMENT_METHODS + @test isnothing(SV._config_value_error("analysis.differential.replacement_method", method)) + @test isnothing(SV._config_value_error("analysis.differential.replacement_method", " $(uppercase(method)) ")) + end + @test !isnothing(SV._config_value_error("analysis.differential.replacement_method", "pseudocount")) + for delta in (0.01, 0.65, 0.99) + @test isnothing(SV._config_value_error("analysis.differential.replacement_delta", delta)) + end + for delta in (0, 1, -0.1, Inf, NaN, true, "0.65", nothing) + @test !isnothing(SV._config_value_error("analysis.differential.replacement_delta", delta)) + end + @test isnothing(SV._config_value_error("analysis.differential.bayesian_alpha", nothing)) + @test isnothing(SV._config_value_error("analysis.differential.bayesian_alpha", 0.5)) + for alpha in (0, -1, Inf, NaN, true, "1", []) + @test !isnothing(SV._config_value_error("analysis.differential.bayesian_alpha", alpha)) + end + end + + @testset "Config PATCH validates and records advanced zero handling" begin + SV = MetaManifold.Server + _da_fixture() do + url = "/api/v1/studies/studyD/config" + path = joinpath(SV.ServerState._root[], "data", "studyD", "pipeline.yml") + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", url, + ["Content-Type" => "application/json"], JSON3.write(Dict( + "analysis.differential.replacement_method" => " MULTIPLICATIVE_REPLACEMENT ", + "analysis.differential.replacement_delta" => 0.7, + "analysis.differential.bayesian_alpha" => 2.5)))) + @test response.status == 200 + body = JSON3.read(String(response.body)) + @test body["analysis.differential.replacement_method"].value == "multiplicative_replacement" + saved = YAML.load_file(path)["analysis"]["differential"] + @test saved["replacement_method"] == "multiplicative_replacement" + @test saved["replacement_delta"] == 0.7 && saved["bayesian_alpha"] == 2.5 + + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", url, + ["Content-Type" => "application/json"], JSON3.write(Dict( + "analysis.differential.bayesian_alpha" => nothing)))) + @test response.status == 200 + @test YAML.load_file(path)["analysis"]["differential"]["bayesian_alpha"] === nothing + + before = read(path, String) + for (key, value) in (("analysis.differential.replacement_delta", 0), + ("analysis.differential.replacement_delta", 1), + ("analysis.differential.bayesian_alpha", -1), + ("analysis.differential.replacement_method", "pseudocount")) + response = SV.Oxygen.internalrequest(HTTP.Request("PATCH", url, + ["Content-Type" => "application/json"], JSON3.write(Dict(key => value)))) + @test response.status == 400 + @test JSON3.read(String(response.body)).error == "invalid_config_value" + @test read(path, String) == before + end + end end @testset "Config rejects incompatible effective dispersion pairs" begin @@ -718,35 +827,93 @@ end effect_taxa, ["A", "A", "B"]; reference="A", contrast="B", config=clr_config) end - @testset "Zero replacement matches a direct zCompositions::cmultRepl call" begin + @testset "Zero replacement matches direct zCompositions calls on three datasets" begin if !HAVE_ZCOMP @test_skip false else - x = [0 3 0 12 5; 7 0 1 0 9; 2 2 2 2 2; 0 6 4 30 1; 5 1 0 3 0] - out = replace_zeros(x; frac=0.65) - direct = rcopy(R""" - as.matrix(zCompositions::cmultRepl($(Float64.(x)), label = 0, method = "GBM", - output = "prop", frac = 0.65, z.warning = 1, z.delete = FALSE, - suppress.print = TRUE)) - """) - @test out.proportions ≈ direct rtol=1e-12 - @test out.zeros_replaced == count(==(0), x) - @test all(>(0), out.proportions) - @test vec(sum(out.proportions; dims=2)) ≈ ones(5) - # The ratios among a sample's observed taxa are kept. - @test out.proportions[1, 2] / out.proportions[1, 4] ≈ 3 / 12 + datasets = ( + [0 3 0 12 5; 7 0 1 0 9; 2 2 2 2 2; 0 6 4 30 1; 5 1 0 3 0], + [0 1 0 4; 2 0 3 1; 5 2 0 0; 1 3 2 2; 0 4 1 3], + [0 0 5; 1 0 4; 2 3 0; 0 2 3; 4 1 0; 2 2 2], + ) + scenarios = (("bayesian_multiplicative", nothing), + ("bayesian_multiplicative", 2.5), + ("multiplicative_replacement", nothing)) + for x in datasets, (method, alpha) in scenarios + out = replace_zeros(x; method, frac=0.65, alpha) + totals = vec(sum(Float64.(x); dims=1)) + prior = [(totals[j] - x[i, j]) / (sum(totals) - sum(x[i, :])) + for i in axes(x, 1), j in axes(x, 2)] + concentration = [isnothing(alpha) ? exp(-sum(log.(prior[i, :])) / size(x, 2)) : + Float64(alpha) for i in axes(x, 1)] + direct = if method == "multiplicative_replacement" + rcopy(R""" + as.matrix(zCompositions::cmultRepl($(Float64.(x)), label = 0, method = "CZM", + output = "prop", frac = 0.65, z.warning = 1, z.delete = FALSE, + suppress.print = TRUE)) + """) + elseif isnothing(alpha) + rcopy(R""" + as.matrix(zCompositions::cmultRepl($(Float64.(x)), label = 0, method = "GBM", + output = "prop", frac = 0.65, z.warning = 1, z.delete = FALSE, + suppress.print = TRUE)) + """) + else + rcopy(R""" + as.matrix(zCompositions::cmultRepl($(Float64.(x)), label = 0, method = "user", + output = "prop", frac = 0.65, t = $prior, s = $concentration, + z.warning = 1, z.delete = FALSE, suppress.print = TRUE)) + """) + end + @test out.proportions ≈ direct rtol=1e-12 atol=1e-14 + @test out.zeros_replaced == count(==(0), x) + @test out.method == method && out.delta == 0.65 && out.alpha == alpha + @test all(>(0), out.proportions) + @test vec(sum(out.proportions; dims=2)) ≈ ones(size(x, 1)) + # Every observed pair in every sample keeps its original ratio. + for i in axes(x, 1), j in axes(x, 2), k in axes(x, 2) + if x[i, j] > 0 && x[i, k] > 0 + @test out.proportions[i, j] / out.proportions[i, k] ≈ x[i, j] / x[i, k] rtol=1e-10 + end + end + if method == "multiplicative_replacement" + @test isnothing(out.posterior) + else + @test out.posterior.prior_mean ≈ prior rtol=1e-12 + @test out.posterior.concentration ≈ concentration rtol=1e-12 + # The returned parameters are sufficient to reconstruct a + # zero's Dirichlet marginal, independently of the output. + i, j = 1, findfirst(==(0), x[1, :]) + shape = concentration[i] * prior[i, j] + total_shape = sum(x[i, :]) + concentration[i] + variance = shape * (total_shape - shape) / (total_shape^2 * (total_shape + 1)) + @test 0 < variance < 1 + end + end + # A sample without zeros is its closure; it has no imputed mass. - @test out.proportions[3, :] ≈ fill(0.2, 5) - @test out.imputed_mass[3] == 0.0 - @test out.imputed_mass[1] ≈ out.proportions[1, 1] + out.proportions[1, 3] + first_out = replace_zeros(datasets[1]; frac=0.65) + @test first_out.proportions[3, :] ≈ fill(0.2, 5) + @test first_out.imputed_mass[3] == 0.0 + @test first_out.imputed_mass[1] ≈ first_out.proportions[1, 1] + first_out.proportions[1, 3] # No zero at all: cmultRepl is not called, the closure is returned. - @test replace_zeros([1 3; 2 2]).proportions ≈ [0.25 0.75; 0.5 0.5] + no_zero = replace_zeros([1 3; 2 2]) + @test no_zero.proportions ≈ [0.25 0.75; 0.5 0.5] + @test isnothing(no_zero.posterior) - msg = try replace_zeros([0 3 4; 1 0 5; 0 2 6]; taxa=["lone", "q", "r"]); "" catch e; e.msg end + sparse = [0 3 4; 1 0 5; 0 2 6] + msg = try replace_zeros(sparse; taxa=["lone", "q", "r"]); "" catch e; e.msg end @test occursin("'lone'", msg) && occursin("min_prevalence", msg) + # CZM does not use the leave-one-out prior, so a taxon observed in + # one sample is valid; a taxon never observed is still refused. + @test all(>(0), replace_zeros(sparse; method="multiplicative_replacement").proportions) + @test_throws ArgumentError replace_zeros([0 1; 0 2; 0 3]; method="multiplicative_replacement") msg = try replace_zeros([0 0; 3 4; 1 2]; samples=["empty", "b", "c"]); "" catch e; e.msg end @test occursin("'empty'", msg) @test_throws ArgumentError replace_zeros(reshape([1, 2, 3], 3, 1)) + @test_throws ArgumentError replace_zeros([0 1]) + @test any(occursin("underflow", w) for w in replace_zeros(datasets[1]; frac=0.005).warnings) + @test any(occursin("substantial mass", w) for w in replace_zeros(datasets[1]; frac=0.9).warnings) end end @@ -813,6 +980,44 @@ end end end + @testset "CLR: replacement policy and Bayesian alpha are honoured end to end" begin + if !HAVE_ZCOMP + @test_skip false + else + x = [10 20 2; 12 18 0; 11 19 3; 30 8 0; 28 9 4; 33 7 5] + s = ["s$i" for i in 1:6] + taxa = ["p", "q", "r"] + g = ["A", "A", "A", "B", "B", "B"] + for (method, alpha, r_name) in (("multiplicative_replacement", nothing, "CZM"), + ("bayesian_multiplicative", 2.5, "user")) + config = DifferentialConfig("tss", 0.5; method="clr_welch", + replacement_method=method, bayesian_alpha=alpha) + res = differential_abundance(x, s, taxa, g; reference="A", contrast="B", config) + @test res["config"]["replacement_method"] == method + @test res["config"]["bayesian_alpha"] == alpha + @test occursin("($r_name, frac = 0.65", res["method"]) + zr = res["diagnostics"]["zero_replacement"] + @test zr["method"] == "zCompositions::cmultRepl ($r_name)" + @test zr["alpha"] == alpha + if method == "bayesian_multiplicative" + @test zr["posterior"]["family"] == "Dirichlet" + @test length(zr["posterior"]["concentration"]) == 6 + @test all(row -> row["value"] == 2.5, zr["posterior"]["concentration"]) + else + @test !haskey(zr, "posterior") + end + replaced = replace_zeros(x; method, alpha) + z = clr_transform(replaced.proportions) + a = g .== "A" + byt = Dict(r["taxon"] => r for r in res["rows"]) + for (j, taxon) in enumerate(taxa) + expected = sum(z[.!a, j]) / 3 - sum(z[a, j]) / 3 + @test byt[taxon]["estimate"] ≈ expected rtol=1e-8 + end + end + end + end + @testset "CLR: a sparse table with rare taxa runs once min_prevalence is set" begin if !HAVE_ZCOMP @test_skip false @@ -938,7 +1143,7 @@ end _da_fixture() do root = MetaManifold.Server.ServerState._root[] write(joinpath(root, "data", "studyD", "pipeline.yml"), - "analysis:\n differential:\n method: clr_welch\n replacement_delta: 0.5\n") + "analysis:\n differential:\n method: clr_welch\n replacement_method: multiplicative_replacement\n replacement_delta: 0.5\n bayesian_alpha: ~\n") r = _da_post("studyD", Dict("runs" => [Dict("run" => "runA"), Dict("run" => "runB")], "table" => "merged", "rank" => "Genus")) body = JSON3.read(String(r.body)) @@ -949,6 +1154,9 @@ end @test r.status == 200 @test body.effect.key == "estimate" && body.effect.label == "CLR difference" @test body.config.method == "clr_welch" && body.config.replacement_delta == 0.5 + @test body.config.replacement_method == "multiplicative_replacement" + @test body.config.bayesian_alpha === nothing + @test body.diagnostics.zero_replacement.method == "zCompositions::cmultRepl (CZM)" @test isempty(body.size_factors) @test body.diagnostics.zero_replacement.zeros_replaced == 0 @test body.diagnostics.zero_replacement.median_imputed_fraction == 0 @@ -956,6 +1164,15 @@ end @test byt["Up"].estimate > 0 && byt["Down"].estimate < 0 @test all(x -> x.log2_fold_change === nothing, body.rows) @test occursin("CLR difference", body.figure.layout.xaxis.title) + + write(joinpath(root, "data", "studyD", "pipeline.yml"), + "analysis:\n differential:\n method: clr_welch\n replacement_method: bayesian_multiplicative\n bayesian_alpha: 2.5\n") + response = _da_post("studyD", Dict("runs" => [Dict("run" => "runA"), Dict("run" => "runB")], + "table" => "merged", "rank" => "Genus")) + @test response.status == 200 + custom = JSON3.read(String(response.body)) + @test custom.config.bayesian_alpha == 2.5 + @test occursin("(user, frac = 0.65, alpha = 2.5)", custom.method) end write(joinpath(root, "data", "studyD", "pipeline.yml"), @@ -964,6 +1181,13 @@ end "table" => "merged", "rank" => "Genus")) @test r.status == 400 @test JSON3.read(String(r.body)).error == "invalid_config" + + write(joinpath(root, "data", "studyD", "pipeline.yml"), + "analysis:\n differential:\n method: clr_welch\n bayesian_alpha: 0\n") + r = _da_post("studyD", Dict("runs" => [Dict("run" => "runA"), Dict("run" => "runB")], + "table" => "merged", "rank" => "Genus")) + @test r.status == 400 + @test JSON3.read(String(r.body)).error == "invalid_config" end end From f7ee0df050bc0b48228606fe71eef2766d54a168 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 23 Oct 2026 00:41:46 +0100 Subject: [PATCH 14/35] =?UTF-8?q?feat(evidence):=20Evidence=20Mode=20?= =?UTF-8?q?=E2=80=94=20claim=20editor,=20fibre=20&=20echo,=20residual=20ex?= =?UTF-8?q?plorer=20(refs=20#7)=20(#201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this does Implements the three deliverables named in #7 — the `avec fibre` claim editor, the fibre/echo visualiser and the residual explorer — as a per-run **Evidence Mode** that records *why* a result is claimed to matter rather than adding another way to look at it. **Backend** - `src/core/evidence.jl` — claim schema and validation. A claim is text, a scope (exactly one projection: a rank or a category set, 1–50 named values) and warrants. `supported` requires at least one `measurement`, `citation` or `inference` warrant; a bare `definition` can never support an empirical statement, which is the anti-cherry-pick rule enforced at write time rather than promised in the UI. - `src/server/routes/evidence.jl` — reads `merged/results.duckdb` and the run's `attestation.yml`. Computes a fibre (the preimage of the scope), its echo through a second projection (the other classifier at the same rank, or the parent rank), and Pearson residuals / log2 ratios under one of two declared nulls (`proportional` or a stated `claim_share`). Writes only `evidence/claims.yml`, atomically, under a lock. - `GET/POST/DELETE /api/v1/studies/{study}/runs/{run}/evidence…` plus a `claims.yml` download. **Frontend** — a new `Evidence` tab in `RunView` with the claims editor, the fibre & echo view and the residual explorer. No existing tab, route or component changed except the tab list and the API client. ## Judgement calls worth reviewing - **Echo is reported, never scored.** Agreement yields a verdict (`concordant` / `minor divergence` / `partial agreement` / `discordant`, or `contained` / `inconsistent` / `disjoint` for parent ranks); a failure to compute surfaces as a note, not a silent absence. - **χ² is printed without a p-value.** The test's assumptions (independence, fixed marginals) are not defensible for amplicon counts, so publishing a p-value would be exactly the scientific misuse the issue warns about. - **The null is in the figure subtitle.** A residual plot that does not name its expectation is not interpretable. - **Performance**: every aggregate is one pass with conditional sums; the only row-level output is a member list capped at 50. Nothing is issued per sample. ## Verification - `npx tsc --noEmit -p .` clean; `npx vite build` succeeds; 23/23 frontend unit assertions pass. - All Julia files added here parse (`tree-sitter-julia`); the evidence SQL was executed against a DuckDB 2.0.0 fixture (6 rows / 177 reads; fibre 3 rows / 81 reads; source echo 2 agree + 1 conflict; parent containment inside). - **The Julia test suite has not been run** — there is no Julia in the environment this was written in. `test/unit/test_evidence.jl` and `test/unit/test_evidence_routes.jl` are registered in `runtests.jl` and are the first thing CI should confirm; the hand-derived residual expectations are the likeliest candidates for correction. ## Issue hygiene Refs #7, not closes: the issue is `status:blocked` by the absorption register (`frontend/src/**`). This branch touches that surface deliberately — one tab, one API namespace, one new directory — so the call on whether that is inside the register's exception is the maintainer's, not this PR's. If it is, the implementation is complete enough to close #7; the only open item is a CI run. 🤖 Generated with [Arena.ai](https://arena.ai) Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- README.md | 37 + frontend/src/api/client.ts | 35 + frontend/src/evidence/ClaimEditor.tsx | 462 +++++++++++ frontend/src/evidence/EvidencePanel.tsx | 241 ++++++ frontend/src/evidence/FiberEcho.tsx | 300 +++++++ frontend/src/evidence/ProjectionControls.tsx | 184 +++++ frontend/src/evidence/ResidualExplorer.tsx | 204 +++++ frontend/src/evidence/claims.ts | 251 ++++++ frontend/src/evidence/evidence.test.ts | 308 +++++++ frontend/src/evidence/figures.ts | 140 ++++ frontend/src/evidence/types.ts | 239 ++++++ frontend/src/evidence/ui.tsx | 72 ++ frontend/src/views/RunView.tsx | 7 +- src/MetaManifold.jl | 3 + src/core/evidence.jl | 586 ++++++++++++++ src/server/routes/evidence.jl | 807 +++++++++++++++++++ src/server/server.jl | 5 +- test/runtests.jl | 2 + test/unit/test_evidence.jl | 275 +++++++ test/unit/test_evidence_routes.jl | 311 +++++++ 20 files changed, 4466 insertions(+), 3 deletions(-) create mode 100644 frontend/src/evidence/ClaimEditor.tsx create mode 100644 frontend/src/evidence/EvidencePanel.tsx create mode 100644 frontend/src/evidence/FiberEcho.tsx create mode 100644 frontend/src/evidence/ProjectionControls.tsx create mode 100644 frontend/src/evidence/ResidualExplorer.tsx create mode 100644 frontend/src/evidence/claims.ts create mode 100644 frontend/src/evidence/evidence.test.ts create mode 100644 frontend/src/evidence/figures.ts create mode 100644 frontend/src/evidence/types.ts create mode 100644 frontend/src/evidence/ui.tsx create mode 100644 src/core/evidence.jl create mode 100644 src/server/routes/evidence.jl create mode 100644 test/unit/test_evidence.jl create mode 100644 test/unit/test_evidence_routes.jl diff --git a/README.md b/README.md index ba1132e2..d7abe3e4 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,40 @@ unavailable required R package produces an explicit error. Implementation and reference tests: `src/analysis/differential.jl`, `src/server/routes/analysis.jl`, and `test/unit/test_differential.jl`. +### Evidence Mode + +The **Evidence** tab turns a run's outputs into a citable epistemic record. A +claim is a sentence a reader could disagree with, a **scope** naming the part of +the data it is about (its *fibre*: the rows whose value at a rank, or whose +category in a set, is one of the named labels), and a **warrant** saying why it +follows. Claims are stored in `projects/{study}/[/]/evidence/claims.yml` +and can be downloaded as that exact YAML. + +Three views share one record: + +- **Claims** — the epistemic editor (avec fibre). A claim cannot be saved + without a scope and at least one value, and cannot be marked `supported` + without a measurement, citation or inference warrant: a definition alone + supports nothing. +- **Fibre & echo** — the fibre's share of every sample, its largest members, and + the same fibre seen through a second projection: the other classifier at the + same rank, and the parent rank. Agreement is reported with a verdict; + disagreement is a limit on how strongly the claim may be stated, and an echo + that cannot be computed says so rather than vanishing. +- **Residuals** — where the fibre departs from a null you declare, either + `proportional` (the fibre distributed across samples as the library is) or a + stated `claim_share`, with Pearson residuals, a log2 ratio and a χ² statistic + printed without a p-value. The worst departure is shown first. + +The tab's Provenance card states whether `attestation.yml` is absent, degraded +or internally inconsistent before any claim is read. Longer notes on the +definitions, the schema and the limits are kept in `docs/evidence-mode.md` +(documentation under `docs/` is kept local by policy — see `.gitignore`). + +Implementation: `src/core/evidence.jl` (schema, residuals, verdicts), +`src/server/routes/evidence.jl` (the SQL), `frontend/src/evidence/` +(editor, fibre visualiser, residual explorer) and `test/unit/test_evidence.jl`. + ### Figures and report The Figures tab lays analysis charts out on a page (A4, letter or a journal @@ -766,6 +800,8 @@ projects/{project_name}/{run}/ | |-- taxonomy.tsv # Top-hit taxonomy assignments (ASV or OTU) | `-- logs/ `-- merged/ + |-- evidence/ + | `-- claims.yml # Evidence record: claims, fibres, warrants |-- merged.csv # Merged taxonomy + counts (all taxa) |-- protist_filter.csv # Filtered subset (one per merge_taxa.filters entry) `-- results.duckdb # DuckDB database for API queries @@ -814,6 +850,7 @@ The server exposes a REST API under `/api/v1/`. Key endpoint groups: | Cross-run | `POST /studies/{study}/analysis/{alpha,chart,chart-facet,nmds,permanova,venn,publication-tables}` | Comparison, faceted charts, NMDS, PERMANOVA, taxon overlap and publication tables across runs | | Composition | `POST .../runs/{run}/composition/summary`, `POST .../composition/{source}/query`, `POST .../composition/{source}/distinct/{column}` | Organism-composition summaries and tables for a run | | Composition library | `GET /composition`, `POST/DELETE /composition/{filters,sets}/{name}`, `GET /category-sets`, `POST/DELETE /category-sets/{name}` | Edit the filters and category sets in `config/composition.yml` | +| Evidence | `GET .../runs/{run}/evidence`, `GET .../evidence/claims.yml`, `POST .../evidence/claims`, `DELETE .../evidence/claims/{id}`, `POST .../evidence/{fiber,residuals}` | Claims, fibres, echoes and residuals for a run; the record is exportable as YAML | | Reference trees | `GET/POST /reference-trees`, `GET/PUT/DELETE /reference-trees/{id}`, `GET/PUT .../fasta`, `POST .../run`, `GET .../log/{step}`, `GET .../qc/{align,trim}`, `GET .../alignment/{raw,trimmed}`, `GET .../treefile`, `POST .../trim-preview` | Build and inspect the shared reference trees | | Placements | `GET/POST /studies/{study}/placements`, `GET/PUT/DELETE .../placements/{id}`, `POST .../placements/preview`, `GET/PUT .../fasta/queries`, `POST .../run`, `GET .../log/{step}`, `GET .../qc/{align,trim}`, `GET .../alignment/{raw,trimmed}`, `POST .../trim-preview` | Place a study's sequences on a reference tree | | Trees | `GET/POST /studies/{study}/trees`, `GET/DELETE /studies/{study}/trees/{file}`, `PUT /studies/{study}/trees/{file}/view` | Upload, read and delete Newick and jplace files; save each tree's view state | diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index b3f1c929..6194ebf5 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -24,6 +24,10 @@ import type { } from './types' import type { FigureDoc } from '../figure/types' +import type { + EvidenceClaim, EvidenceContext, FibreRequest, FibreResult, + ResidualRequest, ResidualResult, +} from '../evidence/types' import { saveBlob } from '../utils/download' // Base URL for the backend API. Empty string means same-origin. @@ -397,6 +401,37 @@ export const api = { post(`/api/v1/studies/${study}/placements/${id}/trim-preview`, trim), }, + /** + * Evidence Mode. A run's claims live beside its outputs; the fibre and + * residual queries read the results database without writing to it. + */ + evidence: { + context: (study: string, run: string, group?: string | null) => + get(`/api/v1/studies/${study}/runs/${run}/evidence${gq(group)}`), + saveClaim: (study: string, run: string, claim: EvidenceClaim, group?: string | null) => + post<{ claim: EvidenceClaim; claims: EvidenceClaim[]; updated: string }>( + `/api/v1/studies/${study}/runs/${run}/evidence/claims${gq(group)}`, { claim }), + deleteClaim: (study: string, run: string, id: string, group?: string | null) => + del<{ deleted: string; claims: EvidenceClaim[]; updated: string }>( + `/api/v1/studies/${study}/runs/${run}/evidence/claims/${encodeURIComponent(id)}${gq(group)}`), + fiber: (study: string, run: string, body: FibreRequest, group?: string | null) => + post(`/api/v1/studies/${study}/runs/${run}/evidence/fiber${gq(group)}`, body), + residuals: (study: string, run: string, body: ResidualRequest, group?: string | null) => + post(`/api/v1/studies/${study}/runs/${run}/evidence/residuals${gq(group)}`, body), + /** The record as the YAML that sits on disk, for download. */ + claimsYaml: async (study: string, run: string, group?: string | null): Promise => { + const res = await fetch(apiUrl(`/api/v1/studies/${study}/runs/${run}/evidence/claims.yml${gq(group)}`), + { cache: 'no-store' }) + if (!res.ok) { + const err: ApiError = await res.json().catch(() => ({ + error: 'network_error', message: res.statusText, + })) + throw Object.assign(new Error(err.message), { apiError: err, status: res.status }) + } + return res.blob() + }, + }, + trees: { list: (study: string) => get(`/api/v1/studies/${study}/trees`), get: (study: string, file: string) => diff --git a/frontend/src/evidence/ClaimEditor.tsx b/frontend/src/evidence/ClaimEditor.tsx new file mode 100644 index 00000000..7fc8b1bd --- /dev/null +++ b/frontend/src/evidence/ClaimEditor.tsx @@ -0,0 +1,462 @@ +// © 2026 Joshua Benjamin Jewell. All rights reserved. +// Licensed under the GNU Affero General Public License version 3 (AGPLv3). + +// The epistemic editor: claims, each with the fibre it is about and the warrant +// that is supposed to carry it. +// +// The editor is deliberately stricter than a free-text note. A claim cannot be +// saved without a scope and at least one value, and cannot be marked 'supported' +// without an evidential warrant. Those two rules are what stop the record from +// becoming a place to write down conclusions the data does not carry - the +// cherry-pick risk the issue names. + +import { useEffect, useMemo, useState } from 'react' +import { api } from '../api/client' +import { errorMessage } from '../api/errorMessage' +import { useToast } from '../components/Toast' +import { saveBlob } from '../utils/download' +import { + claimCountByStatus, draftClaim, emptyWarrant, evidentialWarrants, newClaim, + nextClaimId, parseValues, scopeSummary, statusTone, validateClaim, +} from './claims' +import { Badge, Notice, sectionTitleStyle } from './ui' +import { + CLAIM_STATUSES, WARRANT_KINDS, type ClaimScope, type ClaimStatus, type ClaimWarrant, + type EvidenceClaim, type EvidenceContext, type ProjectionTag, type WarrantKind, +} from './types' +import type { AnnotationSource } from '../api/types' + +interface Props { + study: string + run: string + group?: string | null + context: EvidenceContext + /** A scope sent over from the fibre visualiser. */ + draftScope?: ClaimScope | null + onSaved: () => void + onUseInFibre?: (scope: ClaimScope) => void + /** Called once a handed-over scope has become a draft, so it is not reapplied. */ + onDraftConsumed?: () => void +} + +const inputStyle: React.CSSProperties = { + width: '100%', padding: '5px 7px', borderRadius: 4, boxSizing: 'border-box', + border: '1px solid var(--color-border)', background: 'var(--color-bg)', + color: 'var(--color-fg)', fontSize: '.85rem', fontFamily: 'inherit', +} +const labelStyle: React.CSSProperties = { + fontSize: '.76rem', textTransform: 'uppercase', letterSpacing: '.03em', + color: 'var(--color-muted-fg)', display: 'block', marginBottom: 3, +} +const rowStyle: React.CSSProperties = { marginBottom: 12 } + +function ScopeEditor({ value, onChange, context }: { + value: ClaimScope + onChange: (next: ClaimScope) => void + context: EvidenceContext +}) { + const ranks = value.source + ? context.ranks.filter(r => r.source === value.source) + : context.ranks + const rankNames = [...new Set(ranks.flatMap(r => r.names))] + const setNames = context.category_sets.map(s => s.name) + const suggestions = value.tag === 'category' + ? (context.category_sets.find(s => s.name === value.value)?.categories.map(c => c.name) ?? []) + : [] + const [valueInput, setValueInput] = useState('') + + function setTag(tag: ProjectionTag) { + const projection = tag === 'rank' ? (rankNames[0] ?? '') : (setNames[0] ?? '') + onChange({ ...value, tag, value: projection, values: [] }) + } + + function removeValue(v: string) { + onChange({ ...value, values: value.values.filter(x => x !== v) }) + } + + function addValue() { + const next = [...value.values] + for (const part of parseValues(valueInput)) { + if (!next.includes(part)) next.push(part) + } + if (next.length !== value.values.length) onChange({ ...value, values: next }) + setValueInput('') + } + + return ( +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {value.values.map(v => ( + + {v} + + + ))} + {!value.values.length && ( + + none yet — a claim is never about everything + + )} +
+
+ setValueInput(e.target.value)} + onKeyDown={e => { if (e.key === 'Enter') { e.preventDefault(); addValue() } }} /> + + {suggestions.map(s => + +
+
+ ) +} + +export function ClaimEditor({ study, run, group, context, draftScope, onSaved, onUseInFibre, onDraftConsumed }: Props) { + const toast = useToast() + const [selectedId, setSelectedId] = useState(null) + const [draft, setDraft] = useState(null) + // Caveats are edited as text and parsed on save, so a blank line while typing + // is not a validation error. + const [caveatsText, setCaveatsText] = useState('') + const [saving, setSaving] = useState(false) + const [exporting, setExporting] = useState(false) + + const claims = context.claims + const counts = useMemo(() => claimCountByStatus(claims), [claims]) + + const draftKey = draftScope + ? JSON.stringify([draftScope.table, draftScope.tag, draftScope.value, draftScope.source, draftScope.values]) + : null + useEffect(() => { + if (!draftScope) return + setSelectedId(null) + setDraft(newClaim(draftScope)) + setCaveatsText('') + onDraftConsumed?.() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [draftKey]) + + const parsedCaveats = caveatsText.split('\n').map(c => c.trim()).filter(Boolean) + const effective: EvidenceClaim | null = draft + ? { + ...draft, + caveats: parsedCaveats, + id: draft.id || (draft.text.trim() ? nextClaimId(claims, draft.text) : ''), + } + : null + const errors = effective ? validateClaim(effective) : [] + + function select(claim: EvidenceClaim) { + setSelectedId(claim.id) + setDraft(draftClaim(claim)) + setCaveatsText(claim.caveats.join('\n')) + } + + function startNew() { + setSelectedId(null) + setCaveatsText('') + setDraft(newClaim({ + table: context.merged_table ?? context.tables[0]?.id ?? 'merged', + tag: context.ranks[0]?.names.length ? 'rank' : 'category', + value: context.ranks[0]?.names[0] ?? context.category_sets[0]?.name ?? '', + source: context.sources[0] ?? null, + })) + } + + async function save() { + if (!effective || errors.length) return + setSaving(true) + try { + const saved = await api.evidence.saveClaim(study, run, { + ...effective, + text: effective.text.trim(), + caveats: effective.caveats.map(c => c.trim()).filter(Boolean), + }, group) + toast.success(`Claim “${saved.claim.id}” saved`) + setSelectedId(saved.claim.id) + setDraft(draftClaim(saved.claim)) + setCaveatsText(saved.claim.caveats.join('\n')) + onSaved() + } catch (err) { + toast.error(errorMessage(err, 'Could not save the claim')) + } finally { + setSaving(false) + } + } + + async function remove(claim: EvidenceClaim) { + if (!window.confirm(`Delete claim “${claim.id}”? The record on disk is rewritten.`)) return + try { + await api.evidence.deleteClaim(study, run, claim.id, group) + toast.success(`Claim “${claim.id}” deleted`) + if (selectedId === claim.id) { setSelectedId(null); setDraft(null); setCaveatsText('') } + onSaved() + } catch (err) { + toast.error(errorMessage(err, 'Could not delete the claim')) + } + } + + async function exportYaml() { + setExporting(true) + try { + const blob = await api.evidence.claimsYaml(study, run, group) + saveBlob(blob, `${study}-${run}-claims.yml`) + } catch (err) { + toast.error(errorMessage(err, 'Could not download the record')) + } finally { + setExporting(false) + } + } + + function updateWarrant(i: number, patch: Partial) { + if (!draft) return + const warrant = draft.warrant.map((w, j) => (j === i ? { ...w, ...patch } : w)) + setDraft({ ...draft, warrant }) + } + + const supportedWithoutEvidence = draft?.status === 'supported' && + evidentialWarrants(draft).length === 0 + + return ( +
+
+
+
Claims ({claims.length})
+ +
+
+ {counts.supported} supported · {counts.contested} contested · {counts.refuted} refuted ·{' '} + {counts.draft} draft +
+ + {claims.length === 0 && ( +

+ No claims recorded for this run yet. +

+ )} + + {claims.map(claim => ( + + ))} + + +
+ Written to {context.file} +
+
+ + {!draft && ( +
+

+ Select a claim to edit it, or start a new one. A claim records the fibre it is + about, why it follows, and what would make it false. +

+
+ )} + + {draft && effective && ( +
+
+
+ {selectedId ? 'Edit claim' : 'New claim'} +
+ + {draft.status} + +
+ +
+ +