From 9ab17159a14d99d95931f37f057958ed22b139db Mon Sep 17 00:00:00 2001 From: WaterPancake Date: Sun, 11 Oct 2026 14:19:08 -0400 Subject: [PATCH 1/3] Propose unsupervised military acoustic signature side project Add a proposal for a separately scoped branch that tests whether DVIDS military-vehicle audio contains session-invariant structure matching the postulated physical signature hierarchy (tracked/wheeled, turbine/diesel, engine-order structure, model). Labels are used only for evaluation; CAST supplies the parametric description and receives candidate source priors. Proposal only: no collection, fitting or authorization, and the retired main-branch military program and protected-source roles are unchanged. Co-Authored-By: Claude Opus 5.5 --- experiments/military_signature/PROPOSAL.md | 184 +++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 experiments/military_signature/PROPOSAL.md diff --git a/experiments/military_signature/PROPOSAL.md b/experiments/military_signature/PROPOSAL.md new file mode 100644 index 0000000..58935fa --- /dev/null +++ b/experiments/military_signature/PROPOSAL.md @@ -0,0 +1,184 @@ +# Side project: unsupervised military-vehicle acoustic signatures (proposal) + +Branch `military-signature`, proposed 2026-10-11. **Proposal only. Nothing is +collected, fitted or authorized by this file.** The military tracked/wheeled +program is retired from `main` under `AGENTS.md`. This branch is an explicit, +separately scoped side project: it does not resume Milestone 7, the OpenClaw +workflow or the earlier benchmark, and it does not change their historical +results or the roles of protected sources (including the reviewed T-80BVM/T-90M +segments). + +## Question + +Does real-world audio of particular military vehicle models contain a +**session-invariant structure** that matches the postulated physical signature? +Can an unsupervised method recover it without vehicle labels? CAST supplies the +interpretable parametric description, and the learned structure feeds back into +CAST source priors. + +## Why the earlier corpus cannot answer it + +The archived military corpora (`real_development_v4`: 325 windows; `real_eval_v2`: +271) each draw every vehicle model from **exactly one recording session**, for example +AMX 30 from one 2015 demonstration, the MPF testbed from one 2023 tankodrome and the +TR-85M1 from one 2014 DVIDS range video. Model and session are perfectly confounded: +any classifier or cluster trained there can be identifying the session. The +central design requirement of this project is therefore **many independent sessions +per model**. Earlier DVIDS collection also hit inaccessible assets (the Humvee +regating), so access failures must be recorded rather than silently replaced. + +## Postulated signature as a testable hierarchy + +Coarse levels must emerge before fine ones. A method that separates T-72 from T-90 +but not turbine from diesel has not learned a vehicle signature. + +| Level | Postulated cue | Physical basis | +|---|---|---| +| L1 tracked vs wheeled | track-link/sprocket modulation | modulation rate ≈ speed / track pitch | +| L2 powerplant | gas turbine: tonal whine plus broadband; diesel: firing harmonics tied to RPM | turbine vs reciprocating combustion | +| L3 engine-order structure | harmonic weights normalized by firing frequency | four-stroke firing frequency = RPM/60 × cylinders/2 | +| L4 model | combination of L1–L3 plus exhaust/cooling-fan tonal content | engine, transmission and hull | +| L5 variant | small spectral shifts | engine revisions; often below the noise floor | + +Working powerplant table, **from memory; verify against cited sources and record +paper/page before use**: + +| Model | Powerplant | +|---|---| +| M1 family | gas turbine (AGT1500) | +| T-80 / T-80B / T-80BVM | gas turbine (GTD-1000 family) | +| T-80UD | diesel (exception within the T-80 line) | +| T-72 / T-90 | V-12 diesel (V-46/V-84/V-92 family) | +| Leopard 2 | V-12 diesel (MTU MB 873) | +| Leopard 1 | V-10 diesel (MTU MB 838) | + +Unknown values stay `unknown`. + +## Data protocol (DVIDS first) + +**Phase 1 uses DVIDS only.** Choose allied models with many independent sessions there +(for example M1, Leopard 2, Bradley, Stryker, M109, HMMWV/JLTV). Russian-built models +are scarce on DVIDS and would mostly come from other outlets, which turns +outlet into a confound. They enter only in Phase 4, as a cross-outlet transfer test. + +### Segment manifest (one row per manually trimmed segment) + +| Group | Fields | +|---|---| +| Provenance | DVIDS asset ID, URL, title, date, unit, location, outlet, license/usage note, retrieval time, raw SHA256 | +| Grouping | `recording_session` (the shoot; the asset ID unless provenance links several assets), conservative connected group | +| Label | caption-stated model/variant, confidence (`confirmed` / `caption` / `unknown`), number of audible vehicles | +| Condition | motion state (idle / rev / accelerate / constant / pass-by / brake), approximate distance, mic type if inferable | +| Contamination | music, narration, gunfire, aircraft, wind, crowd, AGC pumping | +| Timing | start/end within the raw media, trimming tool version, trimmer ID | + +### Trimming rules + +1. **Hide labels while trimming.** Trim from audio, waveform and spectrogram with the + caption and title hidden. Record rejection reasons, so trimming choices cannot + encode the label. +2. **Cut background segments in every session.** Also trim vehicle-absent segments from + the same video. They estimate that session's channel and noise floor. +3. **Deduplicate before grouping.** DVIDS republishes the same B-roll in many packages. + Run an audio fingerprint/cross-correlation duplicate search across all assets and + merge linked assets into one conservative group, as was done for MELAUDIS. +4. **Never split a session or connected group across roles.** +5. **Prefer two kinds of footage:** videos containing several vehicle types (a + within-session contrast) and the same model across many units, dates and sites. + +### Support target + +About 10 or more independent sessions per model before any model-level claim. Coarse +levels (L1/L2) can be tested earlier with fewer models. + +Audio, video and derived features stay outside the checkout +(`data:military_signature/...`), resolved through `abvid.paths`. + +## Methods (labels used only for evaluation) + +Session/outlet IDs are provenance, not vehicle labels. They may be used to enforce +invariance. + +**Tier 0: baselines and confound measurement (first, cheap).** Use frozen BEATs768, +MFCC26 and per-segment CAST fits, clustered with UMAP + HDBSCAN, and k-means at fixed +k. The first output is a **confound ratio**: how well session and outlet can be +decoded from each representation, versus vehicle model decoded across held-out +sessions. + +**Tier 1: CAST parametric signature.** +- Fit CAST per segment: f0 track, firing-order-normalized harmonic weights, noise + spectrum and envelope. +- Add a modulation block for track-rate structure. +- Estimate each session's channel and noise from its background segments and divide + them out. +- Cluster on the source block only. + +This tier is interpretable and works with small data. + +**Tier 2: nuisance-invariant contrastive learning.** +- **Representation:** a small projection head on frozen BEATs. The data are far too + small to train an encoder. +- **Positive pairs:** two windows of one segment, made to differ only in nuisance: + random EQ, codec, AGC simulation, and backgrounds swapped from *other* sessions' + background banks. +- **CAST resynthesis:** keep a segment's fitted source block and resample its channel + and noise blocks. This produces positives that differ only in recording coloration. +- **Adversary:** a gradient-reversal head against session and outlet ID. + +## Evaluation + +1. **Cross-session retrieval.** For each segment, rank all others *excluding its own + session/group*. Report mAP at L1, L2, L3 and L4 separately. +2. **Confound probes.** Linear session and outlet decodability should fall while + cross-session model decodability holds. A signature claim needs both. +3. **Null.** Permute labels at the session level, not the segment level. +4. **Open set.** Hold out one model entirely and check whether it forms its own region + rather than merging into a neighbour. +5. **Synthetic recovery.** Generate CAST observations with fixed source parameters and + varied nuisance, and require the pipeline to recover the source clusters (mirroring + the CAST-1 fixtures). +6. **CAST question answered either way.** Are CAST source-block parameters more + session-invariant than BEATs embeddings? This tests whether CAST separates source + from recording coloration, even if model recognition fails. +7. **Hierarchy gate.** No L4 claim unless L1 and L2 retrieval are clearly above the + session-permutation null. + +Report group-level uncertainty, every failed or rejected segment, and the distinction +from the roadmap: discriminating particular physical test vehicles is not model/type +recognition across examples and domains. + +## Phases and gates + +| Phase | Work | Gate to continue | +|---|---|---| +| P0 | protocol, manifest schema, trimming tool, fingerprint deduplication; no learning | schema and tool tests pass; human authorizes collection | +| P1 | DVIDS allied pilot: about 6 models × about 8 sessions; Tier 0 only | confound ratio measured; if session dominates and backgrounds cannot remove it, stop and report | +| P2 | Tier 1 CAST signatures with background normalization | L1/L2 retrieval above session-permutation null | +| P3 | Tier 2 contrastive with CAST-resynthesis positives | beats Tier 1 on cross-session retrieval without higher session decodability | +| P4 | Russian-built models from other outlets: cross-outlet transfer test | separately authorized; protected-source roles unchanged | + +Downstream use for CAST: cluster-conditional priors over source parameters become +candidates for the deferred CAST-6 learned estimator, only after P2/P3 evidence. + +## Repository placement on this branch + +- Reusable code in `src/abvid/signature/`: manifest validation, fingerprint + deduplication, background-normalized CAST features, retrieval metrics and confound + probes. +- Thin protocol/configuration layer in `experiments/military_signature/`. +- Tests use synthetic fixtures only. Adding tests changes the harness canary count; + the baseline refresh is human-only and must be called out in review. +- Do not merge into `main` without a decision on how this branch relates to the + `AGENTS.md` scope statement. + +## Known risks + +- **Recording artifacts:** camera AGC and codec damage to envelope and level cues; + music beds and narration; gunfire and aircraft masking. +- **Labels:** caption misidentification of variants. +- **Coverage and confounds:** scarcity of T-72/T-80/T-90 footage on DVIDS and outlet + confounding. +- **Licensing:** usage terms differ by asset. Many DVIDS assets are US government + works, but not all; record terms per asset. +- **Expected outcome:** coarse levels (L1/L2) are plausible. Reliable cross-session + separation at L4 is the open question this project tests, not an expected result. From 1937c8992da93f33dba7f4a228e4c2812d0535c5 Mon Sep 17 00:00:00 2001 From: WaterPancake Date: Sun, 11 Oct 2026 14:28:53 -0400 Subject: [PATCH 2/3] Add military signature side-project plan outline Working checklist outline for the unsupervised military-vehicle acoustic signature project: scope decision, signature hierarchy, DVIDS trimming protocol, Tier 0-2 methods, evaluation, phased gates and CAST feedback. Plan only; nothing collected, fitted or authorized. Co-Authored-By: Claude Opus 5.5 --- experiments/military_signature/PLAN.md | 136 +++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 experiments/military_signature/PLAN.md diff --git a/experiments/military_signature/PLAN.md b/experiments/military_signature/PLAN.md new file mode 100644 index 0000000..f42e421 --- /dev/null +++ b/experiments/military_signature/PLAN.md @@ -0,0 +1,136 @@ +# Military acoustic signature: plan + +Branch `military-signature`, 2026-10-11. This is a working outline of an +unsupervised side project that learns the postulated acoustic signature of military +vehicle models (T-72, T-80, T-90, M1, Leopard and others) and feeds it back into CAST. +The full design and its rationale are in [PROPOSAL.md](PROPOSAL.md). + +**Status: plan only.** Nothing is collected, fitted or authorized. + +## 0. Scope + +- [ ] Decide how this branch relates to `AGENTS.md` on `main`, which retires the military + program and forbids resuming it implicitly. This is a separately authorized side + project, not Milestone 7, OpenClaw or the earlier benchmark. +- [ ] Earlier protected sources, including the reviewed T-80BVM/T-90M segments, keep + their current roles. +- [ ] Audio, video and features stay outside the checkout + (`data:military_signature/...`). +- [ ] Record licensing per asset. DVIDS terms vary, and non-DVIDS outlets have their own. + +**Core problem.** In the archived military corpora, every model comes from one +recording session, so model and session are perfectly confounded. The project needs +about 10 or more independent sessions per model before any model-level claim. + +## 1. Postulated signature hierarchy + +Coarse levels must appear before fine ones: + +| Level | Cue | +|---|---| +| L1 tracked vs wheeled | track-link/sprocket modulation proportional to speed | +| L2 powerplant | gas turbine (tonal whine plus broadband) vs diesel (RPM-locked firing harmonics) | +| L3 engine-order structure | harmonic weights normalized by firing frequency | +| L4 model | combination of L1–L3 plus exhaust and fan tones | +| L5 variant | small shifts; often below the noise floor | + +- [ ] Verify the powerplant table in PROPOSAL.md against cited sources (paper, page). + It is currently from memory. Example: M1 and most T-80 variants are turbines; + T-72, T-90 and Leopard 2 are diesels; T-80UD is a diesel exception. +- **Sanity gate:** separating T-72 from T-90 without separating turbine from diesel + does not count as learning a signature. + +## 2. Data and trimming (DVIDS first) + +- [ ] **Segment manifest.** Fields: + - provenance: asset ID, URL, date, unit, location, outlet, license, raw SHA256; + - `recording_session`; + - caption model/variant with confidence; + - number of audible vehicles; + - motion state; + - distance and mic type; + - contaminants: music, narration, gunfire, aircraft, wind, AGC pumping. +- [ ] **Trimming rules:** + - Trim with the caption hidden, from audio and spectrogram only, and log rejection + reasons. + - Cut **background-only segments** from every session to measure its channel and + noise floor. + - Run audio fingerprint deduplication across assets, because DVIDS reposts the same + footage in many packages. Merge linked assets into one conservative group. + - Never split a session or group across roles. + - Prefer multi-vehicle videos and the same model across many units, dates and sites. +- [ ] **Phase 1 uses DVIDS allied models only:** for example M1, Leopard 2, Bradley, + Stryker, M109, HMMWV/JLTV. Russian-built models come later as a cross-outlet + test, to avoid an outlet = model confound. + +## 3. Methods (vehicle labels used only for evaluation) + +- [ ] **Tier 0, baselines:** frozen BEATs768, MFCC26 and per-segment CAST fits, + clustered with UMAP + HDBSCAN. First output: the **confound ratio**, meaning + how decodable session and outlet are versus vehicle model across sessions. +- [ ] **Tier 1, CAST parametric signature:** + - Fit per segment: f0 track, order-normalized harmonics, noise spectrum, envelope, + and a modulation block. + - Remove each session's channel and noise using its background segments. + - Cluster on the source block only. +- [ ] **Tier 2, nuisance-invariant contrastive learning:** + - A projection head on frozen BEATs. + - Positive pairs differ only in nuisance: EQ, codec, AGC, and backgrounds swapped + in from other sessions. + - CAST resynthesis keeps the source block and resamples the channel and noise. + - A gradient-reversal adversary works against session and outlet ID. + +## 4. Evaluation + +- [ ] **Cross-session retrieval:** exclude the query's own session/group; report mAP + at L1–L4. +- [ ] **Confound probes:** session and outlet decodability should fall while + cross-session model decodability holds. +- [ ] **Null:** session-level label permutation. +- [ ] **Open set:** hold one model out entirely. +- [ ] **Synthetic recovery:** generate CAST observations with fixed source parameters + and varied nuisance; the pipeline must recover the source clusters. +- [ ] **CAST question, answered either way:** are CAST source parameters more + session-invariant than BEATs embeddings? +- [ ] **Hierarchy gate:** no L4 claim unless L1/L2 beat the session-permutation null. +- [ ] Report group-level uncertainty and every rejected segment. Discriminating + particular physical vehicles is not model recognition. + +## 5. Phases and gates + +| Phase | Work | Continue only if | +|---|---|---| +| P0 | protocol, manifest schema, trimming tool, fingerprint dedup; no learning | tool/schema tests pass; human authorizes collection | +| P1 | DVIDS allied pilot, about 6 models × 8 sessions; Tier 0 | confound ratio measured; **stop** if session dominates and background normalization cannot remove it | +| P2 | Tier 1 CAST signatures | L1/L2 retrieval above session-permutation null | +| P3 | Tier 2 contrastive with CAST-resynthesis positives | beats Tier 1 cross-session without higher session decodability | +| P4 | Russian-built models from other outlets; cross-outlet test | separately authorized; protected-source roles unchanged | + +## 6. Feeding back into CAST + +- Report how session-invariant the source blocks are, whatever the recognition result. +- If P2/P3 succeed: cluster-conditional priors over CAST source parameters become + candidates for the deferred CAST-6 learned estimator. + +## 7. Repository placement + +- Reusable code: `src/abvid/signature/` (manifest validation, dedup, background- + normalized CAST features, retrieval metrics, confound probes). +- Protocol/config layer: `experiments/military_signature/`. +- Tests use synthetic fixtures only. New tests change the harness canary count, and + the baseline refresh is human-only. + +## 8. Risks and expectations + +- **Recording artifacts:** camera AGC and codec damage; music beds and narration; + gunfire and aircraft masking. +- **Labels:** captions misidentify variants. +- **Coverage:** T-72/T-80/T-90 footage is scarce on DVIDS and confounded by outlet. +- **Expected outcome:** L1/L2 are plausible; reliable cross-session L4 separation is + the open question being tested. + +## Immediate next step + +P0: draft the frozen trimming rule and manifest schema, then a small segment manifest +for a few allied models with many DVIDS sessions. Run Tier 0 alone to measure how +strong the session confound is before building anything larger. From 988ff1cbe0271ddda842acafcdcc09b76b2248ea Mon Sep 17 00:00:00 2001 From: WaterPancake Date: Sun, 18 Oct 2026 12:32:37 -0400 Subject: [PATCH 3/3] Add military signature P0 freeze and P1 admission tooling Signature package, P0/P1 protocols, admission evidence, reports and tests. Side project only; not for main. --- .../ANNOTATION_ROUNDTRIP.md | 39 ++ .../military_signature/CURRENT_STATUS.md | 103 ++++ experiments/military_signature/P0_STATUS.md | 41 ++ .../military_signature/P1_ADMISSION_DRAFT.md | 88 +++ .../military_signature/P1_AUTHORIZATION.md | 27 + .../PILOT_METADATA_FEASIBILITY.md | 54 ++ experiments/military_signature/PLAN.md | 6 +- experiments/military_signature/PROPOSAL.md | 4 + .../military_signature/PROTOCOL_DRAFT.md | 281 +++++++++ .../candidate_metadata.json | 336 ++++++++++ .../candidate_metadata_supplement.json | 267 ++++++++ .../military_signature/manifest_schema.json | 421 +++++++++++++ experiments/military_signature/p0_freeze.json | 70 +++ .../military_signature/p0_protocol.json | 80 +++ .../military_signature/p1_admission_5gb.json | 58 ++ .../p1_discovery_seeds.json | 108 ++++ .../p1_extra_discovery.json | 77 +++ .../military_signature/source_evidence.json | 252 ++++++++ .../source_evidence_supplement.json | 259 ++++++++ .../military_signature/verify_freeze.py | 7 + reports/military_signature_P0.md | 166 +++++ reports/military_signature_P0_followup.md | 64 ++ reports/military_signature_P1_admission.md | 240 ++++++++ src/abvid/signature/__init__.py | 8 + src/abvid/signature/__main__.py | 82 +++ src/abvid/signature/admission.py | 576 ++++++++++++++++++ src/abvid/signature/admission_annotations.py | 269 ++++++++ src/abvid/signature/admission_reuse.py | 110 ++++ .../signature/admission_reuse_correlation.py | 43 ++ src/abvid/signature/admission_reuse_votes.py | 56 ++ src/abvid/signature/admission_review.py | 273 +++++++++ src/abvid/signature/audio.py | 30 + src/abvid/signature/config.py | 99 +++ src/abvid/signature/dedup.py | 129 ++++ src/abvid/signature/fixtures.py | 106 ++++ src/abvid/signature/freeze.py | 22 + src/abvid/signature/manifest.py | 261 ++++++++ src/abvid/signature/review_ui.py | 9 + src/abvid/signature/trim.py | 226 +++++++ src/abvid/signature/validation.py | 143 +++++ tests/test_signature_admission.py | 227 +++++++ tests/test_signature_p0.py | 299 +++++++++ 42 files changed, 6015 insertions(+), 1 deletion(-) create mode 100644 experiments/military_signature/ANNOTATION_ROUNDTRIP.md create mode 100644 experiments/military_signature/CURRENT_STATUS.md create mode 100644 experiments/military_signature/P0_STATUS.md create mode 100644 experiments/military_signature/P1_ADMISSION_DRAFT.md create mode 100644 experiments/military_signature/P1_AUTHORIZATION.md create mode 100644 experiments/military_signature/PILOT_METADATA_FEASIBILITY.md create mode 100644 experiments/military_signature/PROTOCOL_DRAFT.md create mode 100644 experiments/military_signature/candidate_metadata.json create mode 100644 experiments/military_signature/candidate_metadata_supplement.json create mode 100644 experiments/military_signature/manifest_schema.json create mode 100644 experiments/military_signature/p0_freeze.json create mode 100644 experiments/military_signature/p0_protocol.json create mode 100644 experiments/military_signature/p1_admission_5gb.json create mode 100644 experiments/military_signature/p1_discovery_seeds.json create mode 100644 experiments/military_signature/p1_extra_discovery.json create mode 100644 experiments/military_signature/source_evidence.json create mode 100644 experiments/military_signature/source_evidence_supplement.json create mode 100644 experiments/military_signature/verify_freeze.py create mode 100644 reports/military_signature_P0.md create mode 100644 reports/military_signature_P0_followup.md create mode 100644 reports/military_signature_P1_admission.md create mode 100644 src/abvid/signature/__init__.py create mode 100644 src/abvid/signature/__main__.py create mode 100644 src/abvid/signature/admission.py create mode 100644 src/abvid/signature/admission_annotations.py create mode 100644 src/abvid/signature/admission_reuse.py create mode 100644 src/abvid/signature/admission_reuse_correlation.py create mode 100644 src/abvid/signature/admission_reuse_votes.py create mode 100644 src/abvid/signature/admission_review.py create mode 100644 src/abvid/signature/audio.py create mode 100644 src/abvid/signature/config.py create mode 100644 src/abvid/signature/dedup.py create mode 100644 src/abvid/signature/fixtures.py create mode 100644 src/abvid/signature/freeze.py create mode 100644 src/abvid/signature/manifest.py create mode 100644 src/abvid/signature/review_ui.py create mode 100644 src/abvid/signature/trim.py create mode 100644 src/abvid/signature/validation.py create mode 100644 tests/test_signature_admission.py create mode 100644 tests/test_signature_p0.py diff --git a/experiments/military_signature/ANNOTATION_ROUNDTRIP.md b/experiments/military_signature/ANNOTATION_ROUNDTRIP.md new file mode 100644 index 0000000..ab6b2d3 --- /dev/null +++ b/experiments/military_signature/ANNOTATION_ROUNDTRIP.md @@ -0,0 +1,39 @@ +# P0 annotation import and export verification + +The supplied `annotation.json` passed the frozen P0 import/export contract on +2026-10-11. All six artificial recordings have an interval or logged rejection. +The seven intervals comprise one foreground, one background and five rejections. + +The input matches `/Users/magi/Downloads/annotations.json` semantically and binds +to the final synthetic review batch. Its original bytes are retained outside the +checkout; the supplied file is unchanged. Batch/config/tool-version bindings, +private mapping commitments, review-audio hashes, connected groups and roles all +passed validation. + +Two mono clips from `fixture-b` were exported at the native 16,000 Hz rate: + +| Kind | Interval | Native frames | Samples | +|---|---|---|---| +| Foreground | 2–6 seconds | 32000–96000, end exclusive | 64000 | +| Background | 0–2 seconds | 0–32000, end exclusive | 32000 | + +Each exported waveform matches its raw fixture slice sample for sample. All five +rejection records remain in the manifest and export record. The original 55-file +P0 freeze still verifies; its historical report and protocol were not rewritten. + +The detailed evidence, copied input, manifest, reuse audit and native clips are at +`artifact:military_signature/p0/browser-roundtrip-20261011T232425Z/`. The main +record is `verification.json`; clip records are in `native_clips/export.json`. + +The user subsequently confirmed the Chrome run: “Yes, playback and display +worked,” in response to a question covering audio playback, waveform, +spectrogram and interval-control display. This completes the requested human +browser check. The exact question, answer and input hash are retained in +`browser_confirmation.json` beside `verification.json`. + +The annotation artifact verifies the synthetic import/export workflow; the +separate user confirmation supplies run-level playback/display evidence. +Browser version, observation time and per-recording playback traces remain +unknown. Vehicle labels remain unknown; these artificial signals provide no +model-recognition evidence. Real-media processing, fitting and P1 still require +separate authorization under [PROTOCOL_DRAFT.md](PROTOCOL_DRAFT.md). diff --git a/experiments/military_signature/CURRENT_STATUS.md b/experiments/military_signature/CURRENT_STATUS.md new file mode 100644 index 0000000..1fc3553 --- /dev/null +++ b/experiments/military_signature/CURRENT_STATUS.md @@ -0,0 +1,103 @@ +# Military acoustic signature current status + +Updated 2026-10-12 local date. This additive status supersedes the browser-handoff +and powerplant-evidence gaps in the dated [P0_STATUS.md](P0_STATUS.md) where the +new evidence below applies. Frozen v1 artifacts remain unchanged. + +All named P0 preparation deliverables have evidence. The user subsequently +authorized P1 collection and preparation with a 5 GB total storage ceiling and +20-minute per-asset limit. Scientific feasibility and model recognition remain +untested; fitting and representation extraction are outside the current scope. + +## P1 collection and admission + +The [current authorization](P1_AUTHORIZATION.md), +[collection contract](p1_admission_5gb.json) and +[P1 report](../../reports/military_signature_P1_admission.md) supersede the earlier +metadata-only handoff for collection. At the current verified snapshot, 54 +recordings total 8,735.906 seconds and occupy 4,879,449,428 bytes across the new +data and artifact roots. The longest recording is 505.706 seconds. The 64-asset +collection target was not reached under the storage/reserve limit. + +Seven caption-hidden packets are available at +[the local review index](http://127.0.0.1:8766/). Real-media human review is pending; +the earlier user confirmation concerns only the P0 synthetic packet. No real +foreground segment has been admitted, no independent recording group has been +verified and all experimental roles remain unassigned. Caption asset counts and +the 38 named originators are not independent-session counts. + +The full 54-recording waveform reuse audit completed at 2026-10-12 07:10:02 UTC. +Its authoritative output is +`artifact:military_signature/p1_admission/20261011_5gb/reuse_audit.json`. +It accounts for 53 fingerprinted assets, 4,175 indexed windows and one entirely +zero stream with unresolved fingerprint status. One four-second suggestion links +Bradley assets 897659 and 897660 with correlation 0.988650272254228, agreeing with +their metadata dependence lead. The frozen original library reproduced that exact +score. Neither this link nor absent waveform matches establishes independence. +Historical/protected audio remains excluded. + +The earlier `provisional_admission_groups.json` snapshot remains preserved. +`admission_groups.json` incorporates the completed waveform audit, metadata links +and zero-stream flag. It binds the 54 unchanged collection records and joins their +unknown sessions in one quarantine component under the frozen protocol. It asserts +no known common shoot or vehicle and is not an experimental admission freeze. +Closing checks verified all source/native hash pairs, all seven review-packet +commitments and 11 protected repository witnesses. Verified independent groups +and admitted foreground segments remain zero. + +Native-channel numerical integrity was subsequently checked for all 54 streams. +One decodes entirely to zero in every channel; the other 53 contain nonzero +samples. No entire mono-display cancellation of nonzero native audio was found. +The external `native_signal_integrity.json` retains per-asset hashes and counts. +These observations do not establish vehicle audibility or replace human review. + +The latest focused verification passed 50 checks. The CAST test-count canary +remains red at 720 collected versus 467 expected; baseline-file and frozen-file +checks passed. Human-only baseline refresh and line-by-line review of the new +test file remain required before merge. The original 55-file P0 freeze is intact. + +## P0 preparation evidence + +| Requested deliverable | Evidence | Status and limit | +|---|---|---| +| Frozen draft protocol; phase gates; eight-versus-ten distinction | [PROTOCOL_DRAFT.md](PROTOCOL_DRAFT.md), [canonical JSON](p0_protocol.json), [freeze ledger](p0_freeze.json) | Verified preparation. Eight groups/model is development feasibility; ten is a proposal minimum, not a power calculation. | +| Manifest schema and recording/reuse/known-individual grouping | [manifest_schema.json](manifest_schema.json), [implementation](../../src/abvid/signature/manifest.py) | Verified with artificial cases. Public asset IDs supply no verified session count. | +| Offline trimming and deduplication tooling | [CLI](../../src/abvid/signature/__main__.py), [round trip](ANNOTATION_ROUNDTRIP.md) | Synthetic import/export passed; user confirmed Chrome playback and display. Real-media behavior remains untested. | +| Synthetic validation evidence | `artifact:military_signature/p0/validation-20261011-final/validation.json`; round-trip `verification.json` | Native sample equality, rejected intervals and provenance preserved; 37 focused regression checks passed at the recorded run. | +| Literature, feasibility, confounds and negative findings | [frozen report](../../reports/military_signature_P0.md), [follow-up](../../reports/military_signature_P0_followup.md), [source supplement](source_evidence_supplement.json) | PDF-authoritative selected claims; unsupported variant/engine labels remain unknown. | +| Public metadata inspection | [candidate_metadata.json](candidate_metadata.json) | Seven leads spanning four model captions; zero verified independent recording groups. Rights and audio suitability are unresolved. | +| Harness integrity and handoff | Original report and `artifact:military_signature/p0/verification-20261011.json` | Recorded count canary failed: 707 versus 467; baseline-file and frozen-file checks passed. Human baseline refresh and line-by-line test review remain required before merge. | + +The user confirmation receipt is +`artifact:military_signature/p0/browser-roundtrip-20261011T232425Z/browser_confirmation.json`. +It preserves the exact question and answer linked to the supplied annotation +hash. The original annotation is unchanged. The follow-up literature contains +six additional PDFs and eight visually checked pages; it records historical or +variant-specific Bradley, M109A6, Stryker and M1165A1 evidence without assigning +engines to recordings. + +The supplements are outside the original 55-file freeze. Their separate hashes +and external-evidence checks are recorded in +`artifact:military_signature/p0/followup-verification-20261011.json`. +The original freeze can still be checked with: + +```sh +PYTHONPATH=src .venv/bin/python experiments/military_signature/verify_freeze.py +``` + +## Next input needed for scientific execution + +Current collection and preparation are authorized. The next input is real-media +blind interval review and supported shoot/physical-vehicle provenance. Reuse and +provenance admission must be frozen before allocating experimental roles. + +Any subsequent fitting/evaluation decision must separately specify the admitted +population, identity granularity, exact checkpoint and extraction configuration, +fit/compute budget and evaluation roles. Representation extraction, model or CAST +fitting, protected-source evaluation and media publication remain outside the +current scope. Confirmatory model-wide claims additionally need new unexposed +groups, physical-vehicle support and an allocation/power plan. The eight-group +pilot and ten-group model-claim minimum remain distinct and unachieved. +The retired benchmark is not resumed by this work. + +No commit or merge was performed. The human-only harness baseline was not changed. diff --git a/experiments/military_signature/P0_STATUS.md b/experiments/military_signature/P0_STATUS.md new file mode 100644 index 0000000..87fa62b --- /dev/null +++ b/experiments/military_signature/P0_STATUS.md @@ -0,0 +1,41 @@ +# Military acoustic signature P0 status + +P0 is separately authorized for literature, public asset metadata, protocol design, +and implementation using synthetic fixtures. Real media collection or processing, +model fitting, protected-source evaluation, and P1 execution remain unauthorized. +This side project does not resume the retired military benchmark. + +## Deliverables + +- [x] Frozen draft [protocol](PROTOCOL_DRAFT.md) and explicit prospective phase gates +- [x] [Manifest schema](manifest_schema.json) with conservative session/reuse/known-individual grouping +- [x] Offline caption-hidden trimming and annotation tooling +- [x] Fingerprint and correlation reuse audit +- [x] Synthetic validation and CLI verification: 37 focused tests passed +- [x] PDF-authoritative [source ledger](source_evidence.json), [metadata leads](candidate_metadata.json), and [research report](../../reports/military_signature_P0.md) +- [x] Integrity and human handoff record; existing canary failure preserved + +These checkboxes indicate preparation deliverables, not real-media admission or +scientific validation. Native browser playback/layout still requires human review: +Firefox computer-use approval was rejected; DOM event checks used mocked media APIs. +The current CAST canary collects 707 against 467 and fails only the count check; +baseline-file/frozen-file checks pass. The baseline is untouched and human-only. + +Final artificial evidence: +`artifact:military_signature/p0/validation-20261011-final/validation.json`. +Freeze check: `PYTHONPATH=src .venv/bin/python experiments/military_signature/verify_freeze.py`. +P1 requires separate authorization, per-asset rights and verified recording groups. + +## Initial evidence + +Before P0 implementation, the CAST harness canary collected 661 tests against +467 in the existing baseline. The baseline-file and frozen-hash checks passed; +the count check failed. Existing tests, lockfiles, configs, and baseline remain +untouched. Additional unrelated work is present in this shared checkout. + +Eight sessions per model is a feasibility target, not a model-recognition claim +gate. The proposal requires at least ten independently verified groups per model +for such a claim, plus the hierarchy, null, and nuisance checks. A DVIDS-only +pilot cannot establish cross-outlet invariance. Producer and camera differences +within DVIDS must still be tracked. Public metadata is not audio-quality or +independence evidence. diff --git a/experiments/military_signature/P1_ADMISSION_DRAFT.md b/experiments/military_signature/P1_ADMISSION_DRAFT.md new file mode 100644 index 0000000..ce080a5 --- /dev/null +++ b/experiments/military_signature/P1_ADMISSION_DRAFT.md @@ -0,0 +1,88 @@ +# Bounded P1 media-admission draft + +2026-10-11 local date; **draft for a separate user decision, not authorization**. +P0 authorization explicitly excludes real-media collection and processing. +The [P0 protocol](PROTOCOL_DRAFT.md) and original 55-file freeze remain unchanged. +This stage prepares a reviewable admitted/rejected media population before a +separate Tier 0 pilot decision. It does not execute the six-model × eight-group +pilot or establish any model-recognition result. + +## Proposed action and bounds + +If explicitly authorized, implement a separately versioned admission path and +inspect rights/provenance for the ten named DVIDS assets below. Collect and +process an asset only after its per-asset restrictions and intended research +use pass a recorded admission decision. Unclear or conflicting rights remain +quarantined and require a specific resolution; a public-domain banner alone +does not clear them. + +| Asset ID | Public metadata URL | +|---|---| +| 1754 | [Abrams b-roll](https://www.dvidshub.net/video/1754/abrams-tank-b-roll) | +| 572786 | [Abrams package](https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package) | +| 855729 | [Finnish Leopard](https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank) | +| 776583 | [Bradley b-roll](https://www.dvidshub.net/video/776583/b-roll-stallions-conduct-bradley-live-fire) | +| 386317 | [Stryker convoy](https://www.dvidshub.net/video/386317/m1126-stryker-combat-vehicle-convoy-ramstein-air-base) | +| 937004 | [Pinon Canyon package](https://www.dvidshub.net/video/937004/pinon-canyon-maneuver-site-b-roll-package) | +| 329602 | [M1151 training](https://www.dvidshub.net/video/329602/humvee-training-b-roll) | +| 607339 | [M109A6 firing](https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike) | +| 967175 | [Army birthday parade](https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll) | +| 966208 | [Staged vehicles](https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll) | + +Bounds are ten asset IDs; one selected media rendition per admitted asset; at +most 3 GB total downloaded media; at most 20 minutes per asset and 45 minutes +total decoded source duration. The current page durations sum to 42 minutes +11 seconds, but actual media must satisfy the bounds. Failed, unavailable or +rejected assets are retained as outcomes. No unlisted substitute is collected. +The restricted NATO Leopard asset 879491 is excluded from this draft. + +Collection, original-media hashes, native audio extraction, blind review packets, +label-blind reuse checks, provenance grouping and reviewed interval import/export +are the proposed processing scope. Representation extraction, classifier fits, +CAST fits, contrastive learning, scientific retrieval/probe metrics and protected +source access remain outside this stage. All new media/results remain external, +resolved through `abvid.paths`; no existing raw data, protected roles, historical +manifests or frozen P0 files are changed. Human-only harness baseline changes +and merge approval are not part of this authorization. + +## Required admission sequence + +1. Preserve the source page, rights statement, date/unit/location conflicts, + producer attribution, rendition metadata and retrieval failures. Record the + intended use as local acoustic research without publication or + redistribution of the media. Resolve restrictions before accessing media. +2. Enforce byte/duration bounds and hash the chosen original rendition. Retain + its native audio rate/channels and extraction provenance. No quality label, + engine identity or session independence is inferred from captions. +3. Use a new admission configuration/tool version. The current P0 configuration + permits synthetic fixtures only and must continue to reject real media. + Verify the new processing mechanics before using admitted files. +4. Build randomized caption-hidden audio/waveform/spectrogram review packets. + A human reviews audible vehicles, foreground/background intervals, + contaminants, motion and rejection reasons. Caption shot-log intervals are + not used to preselect or accept audio. If no reviewer is available, stop at + the packet handoff and report that admission is incomplete. +5. Audit reuse and retain all known or suspected shoot/population links. Session + and individual IDs remain `unknown` unless evidence establishes them. + Assets with unresolved independence cannot fill a verified-group quota or + be divided across experimental roles. No train/development/test allocation + is performed in this admission stage. +6. Import human annotations, retain every rejection and export accepted clips + with native-frame provenance. Attach caption labels separately after blind + trimming, with confidence and evidence. Mixed-model event material remains + diagnostic unless a new protocol establishes valid model/interval identity. + +## Outputs and stopping rule + +Deliver a rights/provenance admission ledger, native-media/extraction hashes, +caption-hidden packets, imported human annotations where available, conservative +group/reuse audit, rejected/unavailable asset list and an admission report. The +report must state admitted foreground/background support, verified independent +groups, known physical-vehicle support and remaining unknowns, including zero +support where appropriate. Store them under a fresh external P1-admission run. + +Stop after these ten assets, at a budget bound, or at a required rights/reviewer +handoff. This draft does not permit collecting until a positive gate appears. +The six-class identity definition, eight-group pilot coverage, exact frozen +representation/checkpoint, fit budgets and Tier 0 evaluation allocation must +be resolved in a separate pilot protocol before fitting or metric evaluation. diff --git a/experiments/military_signature/P1_AUTHORIZATION.md b/experiments/military_signature/P1_AUTHORIZATION.md new file mode 100644 index 0000000..0e9c9d8 --- /dev/null +++ b/experiments/military_signature/P1_AUTHORIZATION.md @@ -0,0 +1,27 @@ +# P1 admission authorization and current limits + +The user authorized more real-media collection after reviewing the bounded +P1 admission request: “Get more data,” retaining the per-asset time limit. +The initial 20 GB allocation was then replaced by “lets start with 5 GB.” +The [current contract](p1_admission_5gb.json) therefore uses **5,000,000,000 bytes +total across the new data and artifact roots**, with **1,200 seconds per asset**. +It expands the original ten-asset discovery list; the old 45-minute aggregate +draft limit no longer defines this expanded batch. Storage is a ceiling, not a +requirement to fill the disk. The first collection target is 64 playable assets +across the six candidate caption classes, subject to the budget and availability. + +Collection and preparation are authorized. Rights/provenance review, original +rendition retention, native audio preparation, caption-hidden human review and +label-blind reuse auditing are within scope. Fitting, representation extraction, +protected-source evaluation and media publication remain outside it. + +All original P0 files and their 55-file freeze remain unchanged. A separately +versioned P1 implementation and external run are used; real files are never +passed through P0's synthetic-only admission guards. Existing archival source +catalogs are read as metadata solely to exclude known historical assets/filenames. +Protected media are not opened or used for waveform comparisons. + +Metadata diversity across dates, units and sites helps candidate selection but +does not establish independent sessions or physical vehicles. All collected +material is pending blind review and conservative provenance admission. Captions +are retained privately and do not preselect accepted audio intervals. diff --git a/experiments/military_signature/PILOT_METADATA_FEASIBILITY.md b/experiments/military_signature/PILOT_METADATA_FEASIBILITY.md new file mode 100644 index 0000000..a4946d8 --- /dev/null +++ b/experiments/military_signature/PILOT_METADATA_FEASIBILITY.md @@ -0,0 +1,54 @@ +# P0 six-class metadata feasibility follow-up + +2026-10-11 local date. This supplement adds public HTML metadata to the frozen +[candidate ledger](candidate_metadata.json). The original ledger and P0 freeze +remain unchanged. The [new candidate ledger](candidate_metadata_supplement.json) +contains four additional assets, retrieval hashes, caption-only observations, +unknowns and dependence leads. + +The combined ledgers now have eleven asset leads spanning six candidate caption +classes: M1 Abrams, Leopard, Bradley, Stryker, M109A6 and HMMWV M1151. This resolves +the lack of a metadata lead for two proposed classes. It does not establish a +six-model roster, eight independent sessions per model or usable vehicle audio. +The verified independent-recording-group count remains zero. + +| Additional asset | Metadata evidence | Admission issue | +|---|---|---| +| [M1151 training, 329602](https://www.dvidshub.net/video/329602/humvee-training-b-roll) | Caption names M1151; April 3, 2014; 128th MPAD; 263 seconds. | One caption lead; engine and audible intervals unknown. The M1165A1 brochure does not identify this M1151 engine. | +| [M109A6 firing, 607339](https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike) | Caption names M109A6 and firing at Camp Guernsey; June 9, 2018; Arkansas NG PAO; 337 seconds. | Gunfire described; contamination and clean engine/background support not auditioned. | +| [Parade package, 967175](https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll) | June 14, 2025; XVIII Airborne Corps; 639 seconds. Caption shot log names M109A6, Stryker and Bradley. | Labelled timecodes cannot become blind audio annotations. Event/unit links imply possible dependence; crowd/ceremony audio is unknown. | +| [Staged vehicles, 966208](https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll) | Caption names M1A2, M2 Bradley and M109A6 arriving for the Army birthday parade; 234 seconds. | Page date June 9 differs from staging text June 10. Transport does not prove running engines. | + +The last two captions identify the same Army birthday event and a 1st Cavalry +Division vehicle population. This is a lead for possible repeated physical +vehicles, not proof of a shared shoot or media reuse. Preserve the link and +resolve it before counting independent groups. Different dates and producer +bylines do not establish independent vehicle populations. + +The parade caption also names an **M1 heavy wrecker**. This term cannot be +converted into an M1 Abrams label by matching “M1.” Other historical and +out-of-roster vehicles in that package do not expand the pilot roster. Caption +shot-log frame rates and correspondence to audible vehicles remain unknown. + +A bounded exact-M1165/M1165A1 search returned publications but no usable +exact-variant video lead. That result does not prove absence from DVIDS. M1151 +is a distinct caption-supported lead; it does not inherit M1165A1 engine evidence. + +## What a future roster must resolve + +The six caption classes mix family and variant granularity. The future protocol +must decide whether L4 identities are families, models or specified variants, +then freeze that choice before collection and evaluation. M109A6 and an +unspecified M109 cannot silently share a confirmed engine label. “HMMWV/JLTV” +cannot be one identity. All recording-specific powerplants remain unknown. + +Both metadata ledgers contain only leads. Asset IDs, durations, public-domain +banners and caption diversity do not establish audio suitability, rights for +the intended use, background availability, independent shoots or physical +vehicle identities. No scripts, embedded players or associated media were +loaded. Public HTML snapshots and extracted visible text are retained at +`artifact:military_signature/p0/public-metadata-followup-20261011/`. + +A bounded [P1 admission draft](P1_ADMISSION_DRAFT.md) makes the next action +reviewable. It is a proposal for rights/provenance and media admission; it does +not authorize collection or a Tier 0/model experiment. diff --git a/experiments/military_signature/PLAN.md b/experiments/military_signature/PLAN.md index f42e421..73c3573 100644 --- a/experiments/military_signature/PLAN.md +++ b/experiments/military_signature/PLAN.md @@ -5,7 +5,11 @@ unsupervised side project that learns the postulated acoustic signature of milit vehicle models (T-72, T-80, T-90, M1, Leopard and others) and feeds it back into CAST. The full design and its rationale are in [PROPOSAL.md](PROPOSAL.md). -**Status: plan only.** Nothing is collected, fitted or authorized. +**Current P0:** preparation is separately authorized and documented in +[PROTOCOL_DRAFT.md](PROTOCOL_DRAFT.md), [P0_STATUS.md](P0_STATUS.md), and the +[research report](../../reports/military_signature_P0.md). No real media has been +collected or processed and no model has been fitted. The outline below is the +original plan; later phases still need separate authorization. ## 0. Scope diff --git a/experiments/military_signature/PROPOSAL.md b/experiments/military_signature/PROPOSAL.md index 58935fa..0f46625 100644 --- a/experiments/military_signature/PROPOSAL.md +++ b/experiments/military_signature/PROPOSAL.md @@ -8,6 +8,10 @@ workflow or the earlier benchmark, and it does not change their historical results or the roles of protected sources (including the reviewed T-80BVM/T-90M segments). +P0 preparation is now separately authorized in the user session. Its corrections, +evidence and remaining gates are in [PROTOCOL_DRAFT.md](PROTOCOL_DRAFT.md) and +[P0_STATUS.md](P0_STATUS.md). The proposal below remains the design history. + ## Question Does real-world audio of particular military vehicle models contain a diff --git a/experiments/military_signature/PROTOCOL_DRAFT.md b/experiments/military_signature/PROTOCOL_DRAFT.md new file mode 100644 index 0000000..0507e05 --- /dev/null +++ b/experiments/military_signature/PROTOCOL_DRAFT.md @@ -0,0 +1,281 @@ +# Military acoustic signature: frozen P0 draft + +Protocol `military_signature_p0_v1`, 2026-10-11. This document and +`p0_protocol.json` specify preparation and a prospective study. The original +[PLAN.md](PLAN.md) and [PROPOSAL.md](PROPOSAL.md) remain the design history; the +corrections below govern P0. Freeze hashes are in `p0_freeze.json`. + +## Authorization and result boundary + +P0 permits literature research, public HTML metadata inspection, protocol design, +and local implementation with artificial fixtures. It permits no real media +collection or processing, representation extraction, classifier/CAST fitting, +protected-source evaluation, or P1 execution. The CLI checks the canonical P0 +configuration and refuses real-media manifests. This is a declared-provenance +guard, not a cryptographic test of whether a waveform is artificial. + +The separately scoped project does not restart the retired military benchmark, +Milestone 7 or OpenClaw. Historical source roles, locks and evidence stay intact. +Reusable implementation is under `src/abvid/signature`. Media, features and +generated validation results use external `data:` and `artifact:` roots resolved +through `abvid.paths`. No large artifact belongs in Git. + +P0 outputs establish preparation-tool behavior on artificial chirp/noise signals. +They establish neither vehicle acoustics nor classification performance. No +synthetic CAST source recovery or transfer experiment has been run. + +## Evidence and labels + +Read each literature Markdown representation before its PDF; verify substantive +claims against the PDF and record paper ID, one-based PDF page, printed page and +section/table. `source_evidence.json` separates author statements, interpretations +and unknowns. Public metadata snapshots have retrieval times and byte hashes in +`candidate_metadata.json`; they are candidates, not admitted recordings. + +Captions supply uncertain model/variant labels. Record the exact supporting +caption, source locator and confidence; keep variants unknown when not stated. +Attach labels after blind trimming. A separate curator holds the captions and +private mapping; a reviewer receives only the blind directory. The acoustic +reviewer must not inspect source pages or filenames while selecting intervals. +Sounds can themselves reveal identity, so caption hiding does not prove complete +experimental blinding. The same agent reviewing both roles is a tooling check, +not an independent blinded human review. + +L1 is tracked/wheeled; L2 is turbine/diesel/other. Apply these labels only with +evidence for the actual identified model/variant. Do not turn an uncertain family +caption into an exact engine. L3 denotes engine-order structure only where order +ground truth is established; otherwise it remains unknown and no L3 AP is +reported. L4 is model; L5 variant is exploratory and has no claim gate here. + +The present evidence verifies the M1/AGT1500 turbine association and the diesel +Leopard 1/2 examples in P04. Exact engines for the candidate variants, and the +proposal's T-72/T-80/T-90/T-80UD table, remain incomplete. Inaccessible primary +sources are recorded as failures. Memory and search snippets do not complete +that table. + +## Manifest and admission + +`manifest_schema.json` is JSON Schema Draft 2020-12. The runtime additionally +checks interval references, provenance consistency, connected groups and roles. +All fields are explicit; unavailable values use `unknown`. + +- Assets retain URL, title, date taken/posted, producing unit, producer, location, + outlet, per-asset rights note/URL, retrieval time, raw SHA256, native rate, + channels, duration, recording session and its evidence, verified-independence + flag, physical vehicle identity and supporting evidence. Preserve conflicting + dates in the metadata ledger; do not invent a session from a filename or ID. +- Segments retain native-time boundaries, foreground/background/rejected kind, + role, audible vehicle count, motion, contaminants, reviewer/tool version, + unknown distance/microphone where unmeasured, and rejection reasons. +- Labels live in a separate table keyed by segment. Reuse links record evidence. + Model/variant labels never enter the fingerprint algorithm. + +Distinct asset IDs, units, posting dates or venues alone do not prove independent +recordings. Session evidence must establish distinct recording events with all +known source links considered. Repeated views of an event, shared raw hashes, +suspected/confirmed waveform reuse and known repeated physical vehicles form +one transitive component. Session IDs are globally scoped, never local counters. +Known same-individual vehicles stay together across roles. Unknown physical +identity stays unknown; it does not certify unseen-individual generalization. + +All unknown-session assets are conservatively joined and cannot enter held-out +roles. They are research leads requiring admission, not extra independent groups. +Roles are assigned to whole connected components after the reuse audit; accepted +segments from one component cannot cross development/validation/evaluation. +Quarantined rejection records remain in the manifest and are never training data. + +The schema can describe future real assets, but P0 executable validation rejects +them. A future authorized phase requires its own versioned configuration and +admission checks; flipping a P0 flag is deliberately rejected. + +## Frozen trimming rules + +The offline viewer provides randomized opaque recording IDs, original-rate audio, +waveform and spectrogram, with no caption/title/original filename/session labels. +Its 8 kHz mono spectrogram is a display only: 512-point Hann FFT, 128-sample hop, +0–4 kHz, 80 dB below the recording maximum. Display amplitude is relative. No +denoising, equalization or loudness change is applied to playable/trimmed audio. +Re-encoding to FLOAT WAV removes source tags; it is not a retained raw master. + +Record all auditioned exclusion intervals and reasons. Unreviewed time is not +silently admitted. Rejections may overlap accepted intervals as revision records; +accepted intervals may not overlap each other. A reviewed asset must have at +least one interval or a logged rejection before annotations can be saved/imported. + +1. Accept foreground only when exactly one vehicle is audibly distinguishable. + Accept 2–60 seconds. Reject music, narration, gunfire or aircraft contamination; + reject masking wind, clipping, unclear count, multiple vehicles, uncertain + audibility, prominent edits and codec damage. Log the applicable reason. + Mark lesser wind/crowd/AGC artifacts; retain a clean-reference subset for the + primary analysis. Unknown contamination prevents admission to that subset. +2. Obtain at least one uncontaminated 2–60 second, zero-audible-vehicle background + interval from the same raw asset, preferably adjacent without a camera edit. + Log absence explicitly. Another asset in a session is not automatically an + equivalent channel/noise reference. Backgrounds remain in the foreground's + group and role. Missing/contaminated backgrounds preclude normalized analyses. +3. Motion may be idle/rev/accelerate/constant/pass_by/brake/unknown. Audio alone + need not distinguish speed or geometry; do not guess distance, RPM or microphone. +4. Preserve raw hashes and export at the native rate and original channel count. + Frame bounds are `floor(start_seconds * rate)` and `ceil(end_seconds * rate)`; + save those integers, clip SHA256 and raw SHA256. This expands by less than one + sample at each end. The exporter changes no samples before FLOAT serialization. +5. Keep blind files and private mapping in separate directories. Import checks + manifest/config/batch commitments, mapping integrity and tool version. Output + paths must be fresh; reruns never overwrite past review or export artifacts. + +The current tooling enforces duration, counts, key forbidden contaminants, overlap +and provenance. Audibility, masking, clipping and semantic review remain reviewer +judgments. A synthetic oracle validates mechanics, not reviewer accuracy. Before +P1, an authorized human should complete the supplied synthetic packet in a real +browser and inspect the annotation/import round trip. + +## Reuse audit + +Reuse `abvid.data.overlap` without changes: mean-channel audio at 8 kHz, landmark +candidate search, then absolute normalized waveform correlation at least 0.9, +with the canonical algorithm settings recorded in `p0_protocol.json`. Index +four-second reference windows every two seconds, at most 600 per asset. Assets +are bounded at 1,200 seconds. Exact raw hashes also join assets. + +Matches are conservative grouping suggestions, retained with correlation, +offsets, votes and input hashes. Suspected reuse is merged before splitting even +when it reduces the apparent sample count. Shorter-than-four-second recordings, +silence and no-landmark windows remain unresolved. Boundary overlaps, averaging +opposite stereo channels, codecs, edits, mixtures or time stretching can defeat +the audit; no match certifies independence. Periodic signals may overmerge true +independent events. Record both limitations rather than deleting awkward assets. + +## Background and physical identifiability + +For `x = Hs + n`, a source-free background observes `n`, not the unknown channel +response `H`. Different `(H,s)` pairs produce the same `Hs`. This is an analyst +identifiability argument, not a claim that the cited papers validated inversion. +Background spectra permit a bounded noise reference or whitening ablation. They +do not remove microphone/propagation effects or turn CAST effective-observation +fits into clean source parameters. Use training-only statistics and freeze the +background operation before target evaluation. Inversion requires independent +calibration or separately justified assumptions in a new protocol. + +Source-model dominance over propagation mismatch remains a hypothesis. Report +effective descriptor stability and failures without calling them combustion, +true RPM, track ground truth or recovered physical sources. + +## Prospective evaluation contract + +Everything in this section is a draft for separately authorized phases. No model +or metric implementation is executed by P0. Freeze checkpoints, layer/pooling, +window length, CAST budget and population before extraction. Use frozen BEATs768, +MFCC26 and declared per-segment CAST observations in Tier 0. UMAP/HDBSCAN plots +are exploratory; primary retrieval uses original normalized descriptors and a +fixed distance, not an embedding selected using vehicle labels. Selection of the +exact checkpoint and extraction conventions is a remaining P1 input. + +Vehicle labels are allowed only for evaluation probes/metrics, not contrastive +representation learning or hyperparameter selection. Probe preprocessing fits +training groups only. Reporting phase gates uses development labels and exposes +development results; it does not create an untouched final test. Nuisance labels +may supervise a declared training-only session/outlet adversary in P3. + +**Eight versus ten.** The P1 target is six specified models with eight admitted, +verified groups each (48 groups, not 48 videos). It is a feasibility/development +pilot. Use four group folds, each holding two groups per model, with fixed seeds +17,29,43,71,101. If quotas are unmet, report the achieved population and unmet +cells, without weakening the threshold after seeing results. Six models and +eight groups are targets; they are not currently evidenced. + +An L4 claim requires at least ten verified groups per model in the claim +population, in addition to label quality, hierarchy/null/nuisance checks and new +physical-vehicle support. This number is an original proposal threshold, not a +power analysis or sufficiency guarantee. A subsequent confirmatory protocol must +reserve new, unexposed groups and document multiple physical vehicles per model +with identities disjoint from training. Unknown physical identity supports only +an unseen-recording claim. Do not relabel the exposed eight-session pilot as +confirmatory evidence by adding two recordings. Final-test allocation and power +analysis require a new pre-evaluation freeze and are unresolved in P0. + +**Retrieval.** Exclude the query's entire connected component from the gallery. +Compute AP at L1–L4 only with supported labels and at least one eligible positive +in another component. Average queries within components, then components within +classes, then classes. Report excluded queries and per-class/group support. For +model-level primary retrieval, use single-model groups; mixed-model events remain +linked and are retained as secondary, matched-event diagnostics. Their role as +confound controls must not create extra independent group counts. + +**Null and uncertainty.** Permute group-level label blocks 1,999 times, preserving +within-group relations; never permute windows independently. For the primary +single-model groups, shuffle complete model labels among exchangeable groups and +derive their hierarchy labels. Use producer/location strata only where there is +exchangeability and class overlap. If the allocation has no defensible +exchangeable support, the null is unknown and the claim gate fails. Use 2,000 +class-stratified group bootstrap replicates, resampling whole components; keep +methods paired in difference estimates. Report the 95% CI and all five seeds. +Seeds/windows are not additional independent experimental units. + +**Confounds.** Report balanced accuracy and the chance-normalized score +`N = (BA - 1/K) / (1 - 1/K)` for each K-class probe, including negative values. +K=1 is undefined and remains unknown. Vehicle probes hold out connected groups. +Session probes use a declared blocked-time 70/30 split within known groups with +at least two temporal blocks; these are within-session diagnostics. Producer and +outlet probes must hold out groups with label overlap and use only training-fit +preprocessing. The different session/model probe tasks prevent causal attribution +from their score difference; report both protocols and class counts. + +Define descriptive session dominance as `N_session - N_model >= 0.10` with its +paired group-bootstrap lower CI above zero, both before and after the fixed noise +reference ablation. Retain a ratio only as an optional secondary diagnostic with +an explicit denominator; raw accuracies with different chance levels are not a +confound ratio. A nuisance reduction accompanied by lost model retrieval is a +failure, not successful invariance. Single constant-outlet DVIDS yields no outlet +probe and cannot establish outlet invariance. Distinguish distribution outlet, +originator, producing unit and camera (camera usually unknown). + +## Measurable phase gates + +| Phase | Required evidence before continuation | +|---|---| +| P0 | Canonical config/schema match; focused regression and artificial end-to-end checks pass; source/unknown ledger and freeze check pass. Report any canary/human-review handoff. P1 requires separate collection and execution authorization. | +| P1 | Admission/reuse audit frozen; 6 × 8 group target achieved or explicitly reported as an incomplete feasibility population; baselines, cross-group retrieval, normalized nuisance probes and missing-background counts reported. Stop escalation if session dominance persists after the frozen noise-reference ablation and coarse hierarchy has no above-null evidence. | +| P2 | No fine model claim without L1/L2 macro mAP-minus-null lower 95% CI above zero and permutation p ≤ 0.025 each. Report only the supported population: M1-only turbine support makes L2 a model-associated contrast, not general turbine/diesel invariance. Order ground truth absent means L3 unknown. CAST source/nuisance recovery needs a separately frozen synthetic generator and negative controls; its numerical thresholds remain a P2 design input. | +| P3 | Paired cross-group mAP improvement over Tier 1 has lower 95% CI above zero; session-probe score-change upper CI ≤ 0; no label-based representation/hyperparameter selection. Report every seed and failed hierarchy level. A constant-outlet probe remains unknown. | +| P4 | Separate outlet/media authorization and admission; overlapping model and hierarchy classes across origin outlets plus frozen unseen-outlet groups. Russian models occurring only at new outlets do not identify an outlet effect. Earlier protected assets keep their roles. | + +The P2 gate's L2 limit is material: a DVIDS pilot with only M1 turbines cannot +separate turbine characteristics from M1-associated cues. It can document this +conditional contrast and feasibility; broad powerplant or model recognition +requires additional matched support. No phase gate authorizes its successor. + +For open-set evaluation, hold one model and all its linked groups out of training; +freeze a rejection threshold using training/validation only, report unknown-model +recall and known-model false rejection. This requires additional group support +and is not a result of the pilot by default. + +## Reproduction and handoff + +Run from the repo using its existing CAST environment, without changing extras, +lockfiles or the baseline. Choose fresh external output suffixes: + +```sh +PYTHONPATH=src .venv/bin/python -m abvid.signature validate-p0 \ + --fixtures data:military_signature/p0_fixtures/my-check \ + --output artifact:military_signature/p0/my-check +.venv/bin/python -m pytest tests/test_signature_p0.py tests/test_overlap_audit.py +PYTHONPATH=src .venv/bin/python experiments/military_signature/verify_freeze.py +``` + +The validator records fixture seeds/raw hashes, environment, commit, dirty-tree +status, code/config/schema hashes, native-frame comparisons and rejections. +Its blind directory can be opened offline and reviewed; keep `private/` with the +curator. Use the CLI `review`, `import-annotations`, `export`, `dedup`, `group`, +`validate` and `readiness` subcommands for individual steps (`--help` lists args). + +Human-only work: inspect all new test lines before merge, refresh the harness +baseline in BOTH environments in the same legitimate suite-change commit, and +review any other protected-file diffs in this shared dirty checkout. An agent +must not refresh the baseline or suppress its failure. Browser playback/layout +requires a human check because Firefox computer-use access was rejected. + +Scientific handoff: authorize a bounded P1 collection/execution protocol, identify +its exact checkpoint/extraction choices, resolve asset rights and independence, +and specify the confirmatory allocation before any model-level claim. Current +metadata, artificial tooling checks and historical protected sources do not +satisfy those inputs. diff --git a/experiments/military_signature/candidate_metadata.json b/experiments/military_signature/candidate_metadata.json new file mode 100644 index 0000000..832089d --- /dev/null +++ b/experiments/military_signature/candidate_metadata.json @@ -0,0 +1,336 @@ +{ + "scope": "public_HTML_metadata_only_not_a_media_manifest", + "created_at": "2026-10-11T21:39:18.817740+00:00", + "candidates": [ + { + "asset_id": "DVIDS-1754", + "url": "https://www.dvidshub.net/video/1754/abrams-tank-b-roll", + "title": "Abrams Tank b-roll", + "page_date_taken": "2005-01-27", + "page_date_taken_raw": "01.27.2005", + "page_date_posted_raw": "03.01.2005 04:22", + "unit": "AFN Iraq", + "location": "unknown", + "outlet": "DVIDS", + "originator": "AFN Iraq", + "duration_seconds_from_page": 83, + "caption_model": "M1 Abrams", + "caption_variant": "unknown", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_1754", + "url": "https://www.dvidshub.net/video/1754/abrams-tank-b-roll", + "retrieved_at": "2026-10-11T21:33:55.986155+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/1754/abrams-tank-b-roll", + "sha256": "fd145fdf5ce48657a6f515a1a39b638321d9a0985646e2c7a09b0efd4a774888", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_1754.html" + }, + "flags": [], + "producer_byline": "unknown", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-572786", + "url": "https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package", + "title": "TAA Abrams Set UP B-roll Package", + "page_date_taken": "2017-12-04", + "page_date_taken_raw": "12.04.2017", + "page_date_posted_raw": "01.18.2018 22:25", + "unit": "1st Marine Division", + "location": "Twentynine Palms, California, US", + "outlet": "DVIDS", + "originator": "1st Marine Division", + "duration_seconds_from_page": 505, + "caption_model": "M1 Abrams", + "caption_variant": "M1A1 (tag only)", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_572786", + "url": "https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package", + "retrieved_at": "2026-10-11T21:33:55.986747+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package", + "sha256": "838dfa8f02d0a66f23291dfe23b68b5314af514d9f5c7de2225cf5258c9befe9", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_572786.html" + }, + "flags": [], + "producer_byline": "Sgt. Christopher Thompson", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-855729", + "url": "https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank", + "title": "U.S. Soldiers explore the Finnish Leopard battle tank", + "page_date_taken": "2022-08-15", + "page_date_taken_raw": "08.15.2022", + "page_date_posted_raw": "08.31.2022 05:18", + "unit": "5th Mobile Public Affairs Detachment", + "location": "Finland (page); Rovaniemi Training Area (caption)", + "outlet": "DVIDS", + "originator": "5th Mobile Public Affairs Detachment", + "duration_seconds_from_page": 38, + "caption_model": "Leopard 2", + "caption_variant": "2A4 (caption)", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_855729", + "url": "https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank", + "retrieved_at": "2026-10-11T21:33:55.987939+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank", + "sha256": "5ca528f96c4341a83db81a924c642a1af5b1be0c18199edc2294ad99c48ab45e", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_855729.html" + }, + "flags": [ + "Cultural-exchange scene may have speech/static inspection; audio quality unknown", + "Page byline and caption credit differ; retain both" + ], + "producer_byline": "Spc. Elizabeth DeGroot", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-776583", + "url": "https://www.dvidshub.net/video/776583/b-roll-stallions-conduct-bradley-live-fire", + "title": "B-roll Stallions conduct Bradley live fire", + "page_date_taken": "2020-12-09", + "page_date_taken_raw": "12.09.2020", + "page_date_posted_raw": "12.15.2020 05:05", + "unit": "319th Mobile Public Affairs Detachment", + "location": "Pabrade, Lithuania", + "outlet": "DVIDS", + "originator": "319th Mobile Public Affairs Detachment", + "duration_seconds_from_page": 43, + "caption_model": "Bradley", + "caption_variant": "unknown", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_776583", + "url": "https://www.dvidshub.net/video/776583/b-roll-stallions-conduct-bradley-live-fire", + "retrieved_at": "2026-10-11T21:33:55.988195+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/776583/b-roll-stallions-conduct-bradley-live-fire", + "sha256": "600241370b41003c267e662fe47d202c0362238beb23a4e87036c9b2a14c1658", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_776583.html" + }, + "flags": [ + "Caption describes attempted firing; gunfire contamination unknown" + ], + "producer_byline": "Spc. Daria Jackson", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-386317", + "url": "https://www.dvidshub.net/video/386317/m1126-stryker-combat-vehicle-convoy-ramstein-air-base", + "title": "M1126 Stryker Combat Vehicle Convoy to Ramstein Air Base", + "page_date_taken": "2015-01-09", + "page_date_taken_raw": "01.09.2015", + "page_date_posted_raw": "01.09.2015 23:07", + "unit": "Defense Imagery Management Operations Center", + "location": "Ramstein Air Base, Germany", + "outlet": "DVIDS", + "originator": "Defense Imagery Management Operations Center", + "duration_seconds_from_page": 126, + "caption_model": "Stryker", + "caption_variant": "M1126 (caption)", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_386317", + "url": "https://www.dvidshub.net/video/386317/m1126-stryker-combat-vehicle-convoy-ramstein-air-base", + "retrieved_at": "2026-10-11T21:33:55.988748+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/386317/m1126-stryker-combat-vehicle-convoy-ramstein-air-base", + "sha256": "9ff9f512351f7ba585aefd8f0e4c9737e85d7b73688f9296817eb34e0fd0f68e", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_386317.html" + }, + "flags": [ + "Convoy caption does not establish one audible vehicle" + ], + "producer_byline": "unknown", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-937004", + "url": "https://www.dvidshub.net/video/937004/pinon-canyon-maneuver-site-b-roll-package", + "title": "Piñon Canyon Maneuver Site B-Roll Package", + "page_date_taken": "2024-09-04", + "page_date_taken_raw": "09.04.2024", + "page_date_posted_raw": "09.17.2024 20:24", + "unit": "4th Infantry Division Public Affairs Office", + "location": "Fort Carson (page); Piñon Canyon to Fort Carson (caption)", + "outlet": "DVIDS", + "originator": "4th Infantry Division Public Affairs Office", + "duration_seconds_from_page": 263, + "caption_model": "Stryker", + "caption_variant": "M1126 (caption)", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Public-domain banner shown; inspect full asset restrictions before collection", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_937004", + "url": "https://www.dvidshub.net/video/937004/pinon-canyon-maneuver-site-b-roll-package", + "retrieved_at": "2026-10-11T21:33:55.989906+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/937004/pinon-canyon-maneuver-site-b-roll-package", + "sha256": "7489eab33b97145305b7e04ecc8a27c33462cecd52314b9d6a680daa8af41aee", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_937004.html" + }, + "flags": [ + "Caption date 2024-09-03 differs from page date 2024-09-04; unresolved", + "Convoy may contain multiple audible vehicles; unknown" + ], + "producer_byline": "Pfc. Jonathan Reyes", + "camera_identity": "unknown" + }, + { + "asset_id": "DVIDS-879491", + "url": "https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll", + "title": "NATO Allies train Ukrainian soldiers on Leopard tanks in Poland (B-ROLL)", + "page_date_taken": "2023-05-04", + "page_date_taken_raw": "05.04.2023", + "page_date_posted_raw": "04.13.2023 10:51", + "unit": "Natochannel", + "location": "Poland", + "outlet": "DVIDS", + "originator": "Natochannel", + "duration_seconds_from_page": 283, + "caption_model": "Leopard", + "caption_variant": "unknown", + "label_confidence": "caption", + "license_status": "restricted", + "license_note": "NATO editorial-only purpose restriction; research reuse unresolved", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "media_access_performed": false, + "metadata_snapshot": { + "source_id": "DVIDS_879491", + "url": "https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll", + "retrieved_at": "2026-10-11T21:33:55.990920+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll", + "sha256": "9d407bb7dc6afb7b4b4259b2f6d1661c9fd781ed67e21daad70786e25907ff35", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_879491.html" + }, + "flags": [ + "Page Date Taken 05.04.2023 follows posting 04.13.2023; unresolved", + "Editorial-use restriction despite public-domain banner; quarantine pending purpose-specific rights review" + ], + "producer_byline": "unknown", + "camera_identity": "unknown" + } + ], + "guidance_source": { + "source_id": "DVIDS_FAQ", + "url": "https://www.dvidshub.net/about/faq", + "retrieved_at": "2026-10-11T21:33:55.992330+00:00", + "mode": "HTML_only_no_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/about/faq", + "sha256": "d2212950d4cf93d54a509184e2d539a1d4ce9e4c82dcf99caa555750fb3865d9", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_FAQ.html" + }, + "summary": { + "candidate_assets": 7, + "verified_independent_recording_groups": 0, + "rights_quarantined_candidates": 1, + "models_with_caption_leads": [ + "M1 Abrams", + "Leopard 2/Leopard unspecified", + "Bradley", + "Stryker" + ], + "six_model_eight_group_quota_established": false + } +} diff --git a/experiments/military_signature/candidate_metadata_supplement.json b/experiments/military_signature/candidate_metadata_supplement.json new file mode 100644 index 0000000..fa00643 --- /dev/null +++ b/experiments/military_signature/candidate_metadata_supplement.json @@ -0,0 +1,267 @@ +{ + "supplement_id": "military_signature_p0_metadata_followup_20261011", + "scope": "public_HTML_metadata_only_not_a_media_manifest", + "relationship_to_frozen_candidates": "Additive evidence; original candidate_metadata.json and 55-file freeze unchanged.", + "retrieval_record_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/retrieval.json", + "candidates": [ + { + "outlet": "DVIDS", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Page shows a public-domain banner and refers to DVIDS copyright restrictions; per-asset/intended-use admission remains pending", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "engine_in_recording": "unknown", + "camera_identity": "unknown", + "media_access_performed": false, + "asset_id": "DVIDS-329602", + "url": "https://www.dvidshub.net/video/329602/humvee-training-b-roll", + "title": "Humvee Training B-roll", + "page_date_taken": "2014-04-03", + "page_date_taken_raw": "04.03.2014", + "page_date_posted_raw": "04.14.2014 21:44", + "unit": "128th Mobile Public Affairs Detachment", + "originator": "128th Mobile Public Affairs Detachment", + "producer_byline": "Richard Stowell", + "location": "JOINT BASE MCGUIRE-DIX-LAKEHURST, NEW JERSEY, US", + "duration_seconds_from_page": 263, + "caption_model": "HMMWV M1151", + "caption_variant": "M1151", + "caption_models": [ + "HMMWV M1151" + ], + "metadata_observation": "Caption identifies M1151 driver-licensing training by the 4th Financial Management Sustainment Detachment.", + "flags": [ + "One caption lead does not establish eight independent groups", + "M1165A1 specifications cannot be transferred to this M1151 recording" + ], + "metadata_snapshot": { + "source_id": "DVIDS_329602", + "url": "https://www.dvidshub.net/video/329602/humvee-training-b-roll", + "retrieved_at": "2026-10-12T00:04:37.853847+00:00", + "mode": "HTML_only_no_scripts_or_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/329602/humvee-training-b-roll", + "sha256": "cef27a9eba6e55d290de218180677c1276bd5a98ba7003c346e3e7584903c91b", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_329602.html", + "visible_text_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_329602.txt" + } + }, + { + "outlet": "DVIDS", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Page shows a public-domain banner and refers to DVIDS copyright restrictions; per-asset/intended-use admission remains pending", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "engine_in_recording": "unknown", + "camera_identity": "unknown", + "media_access_performed": false, + "asset_id": "DVIDS-607339", + "url": "https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike", + "title": "Arkansas Field Artillery Unit Fires M109A6 Paladins During Operation Western Strike", + "page_date_taken": "2018-06-09", + "page_date_taken_raw": "06.09.2018", + "page_date_posted_raw": "06.13.2018 17:55", + "unit": "Arkansas National Guard Public Affairs Office", + "originator": "Arkansas National Guard Public Affairs Office", + "producer_byline": "Stephen Wright", + "location": "GUERNSEY, WYOMING, US", + "duration_seconds_from_page": 337, + "caption_model": "M109A6 Paladin", + "caption_variant": "M109A6", + "caption_models": [ + "M109A6 Paladin" + ], + "metadata_observation": "Caption identifies M109A6 firing during Operation Western Strike at Camp Guernsey.", + "flags": [ + "Gunfire described in caption; audio contamination not observed", + "Caption identity does not identify audible vehicle intervals" + ], + "metadata_snapshot": { + "source_id": "DVIDS_607339", + "url": "https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike", + "retrieved_at": "2026-10-12T00:04:37.854690+00:00", + "mode": "HTML_only_no_scripts_or_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike", + "sha256": "facfcf1f08cc02975b3aa3e0e5b5df932de4eeb67e1943e700640fa1a4362a96", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_607339.html", + "visible_text_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_607339.txt" + } + }, + { + "outlet": "DVIDS", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Page shows a public-domain banner and refers to DVIDS copyright restrictions; per-asset/intended-use admission remains pending", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "engine_in_recording": "unknown", + "camera_identity": "unknown", + "media_access_performed": false, + "asset_id": "DVIDS-967175", + "url": "https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll", + "title": "Army 250 Parade Celebration: B-Roll", + "page_date_taken": "2025-06-14", + "page_date_taken_raw": "06.14.2025", + "page_date_posted_raw": "06.17.2025 05:56", + "unit": "XVIII Airborne Corps", + "originator": "XVIII Airborne Corps", + "producer_byline": "Rene Rosas", + "location": "WASHINGTON, DISTRICT OF COLUMBIA, US", + "duration_seconds_from_page": 639, + "caption_model": "multiple models", + "caption_variant": "unknown", + "caption_models": [ + "M109A6 Paladin", + "Stryker", + "Bradley Fighting Vehicle" + ], + "metadata_observation": "The caption shot log names M109A6, Stryker and Bradley among parade entries.", + "caption_segment_log_handling": "Source caption metadata only. Do not import shot-log intervals as blind audio annotations; frame rate, audible vehicles and clean audio are unknown.", + "flags": [ + "Potential mixed-model event diagnostic, not primary single-model grouping evidence", + "Event/unit link to DVIDS-966208; independent physical populations not established", + "Caption also names an M1 heavy wrecker; this is not an M1 Abrams label", + "Caption includes historical and other out-of-roster vehicles", + "Crowd and ceremony context; contamination not auditioned" + ], + "metadata_snapshot": { + "source_id": "DVIDS_967175", + "url": "https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll", + "retrieved_at": "2026-10-12T00:04:37.854841+00:00", + "mode": "HTML_only_no_scripts_or_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll", + "sha256": "7779c6beaecb175d9b3b19405a515775780184d31e251d74e8157518b6f64f88", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_967175.html", + "visible_text_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_967175.txt" + } + }, + { + "outlet": "DVIDS", + "label_confidence": "caption", + "license_status": "public_domain_banner_pending_per_asset_review", + "license_note": "Page shows a public-domain banner and refers to DVIDS copyright restrictions; per-asset/intended-use admission remains pending", + "recording_session": "unknown", + "session_evidence": "unknown", + "independence_verified": false, + "physical_vehicle_id": "unknown", + "raw_media_sha256": "unknown", + "native_sample_rate_hz": "unknown", + "audio_quality": "unknown", + "usable_foreground_seconds": "unknown", + "background_support": "unknown", + "engine_in_recording": "unknown", + "camera_identity": "unknown", + "media_access_performed": false, + "asset_id": "DVIDS-966208", + "url": "https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll", + "title": "U.S. Army vehicles staged at the West Potomac Park Waterfront [B-Roll]", + "page_date_taken": "2025-06-09", + "page_date_taken_raw": "06.09.2025", + "page_date_posted_raw": "06.10.2025 13:44", + "unit": "U.S. Army Military District of Washington", + "originator": "U.S. Army Military District of Washington", + "producer_byline": "Bernabe Lopez", + "location": "DISTRICT OF COLUMBIA, US", + "duration_seconds_from_page": 234, + "caption_model": "multiple models", + "caption_variant": "unknown", + "caption_models": [ + "M1A2 Abrams", + "M2 Bradley", + "M109A6 Paladin" + ], + "metadata_observation": "Caption describes staging for the Army birthday parade and identifies vehicles arriving from 1st Cavalry Division units at Fort Cavazos.", + "flags": [ + "Date Taken June 9 versus caption staging June 10; preserve both", + "Event/unit link to DVIDS-967175; repeated physical vehicles possible", + "Transport/staging context does not prove running engines or clean vehicle audio" + ], + "metadata_snapshot": { + "source_id": "DVIDS_966208", + "url": "https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll", + "retrieved_at": "2026-10-12T00:04:37.855300+00:00", + "mode": "HTML_only_no_scripts_or_associated_media", + "status": "retrieved", + "content_type": "text/html", + "final_url": "https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll", + "sha256": "d5557ac414136ac0264ac08cd969bb1b1f601894e17a0616f5ebb20ef897858e", + "snapshot_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_966208.html", + "visible_text_ref": "artifact:military_signature/p0/public-metadata-followup-20261011/DVIDS_966208.txt" + } + } + ], + "suspected_population_links": [ + { + "asset_ids": [ + "DVIDS-966208", + "DVIDS-967175" + ], + "basis": "Both captions identify the June 2025 Army birthday parade; M109A6 and Bradley entries identify the 1st Cavalry Division population.", + "link_type": "possible common physical-vehicle population; not a proven shared recording session", + "action_before_admission": "Retain this dependence lead. Resolve shoot, reuse and individual identities; do not count the two assets as independent groups from metadata alone." + } + ], + "search_negative_findings": [ + { + "queries": [ + "site:dvidshub.net/video/ \"M1165\"", + "site:dvidshub.net/video/ \"M1165A1\"" + ], + "observation": "Bounded web search returned publications but no usable exact-variant video lead.", + "interpretation": "Search failure does not establish absence from DVIDS. Do not invent M1165A1 independence or substitute M1151 engine evidence." + } + ], + "summary": { + "additional_candidate_assets": 4, + "total_candidate_assets_with_frozen_ledger": 11, + "models_with_caption_leads_across_both_ledgers": [ + "M1 Abrams family", + "Leopard 2 / Leopard unspecified", + "Bradley family", + "Stryker family", + "M109A6 Paladin", + "HMMWV M1151" + ], + "caption_class_count": 6, + "verified_independent_recording_groups": 0, + "six_model_eight_group_quota_established": false, + "usable_audio_support": "unknown", + "model_granularity_frozen_for_P1": false, + "powerplant_labels_for_recordings": "unknown", + "P1_authorized": false + }, + "restrictions": [ + "Caption classes mix family and variant granularity; a future roster must fix the identity definition before collection/evaluation.", + "Shot logs are source metadata; blind trimming cannot use their labelled interval boundaries.", + "All raw media, engine identity, audibility and background-support fields remain unknown.", + "Metadata retrieval did not execute scripts, embedded players or load associated media." + ] +} diff --git a/experiments/military_signature/manifest_schema.json b/experiments/military_signature/manifest_schema.json new file mode 100644 index 0000000..3dd4cc1 --- /dev/null +++ b/experiments/military_signature/manifest_schema.json @@ -0,0 +1,421 @@ +{ + "type": "object", + "properties": { + "schema_version": { + "const": 1 + }, + "protocol_id": { + "const": "military_signature_p0_v1" + }, + "assets": { + "type": "array", + "items": { + "$ref": "#/$defs/asset" + } + }, + "segments": { + "type": "array", + "items": { + "$ref": "#/$defs/segment" + } + }, + "labels": { + "type": "array", + "items": { + "$ref": "#/$defs/label" + } + }, + "reuse_links": { + "type": "array", + "items": { + "$ref": "#/$defs/link" + } + } + }, + "required": [ + "schema_version", + "protocol_id", + "assets", + "segments", + "labels", + "reuse_links" + ], + "additionalProperties": false, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:abvid:military-signature:manifest:1", + "$defs": { + "asset": { + "type": "object", + "properties": { + "asset_id": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "url": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "date_taken": { + "type": "string", + "minLength": 1, + "pattern": "^(unknown|\\d{4}-\\d{2}-\\d{2})$" + }, + "date_posted": { + "type": "string", + "minLength": 1 + }, + "unit": { + "type": "string", + "minLength": 1 + }, + "location": { + "type": "string", + "minLength": 1 + }, + "outlet": { + "type": "string", + "minLength": 1 + }, + "producer": { + "type": "string", + "minLength": 1 + }, + "license_status": { + "enum": [ + "public_domain", + "permitted", + "restricted", + "unknown" + ] + }, + "license_note": { + "type": "string", + "minLength": 1 + }, + "license_url": { + "type": "string", + "minLength": 1 + }, + "retrieved_at": { + "type": "string", + "minLength": 1 + }, + "raw_sha256": { + "type": "string", + "minLength": 1, + "pattern": "^[0-9a-f]{64}$" + }, + "media_ref": { + "type": "string", + "minLength": 1, + "pattern": "^data:military_signature/[^:]+$" + }, + "media_kind": { + "enum": [ + "synthetic_fixture", + "real" + ] + }, + "duration_seconds": { + "type": "number", + "exclusiveMinimum": 0 + }, + "native_sample_rate_hz": { + "type": "integer", + "minimum": 1000 + }, + "channels": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "recording_session": { + "type": "string", + "minLength": 1 + }, + "session_evidence": { + "type": "string", + "minLength": 1 + }, + "independence_verified": { + "type": "boolean" + }, + "physical_vehicle_id": { + "type": "string", + "minLength": 1 + }, + "vehicle_identity_evidence": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "asset_id", + "url", + "title", + "date_taken", + "date_posted", + "unit", + "location", + "outlet", + "producer", + "license_status", + "license_note", + "license_url", + "retrieved_at", + "raw_sha256", + "media_ref", + "media_kind", + "duration_seconds", + "native_sample_rate_hz", + "channels", + "recording_session", + "session_evidence", + "independence_verified", + "physical_vehicle_id", + "vehicle_identity_evidence" + ], + "additionalProperties": false + }, + "segment": { + "type": "object", + "properties": { + "segment_id": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "asset_id": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "start_seconds": { + "type": "number", + "minimum": 0 + }, + "end_seconds": { + "type": "number", + "exclusiveMinimum": 0 + }, + "kind": { + "enum": [ + "foreground", + "background", + "rejected" + ] + }, + "role": { + "enum": [ + "development", + "validation", + "evaluation", + "quarantine" + ] + }, + "rejection_reasons": { + "type": "array", + "items": { + "enum": [ + "music", + "narration", + "gunfire", + "aircraft", + "wind_masking", + "multiple_vehicles", + "no_vehicle", + "no_background", + "too_short", + "clipping", + "editing_or_codec_artifact", + "uncertain_audibility", + "access_failure", + "rights_unresolved", + "other" + ] + }, + "uniqueItems": true + }, + "trimmer_id": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "tool_version": { + "const": "signature-preparation-1" + }, + "blind_review": { + "const": true + }, + "audible_vehicle_count": { + "anyOf": [ + { + "const": "unknown" + }, + { + "type": "integer", + "minimum": 0 + } + ] + }, + "motion_state": { + "enum": [ + "idle", + "rev", + "accelerate", + "constant", + "pass_by", + "brake", + "unknown" + ] + }, + "distance_m": { + "anyOf": [ + { + "const": "unknown" + }, + { + "type": "number", + "minimum": 0 + } + ] + }, + "mic_type": { + "type": "string", + "minLength": 1 + }, + "contaminants": { + "type": "array", + "items": { + "enum": [ + "music", + "narration", + "gunfire", + "aircraft", + "wind", + "crowd", + "agc_pumping", + "unknown" + ] + }, + "uniqueItems": true + } + }, + "required": [ + "segment_id", + "asset_id", + "start_seconds", + "end_seconds", + "kind", + "role", + "rejection_reasons", + "trimmer_id", + "tool_version", + "blind_review", + "audible_vehicle_count", + "motion_state", + "distance_m", + "mic_type", + "contaminants" + ], + "additionalProperties": false + }, + "label": { + "type": "object", + "properties": { + "segment_id": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "caption_model": { + "type": "string", + "minLength": 1 + }, + "caption_variant": { + "type": "string", + "minLength": 1 + }, + "confidence": { + "enum": [ + "confirmed", + "caption", + "unknown" + ] + }, + "L1": { + "enum": [ + "tracked", + "wheeled", + "unknown" + ] + }, + "L2": { + "enum": [ + "turbine", + "diesel", + "other", + "unknown" + ] + }, + "L3": { + "type": "string", + "minLength": 1 + }, + "label_evidence": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "segment_id", + "caption_model", + "caption_variant", + "confidence", + "L1", + "L2", + "L3", + "label_evidence" + ], + "additionalProperties": false + }, + "link": { + "type": "object", + "properties": { + "left": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "right": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$" + }, + "kind": { + "enum": [ + "provenance", + "suspected_reuse", + "waveform_reuse" + ] + }, + "evidence": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "left", + "right", + "kind", + "evidence" + ], + "additionalProperties": false + } + } +} diff --git a/experiments/military_signature/p0_freeze.json b/experiments/military_signature/p0_freeze.json new file mode 100644 index 0000000..a3dbfd7 --- /dev/null +++ b/experiments/military_signature/p0_freeze.json @@ -0,0 +1,70 @@ +{ + "protocol_id": "military_signature_p0_v1", + "status": "frozen_draft_not_collection_authorization", + "frozen_at": "2026-10-11T21:45:22.922667+00:00", + "base_commit": "1937c8992da93f33dba7f4a228e4c2812d0535c5", + "working_tree_dirty": true, + "file_sha256": { + "archive:ABVID Lit Review/markdown-ver/Acoustic and seismic signals of heavy military vehicles for co-operative verification/Acoustic and seismic signals of heavy military vehicles for co-operative verification.md": "055a17ffb234e3e36c8b3a31cf0c222134fd23a694814a9760410603d9bf1de9", + "archive:ABVID Lit Review/markdown-ver/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping.md": "4857d9826423a2dd90024bb59f5c12628389471db89edc62b6f5112653954e04", + "archive:ABVID Lit Review/markdown-ver/Engine Speed–Independent Acoustic Signature for Vehicles/Engine Speed–Independent Acoustic Signature for Vehicles.md": "b89b69b6b09a0484bc9d90941d8d9fb18556c7d5de2e4fe28385ba499bf92876", + "archive:ABVID Lit Review/markdown-ver/Improving acoustic vehicle classification by information fusion/Improving acoustic vehicle classification by information fusion.md": "f26d5a00dfb6907be9a4105073f374ef991efacdaab76574a424572dacaae020", + "archive:ABVID Lit Review/pdf-ver/Acoustic and seismic signals of heavy military vehicles for co-operative verification.pdf": "0c60efc61e1f102de2b1ccd96167ddb6ce3a73f98111cc0500112999c3280925", + "archive:ABVID Lit Review/pdf-ver/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping.pdf": "638552d4f7b34ccd17f24b99ac65eb940c278b78c0498996f6dd40e0c5dd4e09", + "archive:ABVID Lit Review/pdf-ver/Engine Speed–Independent Acoustic Signature for Vehicles.pdf": "b5e9aed51751274f9596e5c6244ae536f6db9c06a2919d215b003570fd3f2a23", + "archive:ABVID Lit Review/pdf-ver/Improving acoustic vehicle classification by information fusion.pdf": "f612af2c64d26c21c977982d8245690940bfea62788a79aa903579f909bd2c25", + "artifact:military_signature/p0/literature/HW_AGT1500.md": "a3bf5e242a7a03041f22b3dfc9c31f8fdb72ad4ec659110d3ac6f0d16d72889e", + "artifact:military_signature/p0/literature/HW_AGT1500.pdf": "c3dd615fb1b5dcb31ea8e1e0765f276ef07fb630724b26c1a621b99815cfacd8", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_1754.html": "fd145fdf5ce48657a6f515a1a39b638321d9a0985646e2c7a09b0efd4a774888", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_386317.html": "9ff9f512351f7ba585aefd8f0e4c9737e85d7b73688f9296817eb34e0fd0f68e", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_572786.html": "838dfa8f02d0a66f23291dfe23b68b5314af514d9f5c7de2225cf5258c9befe9", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_776583.html": "600241370b41003c267e662fe47d202c0362238beb23a4e87036c9b2a14c1658", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_855729.html": "5ca528f96c4341a83db81a924c642a1af5b1be0c18199edc2294ad99c48ab45e", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_879491.html": "9d407bb7dc6afb7b4b4259b2f6d1661c9fd781ed67e21daad70786e25907ff35", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_937004.html": "7489eab33b97145305b7e04ecc8a27c33462cecd52314b9d6a680daa8af41aee", + "artifact:military_signature/p0/public-metadata-20261011-r1/DVIDS_FAQ.html": "d2212950d4cf93d54a509184e2d539a1d4ce9e4c82dcf99caa555750fb3865d9", + "artifact:military_signature/p0/validation-20261011-final/manifest.json": "47c884035075644a21489c9bb797300d104349a08e39a15afc9316bd7d01954c", + "artifact:military_signature/p0/validation-20261011-final/native_clips/export.json": "dad54b331afcad5086041692ac150632cc8016a3b751a2be39fd2be0753afeb8", + "artifact:military_signature/p0/validation-20261011-final/reuse_audit.json": "f5f6afd334e7da349cedc65ece1ff7b45d6927dc34d3f7d133c07b17da41ab81", + "artifact:military_signature/p0/validation-20261011-final/validation.json": "6b90c716418fcbf1add0f43e31453367faf08b6735d52c3ae5921a85d4a86d1c", + "artifact:military_signature/p0/validation-20261011-r2/dom-check/annotations.json": "b0523a44e967090b22bb01d4df71c4698158e2731f0263880bab2eea0000ef3b", + "artifact:military_signature/p0/validation-20261011-r2/dom-check/check.cjs": "626b35751d750b113cc310edf021672a9b328964036ebe2123bccef8d3e4b756", + "artifact:military_signature/p0/validation-20261011-r2/dom-check/package-lock.json": "27061a714c7bc84d720463e95a7841b49700a7cb524db156761387501bc1f2fb", + "artifact:military_signature/p0/validation-20261011-r2/dom-check/validation.json": "c1da22da67b45af54b978a9a6d16df29a7531367cf4d2c76f7cb198ad61e2b72", + "artifact:military_signature/p0/verification-20261011.json": "442b2a65829328b2eb3bb6dd2d237b63b0c962ab079194813f06ca6861277dce", + "repo:experiments/military_signature/P0_STATUS.md": "09abd1f0b7ff0812c042fd60a0964d81419e83d0cb9d81e8ff9f34e666c572c9", + "repo:experiments/military_signature/PLAN.md": "877bedf6698d9c46f7172043e3b1b47e4856c4b6f8c704a0337fa3352c8ac824", + "repo:experiments/military_signature/PROPOSAL.md": "61434853d9527c6cf926e72ef4b484300159cd8a6a021f73d707b5e4af2510b9", + "repo:experiments/military_signature/PROTOCOL_DRAFT.md": "6509d9a6cb3290cc35937c46b1d29e90ac426dc449ce6ce926ebff0089a4763e", + "repo:experiments/military_signature/candidate_metadata.json": "8ba912042971391f4be0c8f160cb396467f94cddd56a3498d113128b4323a322", + "repo:experiments/military_signature/manifest_schema.json": "b7f52fc1fdad56633800ea4aebba58ef5d3fc052a8dc92e4683e96adf8270c21", + "repo:experiments/military_signature/p0_protocol.json": "30096b2ee479219f2d174b5abebfbb2603e98bcb12cad113b47e92cfc225a0e5", + "repo:experiments/military_signature/source_evidence.json": "5323f2c582700d2183165aa4c5283bfd3ad2caf2547adbe9c8effa33e325a943", + "repo:experiments/military_signature/verify_freeze.py": "d888ac74293a402f4bdc13fa766237106b2195a2e3245dc0dd42bc6527ed5f57", + "repo:pyproject.toml": "7cdd50d42d9ed0e92eefbca0b6ce4a4bbb59bfb6c1d929a1631e9dcaa8e9e7a7", + "repo:reports/military_signature_P0.md": "e2d6026eaeee7b93b90aef4db8aad0a9098c339ebbc3a435b2b355b8e395f16c", + "repo:src/abvid/data/grouping.py": "f8c123c68dc24ad497ff02300cc477ea2b0c32e35c46ba338c1a6aebe27cdd50", + "repo:src/abvid/data/overlap.py": "2a307c21f644419a21a7dd534afab4d63e0b5ccd6bd5d3197f6f9a79d4a61450", + "repo:src/abvid/paths.py": "b6f985036a59c29ab60033a1e7c8258155f52729639abc10a2c690f0ff7ec933", + "repo:src/abvid/provenance/hashing.py": "b15cf25fe7ed0789af6e140b1202fdd3054a752d70710ce5aecf0b5a83594772", + "repo:src/abvid/signature/__init__.py": "7b2b382581439ba8fdea1ad847f83c8e02dd6f369e51e1667b0378a3959ba756", + "repo:src/abvid/signature/__main__.py": "b06c9f45625ed549658e97a2eb1598e8f03749fd6fd99c59093cdb6e48142c1a", + "repo:src/abvid/signature/audio.py": "a425d39db0bcf60bc5bb80f448fd1d1d917db2f4ef60a9da296ddbf58cb67f91", + "repo:src/abvid/signature/config.py": "c01e1aa6264fc09c95e4ff3d02004dd8e8457611032fc513248556b02b003292", + "repo:src/abvid/signature/dedup.py": "926354247ca253ca8afb55bd850ef07ded9786fe63a08943d7789cb7fd249305", + "repo:src/abvid/signature/fixtures.py": "e81c3cdc3ea6292525531e4e3ad1aefe43c93dda25b60ee68062a898f6296283", + "repo:src/abvid/signature/freeze.py": "4a81484bf0f268818c45dd3b21549c3a362a2fea3413bd0f8b31ce8cc5721283", + "repo:src/abvid/signature/manifest.py": "de29235971f01d249754607ceb5b5ac850837deb5fdcbff138b56b2010d330ae", + "repo:src/abvid/signature/trim.py": "3fdb7cd51d1b44bab10cd62aed4f983167816ca23097aa08ffed670f9628e085", + "repo:src/abvid/signature/validation.py": "0321c7a9d75a9607f66d08da50be6b75c0334993e8bc41f936beaa2569cd2b6f", + "repo:tests/test_overlap_audit.py": "b04f1cfd448890883323f9ddd78dae02a6c7482b3f0db5b2ade23df89cbd5416", + "repo:tests/test_signature_p0.py": "250e603a40296933df138ff88c58486dfc20d5e6d84abf93f25ff3f1933bc5e4", + "repo:uv.lock": "457a95df2ad356c9e9dd862ba233c6dc9aff75f8ec52e03d9067705b3fed4dc5" + }, + "limitations": [ + "Native browser playback/layout requires human check", + "Global canary count fails; human-only baseline handoff recorded", + "Real collection/processing and all fitting remain unauthorized", + "Unknown scientific facts remain unknown" + ] +} diff --git a/experiments/military_signature/p0_protocol.json b/experiments/military_signature/p0_protocol.json new file mode 100644 index 0000000..373e024 --- /dev/null +++ b/experiments/military_signature/p0_protocol.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "id": "military_signature_p0_v1", + "phase": "P0", + "authorization": { + "literature_and_public_metadata": true, + "synthetic_fixture_processing": true, + "real_media_collection": false, + "real_media_processing": false, + "model_fitting": false, + "protected_source_evaluation": false + }, + "storage": { + "fixture_root": "data:military_signature/p0_fixtures", + "artifact_root": "artifact:military_signature/p0", + "manifest_schema": "repo:experiments/military_signature/manifest_schema.json" + }, + "trimming": { + "minimum_foreground_seconds": 2.0, + "minimum_background_seconds": 2.0, + "maximum_segment_seconds": 60.0, + "maximum_asset_seconds": 1200.0, + "display_sample_rate_hz": 8000, + "caption_hidden": true, + "rounding": "floor_start_ceil_end_at_native_rate", + "forbidden_foreground_contaminants": [ + "music", + "narration", + "gunfire", + "aircraft" + ] + }, + "fingerprint": { + "algorithm": "abvid.data.overlap.landmarks_plus_waveform_correlation", + "window_seconds": 4.0, + "hop_seconds": 2.0, + "maximum_windows_per_asset": 600, + "settings": { + "rate": 8000, + "fft": 512, + "hop": 128, + "neighbourhood": [ + 15, + 15 + ], + "frame_threshold_db": 10, + "peaks_per_frame": 2, + "fan_out": 5, + "max_dt": 63, + "max_df": 63, + "min_votes": 12, + "min_vote_fraction": 0.1, + "min_correlation": 0.9, + "lag_search_samples": 256 + }, + "short_asset_policy": "exact_hash_only_unresolved_fingerprint", + "no_match_means": "no_detected_reuse_not_proven_independence" + }, + "evaluation_design": { + "pilot_models_target": 6, + "pilot_sessions_per_model_target": 8, + "model_claim_minimum_verified_groups_per_model": 10, + "seeds": [ + 17, + 29, + 43, + 71, + 101 + ], + "permutations": 1999, + "group_bootstrap_replicates": 2000, + "confidence_level": 0.95, + "hierarchy_gate": "L1_and_L2_group_macro_mAP_minus_session_null_lower_CI_above_zero", + "minimum_session_probe_excess": 0.1, + "vehicle_labels_in_representation_learning_or_hyperparameter_selection": false, + "phase_gates_use_exposed_development_labels": true, + "single_outlet_claim": "outlet_invariance_unknown", + "background_interpretation": "noise_reference_not_identified_channel_inverse" + } +} diff --git a/experiments/military_signature/p1_admission_5gb.json b/experiments/military_signature/p1_admission_5gb.json new file mode 100644 index 0000000..6575ef6 --- /dev/null +++ b/experiments/military_signature/p1_admission_5gb.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "id": "military_signature_p1_admission_5gb_v1", + "phase": "P1_media_admission", + "authorization": { + "real_media_collection": true, + "real_media_processing": true, + "model_fitting": false, + "representation_extraction": false, + "protected_source_evaluation": false, + "media_publication": false + }, + "authorization_basis": { + "user_request": "I allocate to you 20GB of storange, but keep the time per assest. Get more data", + "latest_override": "lets start with 5 GB", + "interpretation": "5 GB replaces 20 GB; retain the 20-minute per-asset cap and expand the candidate population. The old ten-asset and 45-minute-total draft bounds are replaced by this collection contract." + }, + "storage": { + "data_root": "data:military_signature/p1_admission/20261011_5gb", + "artifact_root": "artifact:military_signature/p1_admission/20261011_5gb", + "maximum_total_bytes": 5000000000, + "minimum_free_disk_bytes": 2000000000, + "reserved_workspace_bytes": 100000000 + }, + "collection": { + "maximum_asset_seconds": 1200, + "target_collected_assets": 64, + "maximum_candidate_assets": 160, + "caption_classes": [ + "abrams", + "leopard", + "bradley", + "stryker", + "m109", + "hmmwv" + ], + "rendition_policy": "explicit page MP4 first; otherwise highest-bandwidth HLS at at most 720p; copy streams without encoding", + "request_timeout_seconds": 20, + "maximum_metadata_bytes": 2000000, + "maximum_encoded_asset_bytes": 500000000, + "never_split_long_assets": true + }, + "scientific_scope": { + "recording_session": "unknown unless supported by evidence", + "independence_from_asset_id": false, + "vehicle_labels": "caption only; not confirmed interval identity", + "human_blind_review_required": true, + "experimental_roles": "unassigned", + "verified_group_quota_not_inferred_from_asset_count": true + }, + "protected_catalogs": [ + "archive:configs/audio_sources.yaml", + "archive:benchmarks/v0.1/native_real_7t5w/audio_sources.yaml", + "archive:benchmarks/v0.1/preprocessing_grid_v1/audio_sources.yaml", + "archive:benchmarks/v0.1/beats_comparison_v1/audio_sources.yaml", + "archive:benchmarks/v0.1/beats_comparison_v2/audio_sources.yaml" + ] +} diff --git a/experiments/military_signature/p1_discovery_seeds.json b/experiments/military_signature/p1_discovery_seeds.json new file mode 100644 index 0000000..c8552ff --- /dev/null +++ b/experiments/military_signature/p1_discovery_seeds.json @@ -0,0 +1,108 @@ +{ + "created_at": "2026-10-12T02:44:13.656Z", + "mode": "primary_DVIDS_search_leads_only", + "searches": [ + { + "label": "abrams", + "query": "site:dvidshub.net/video/ \"Abrams\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/1754/abrams-tank-b-roll", + "https://www.dvidshub.net/video/524546/abrams-tank-maintainers-keeping-us-soldiers-2017-nordic-tank-challenge-b-roll", + "https://www.dvidshub.net/video/841851/abrams-operations-summit-tank-b-roll", + "https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package", + "https://www.dvidshub.net/video/1001299/enhancing-fighting-capabilities-us-army-m1a2-abrams-conducts-live-fire-training-with-nato-allies-b-roll", + "https://www.dvidshub.net/video/996227/b-roll-3-8-cav-abrams-platoon-live-fire-operation-warhorse-surge", + "https://www.dvidshub.net/video/841356/us-and-polish-partners-attend-abrams-operations-summit-b-roll", + "https://www.dvidshub.net/video/987082/iron-heroes-m1a2-abrams-tank-live-fire-accuracy-screening-test-b-roll", + "https://www.dvidshub.net/video/410989/m1a2-abrams-tank-offload-bulgaria-b-roll", + "https://www.dvidshub.net/video/742736/b-roll-gen-robert-abrams-visits-usag-daegu-evaluates-us-response-covid-19", + "https://www.dvidshub.net/video/882364/m1a2-sepv3-abrams-main-battle-tank-loader-pov-b-roll", + "https://www.dvidshub.net/video/984276/1st-infantry-division-conducts-m1a2-abrams-table-iv-v-gunnery-bulgaria-b-roll" + ] + }, + { + "label": "leopard", + "query": "site:dvidshub.net/video/ \"Leopard\" \"tank\"", + "urls": [ + "https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll", + "https://www.dvidshub.net/video/1011696/germanys-tank-brigade-participates-its-first-exercise-lithuania", + "https://www.dvidshub.net/video/932199/ukrainian-tankers-train-poland-with-nato-allies-mastersubs", + "https://www.dvidshub.net/video/946216/ukrainian-troops-train-poland-master-allied-leopard-2-tanks", + "https://www.dvidshub.net/video/686162/norwegian-tanks-arrive-poland-exercise-noble-jump-2019-one-result", + "https://www.dvidshub.net/video/831895/norwegian-tanks-launch-live-fire-drills-lithuania", + "https://www.dvidshub.net/video/688131/german-leopard-tank-target-practice-strike-back-19", + "https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank", + "https://www.dvidshub.net/video/882815/exercise-noble-jump-23-norwegian-dutch-german-soldiers-lift-leopard-tank", + "https://www.dvidshub.net/video/952520/usareur-af-international-tank-challenge", + "https://www.dvidshub.net/video/916378/nato-allies-demonstrate-strengthened-deterrence-and-defence-during-polish-led-dragon-2024-exercise", + "https://www.dvidshub.net/video/819508/biggest-nato-exercise-lithuania-marks-10th-rotation-efp-battlegroup" + ] + }, + { + "label": "bradley", + "query": "site:dvidshub.net/video/ \"Bradley\" \"convoy\"", + "urls": [ + "https://www.dvidshub.net/video/766451/b-roll-bradley-convoy-northeastern-syria", + "https://www.dvidshub.net/video/590296/nc-guard-abrams-tanks-and-bradley-fighting-vehicles-95", + "https://www.dvidshub.net/video/1705/raider-base-b-roll", + "https://www.dvidshub.net/video/795825/alwyn-c-cashe-garden-dedicated", + "https://www.dvidshub.net/video/286235/exercise-raider-ready", + "https://www.dvidshub.net/video/419720/116th-cavalry-brigade-combat-team-begins-training-national-training-center" + ] + }, + { + "label": "stryker", + "query": "site:dvidshub.net/video/ \"Stryker\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/1010471/1-23-infantry-30mm-stryker-b-roll", + "https://www.dvidshub.net/video/972885/stryker-survey-and-black-hawk-aircraft-decon-b-roll", + "https://www.dvidshub.net/video/1024929/2d-cavalry-regiment-stryker-familiarization-gunnery-broll", + "https://www.dvidshub.net/video/481459/usa-murphy-stryker-b-roll", + "https://www.dvidshub.net/video/666409/stryker-redeployment-b-roll", + "https://www.dvidshub.net/video/943782/stryker-jrtc-b-roll-package", + "https://www.dvidshub.net/video/483942/orient-shield-2016-stryker-demo-b-roll", + "https://www.dvidshub.net/video/589501/stryker-mission-ops-convoy-b-roll", + "https://www.dvidshub.net/video/861223/broll-8-37th-stryker-download", + "https://www.dvidshub.net/video/358437/b-roll-stryker-operations", + "https://www.dvidshub.net/video/28750/stryker", + "https://www.dvidshub.net/video/873196/2-2-stryker-gunnery-broll" + ] + }, + { + "label": "m109", + "query": "site:dvidshub.net/video/ \"M109\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/795698/m109-paladin-howitzer-b-roll", + "https://www.dvidshub.net/video/846427/m109-howitzer-degraded-training", + "https://www.dvidshub.net/video/485602/centcom-commander-fires-m-109-paladin", + "https://www.dvidshub.net/video/974185/us-and-british-artillery-units-fire-m109a6-paladins-during-northern-strike-25-2", + "https://www.dvidshub.net/video/801177/xctc-21-05-camp-guernsey-m109-howitzer-lanes-130th-field-artillery-brigade", + "https://www.dvidshub.net/video/884357/moving-equipment-african-lion", + "https://www.dvidshub.net/video/516696/3rd-battalion-29th-field-artillery-regiment-bring-equipment-zagan-poland-b-roll", + "https://www.dvidshub.net/video/932935/m109-a6-paladins-zeroing-exercise-operation-fortitude-2024", + "https://www.dvidshub.net/video/5794/1-41-field-artillery", + "https://www.dvidshub.net/video/893652/2-222-artillery-b-roll", + "https://www.dvidshub.net/video/800891/african-lion-2021-royal-moroccan-armed-forces-paladin-live-fire", + "https://www.dvidshub.net/video/847461/soldiers-us-and-norway-train-armed-forces-ukraine" + ] + }, + { + "label": "hmmwv", + "query": "site:dvidshub.net/video/ \"Humvee\" \"driving\"", + "urls": [ + "https://www.dvidshub.net/video/35282/humvee-capabilities", + "https://www.dvidshub.net/video/64777/driven-drive", + "https://www.dvidshub.net/video/842870/156-crg-southern-strike-2022-humvee-driving-training", + "https://www.dvidshub.net/video/64304/roadside-assistance", + "https://www.dvidshub.net/video/11647/4th-id-ia-shooting-and-drivers-training-b-roll", + "https://www.dvidshub.net/video/921247/b-roll-african-lion-24-conducts-port-operations-gabes-tunisia", + "https://www.dvidshub.net/video/973361/spectrum-situational-awareness-system-s2as-user-assessment-b-roll", + "https://www.dvidshub.net/video/638757/trident-juncture-2018-united-states-marine-corps-near-oppdal", + "https://www.dvidshub.net/video/914522/b-roll-justified-accord-2024-participants-conduct-quick-response-force-drills-and-react-contact", + "https://www.dvidshub.net/video/12865/gun-turret-b-roll", + "https://www.dvidshub.net/video/84442/light-weight-modular-causeway-system-part-1", + "https://www.dvidshub.net/video/2235/sfc-smith-battlesite" + ] + } + ] +} diff --git a/experiments/military_signature/p1_extra_discovery.json b/experiments/military_signature/p1_extra_discovery.json new file mode 100644 index 0000000..0587a33 --- /dev/null +++ b/experiments/military_signature/p1_extra_discovery.json @@ -0,0 +1,77 @@ +{ + "created_at": "2026-10-12T02:52:19.429Z", + "searches": [ + { + "label": "leopard_extra", + "query": "site:dvidshub.net/video/ leopard tanks training US soldiers", + "urls": [ + "https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank", + "https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll", + "https://www.dvidshub.net/video/688131/german-leopard-tank-target-practice-strike-back-19", + "https://www.dvidshub.net/video/634874/trident-juncture-2018-panzer-tank-training", + "https://www.dvidshub.net/video/882815/exercise-noble-jump-23-norwegian-dutch-german-soldiers-lift-leopard-tank", + "https://www.dvidshub.net/video/932199/ukrainian-tankers-train-poland-with-nato-allies-mastersubs", + "https://www.dvidshub.net/video/946216/ukrainian-troops-train-poland-master-allied-leopard-2-tanks", + "https://www.dvidshub.net/video/1011696/germanys-tank-brigade-participates-its-first-exercise-lithuania", + "https://www.dvidshub.net/video/883088/sweden-trains-with-nato-allies-exercise-aurora-2023", + "https://www.dvidshub.net/video/605359/strong-europe-tank-challenge-offensive-operations-with-french-and-austrian-soldiers", + "https://www.dvidshub.net/video/635676/trident-juncture-2018-prime-minister-visits-norwegian-troops", + "https://www.dvidshub.net/video/941336/tanks-action" + ] + }, + { + "label": "hmmwv_extra", + "query": "site:dvidshub.net/video/ \"HMMWV\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/972570/caribbean-thunder-2025-hmmwv-engine-swap-b-roll", + "https://www.dvidshub.net/video/27421/hmmwv-egress-assistance-trainer", + "https://www.dvidshub.net/video/762326/hmmwv-egress-assistance-trainer-heat-b-roll", + "https://www.dvidshub.net/video/294093/hmmwv-egress-assistance-trainer-roll", + "https://www.dvidshub.net/video/868525/tow-missiles-fired-hmmwv-platform", + "https://www.dvidshub.net/video/972570/caribbean-thunder-2025-maintenance-team-swap-hmmwv", + "https://www.dvidshub.net/video/451617/hmmwv-load-and-unload", + "https://www.dvidshub.net/video/421282/swift-response-2015-hmmwv-egress-assistance-trainer", + "https://www.dvidshub.net/video/294956/jmtc-hmmwv-egress-assistance-trainer", + "https://www.dvidshub.net/video/27424/hmmwv-egress-assistance-trainer-spanish", + "https://www.dvidshub.net/video/951135/us-army-soldiers-action-combined-resolve-25-1", + "https://www.dvidshub.net/video/994737/task-force-brawlers-conduct-vehicle-terrain-familiarization-training-b-roll" + ] + }, + { + "label": "m109_extra", + "query": "site:dvidshub.net/video/ \"M109A6\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/934132/northern-strike-24-2-1-201-fa-m109a6-paladin-training-social-media-b-roll", + "https://www.dvidshub.net/video/863472/b-roll-idaho-national-guard-m109a6-howitzer-exercise-african-lion-22", + "https://www.dvidshub.net/video/863491/b-roll-idaho-national-guard-m109a6-howitzer-exercise-african-lion-22", + "https://www.dvidshub.net/video/974185/us-and-british-artillery-units-fire-m109a6-paladins-during-northern-strike-25-2", + "https://www.dvidshub.net/video/607339/arkansas-field-artillery-unit-fires-m109a6-paladins-during-operation-western-strike", + "https://www.dvidshub.net/video/967175/army-250-parade-celebration-b-roll", + "https://www.dvidshub.net/video/1006440/ivy-mass-observation-point", + "https://www.dvidshub.net/video/912344/b-roll-task-force-marne-paladin-crews-conduct-live-fire-exercise-poland", + "https://www.dvidshub.net/video/110284/combined-arms-live-fire-exercise-builds-camaraderie-among-soldiers", + "https://www.dvidshub.net/video/137865/howitzers-kuwait", + "https://www.dvidshub.net/video/898609/b-roll-task-force-marne-soldiers-conduct-artillery-live-fire-poland", + "https://www.dvidshub.net/video/966208/us-army-vehicles-staged-west-potomac-park-waterfront-b-roll" + ] + }, + { + "label": "bradley_extra", + "query": "site:dvidshub.net/video/ \"Bradley\" \"maneuver\" \"B-Roll\"", + "urls": [ + "https://www.dvidshub.net/video/897660/2-15-cav-bradley-fighting-vehicle-maneuver-training-b-roll", + "https://www.dvidshub.net/video/897659/2-15-cav-bradley-maneuver-training-vertical-b-roll", + "https://www.dvidshub.net/video/993639/3-8-cavalry-conducts-platoon-level-stx-bradley-aerial-footage", + "https://www.dvidshub.net/video/993637/us-soldiers-train-with-m2a3-bradleys-during-platoon-stx-poland", + "https://www.dvidshub.net/video/524204/iron-focus-2017-broll", + "https://www.dvidshub.net/video/843074/b-roll-spartan-bradley-crew-honored-best-army", + "https://www.dvidshub.net/video/357693/m2-bradley-fighting-vehicles-tango-with-opposing-forces-ofpor-annual-training", + "https://www.dvidshub.net/video/374080/estonian-and-us-armies-show-and-tell-fathers-day", + "https://www.dvidshub.net/video/374089/cavalry-scouts-show-off-their-gear-estonian-army-conscripts", + "https://www.dvidshub.net/video/822687/gulf-gunnery-armor-platoon-live-fire", + "https://www.dvidshub.net/video/373655/estonian-prime-minister-receives-cav-welcome-tapa", + "https://www.dvidshub.net/video/478354/tank-live-fire-romania" + ] + } + ] +} diff --git a/experiments/military_signature/source_evidence.json b/experiments/military_signature/source_evidence.json new file mode 100644 index 0000000..d3c9296 --- /dev/null +++ b/experiments/military_signature/source_evidence.json @@ -0,0 +1,252 @@ +{ + "protocol_id": "military_signature_p0_v1", + "created_at": "2026-10-11T21:39:18.814792+00:00", + "source_rows": [ + { + "source_id": "P04", + "title": "Acoustic and seismic signals of heavy military vehicles for co-operative verification", + "pdf_ref": "archive:ABVID Lit Review/pdf-ver/Acoustic and seismic signals of heavy military vehicles for co-operative verification.pdf", + "markdown_ref": "archive:ABVID Lit Review/markdown-ver/Acoustic and seismic signals of heavy military vehicles for co-operative verification/Acoustic and seismic signals of heavy military vehicles for co-operative verification.md", + "pdf_sha256": "0c60efc61e1f102de2b1ccd96167ddb6ce3a73f98111cc0500112999c3280925", + "workflow": "Markdown read first, then PDF text and visual page verification", + "claims": [ + { + "pdf_page": 16, + "printed_page": "728", + "section": "5.1 Single spectra", + "author_statement": "Engine periodicity contributes spectral harmonics; tracked motion adds track/sprocket periodicity. Track contributions need not dominate airborne sound.", + "interpretation": "Mechanical plausibility for coarse cues does not establish model recognition." + }, + { + "pdf_page": 21, + "printed_page": "733", + "section": "5.3 Relative harmonic powers; Table 5", + "author_statement": "Table 5 lists Leopard 1 and Leopard 2 as four-stroke V10 and V12 diesels. Dominant spectral lines are not necessarily firing-frequency peaks.", + "interpretation": "Do not infer cylinders, RPM or firing order from the largest peak alone." + } + ], + "markdown_sha256": "055a17ffb234e3e36c8b3a31cf0c222134fd23a694814a9760410603d9bf1de9" + }, + { + "source_id": "P05", + "title": "Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping", + "pdf_ref": "archive:ABVID Lit Review/pdf-ver/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping.pdf", + "markdown_ref": "archive:ABVID Lit Review/markdown-ver/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping/Acoustic–seismic detection and classification of military vehicles—developing tools for disarmament and peace-keeping.md", + "pdf_sha256": "638552d4f7b34ccd17f24b99ac65eb940c278b78c0498996f6dd40e0c5dd4e09", + "workflow": "Markdown read first, then PDF text and visual page verification", + "claims": [ + { + "pdf_page": 20, + "printed_page": "1104", + "section": "Table 3 and its split footnote", + "author_statement": "Within each pass, two thirds trained and one third tested; acoustic-only full-pass recognition was 89%.", + "interpretation": "This split does not verify transfer to new independent sessions or physical vehicles." + }, + { + "pdf_page": 21, + "printed_page": "1105", + "section": "6 Conclusions and outlook", + "author_statement": "Relative harmonic powers vary with rotation rate and observation angle.", + "interpretation": "Order normalization alone need not remove state or recording geometry." + }, + { + "pdf_page": 22, + "printed_page": "1106", + "section": "6 Conclusions and outlook, continuation", + "author_statement": "The method chiefly recognizes engine/exhaust types; different vehicle types sharing engines can be grouped together.", + "interpretation": "A strong engine cue is not automatically a model identity cue." + } + ], + "markdown_sha256": "4857d9826423a2dd90024bb59f5c12628389471db89edc62b6f5112653954e04" + }, + { + "source_id": "P11", + "title": "Engine Speed–Independent Acoustic Signature for Vehicles", + "pdf_ref": "archive:ABVID Lit Review/pdf-ver/Engine Speed–Independent Acoustic Signature for Vehicles.pdf", + "markdown_ref": "archive:ABVID Lit Review/markdown-ver/Engine Speed–Independent Acoustic Signature for Vehicles/Engine Speed–Independent Acoustic Signature for Vehicles.md", + "pdf_sha256": "b5e9aed51751274f9596e5c6244ae536f6db9c06a2919d215b003570fd3f2a23", + "workflow": "Markdown read first, then PDF text and visual page verification", + "claims": [ + { + "pdf_page": 2, + "printed_page": "95", + "section": "II Material", + "author_statement": "Four cars were recorded near their engines during repeated 800–3000 rpm sweeps.", + "interpretation": "This is limited physical-vehicle and recording-condition support." + }, + { + "pdf_page": 3, + "printed_page": "96", + "section": "II Material, continuation", + "author_statement": "The first increase/decrease episode trained and the second tested.", + "interpretation": "Speed-sweep holdout is not an independent-session/model-population test." + } + ], + "markdown_sha256": "b89b69b6b09a0484bc9d90941d8d9fb18556c7d5de2e4fe28385ba499bf92876" + }, + { + "source_id": "P12", + "title": "Improving acoustic vehicle classification by information fusion", + "pdf_ref": "archive:ABVID Lit Review/pdf-ver/Improving acoustic vehicle classification by information fusion.pdf", + "markdown_ref": "archive:ABVID Lit Review/markdown-ver/Improving acoustic vehicle classification by information fusion/Improving acoustic vehicle classification by information fusion.md", + "pdf_sha256": "f612af2c64d26c21c977982d8245690940bfea62788a79aa903579f909bd2c25", + "workflow": "Markdown read first, then PDF text and visual page verification", + "claims": [ + { + "pdf_page": 9, + "printed_page": "37", + "section": "6 Experimental results", + "author_statement": "The US ARL dataset uses five anonymized vehicle classes, two tracked and three wheeled.", + "interpretation": "Named model and variant identities remain unknown." + }, + { + "pdf_page": 10, + "printed_page": "38", + "section": "6 Experimental results; Table 1", + "author_statement": "Each class has six runs; three runs train and three test. Harmonic features vary within classes and overlap across classes.", + "interpretation": "Run holdout offers more separation than within-pass splitting; session and individual identities remain unknown." + }, + { + "pdf_page": 11, + "printed_page": "39", + "section": "6 Experimental results; Fig. 3 and continuation", + "author_statement": "The other three runs test; SVM parameter selection uses training data only.", + "interpretation": "Adopt training-only preprocessing/selection; do not infer independent recording sessions from runs." + } + ], + "markdown_sha256": "f26d5a00dfb6907be9a4105073f374ef991efacdaab76574a424572dacaae020" + }, + { + "source_id": "HW_M1_AGT1500", + "title": "AGT1500 Turbine Engine brochure", + "publisher": "Honeywell", + "publication_year": "unknown", + "url": "https://aerospace.honeywell.com/content/dam/aerobt/en/documents/landing-pages/brochures/AGT1500-Turbine-Engine-Brochure.pdf", + "pdf_ref": "artifact:military_signature/p0/literature/HW_AGT1500.pdf", + "markdown_ref": "artifact:military_signature/p0/literature/HW_AGT1500.md", + "pdf_sha256": "c3dd615fb1b5dcb31ea8e1e0765f276ef07fb630724b26c1a621b99815cfacd8", + "workflow": "Locally extracted Markdown read first; PDF page visually verified", + "claims": [ + { + "pdf_page": 1, + "printed_page": "unknown", + "section": "Product description", + "author_statement": "The AGT1500 gas turbine powers the M1 Abrams.", + "interpretation": "Supports the coarse M1 turbine association, not every candidate variant or an acoustic classifier." + } + ] + } + ], + "powerplant_table": [ + { + "vehicle": "M1 Abrams", + "powerplant": "turbine", + "engine": "AGT1500", + "evidence": "HW_M1_AGT1500 PDF p1 product description", + "candidate_variant_specific_verification": "unknown" + }, + { + "vehicle": "Leopard 1, paper example", + "powerplant": "diesel", + "engine": "V10 four-stroke; exact model unknown", + "evidence": "P04 PDF p21 printed p733 Table 5" + }, + { + "vehicle": "Leopard 2, paper example", + "powerplant": "diesel", + "engine": "V12 four-stroke; exact model unknown", + "evidence": "P04 PDF p21 printed p733 Table 5", + "candidate_2A4_variant_specific_verification": "unknown" + }, + { + "vehicle": "T-72 variants", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "T-80 variants including BVM", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "T-80UD", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "T-90 variants", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "Bradley candidate", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "Stryker candidate", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "M109", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + }, + { + "vehicle": "HMMWV/JLTV", + "powerplant": "unknown", + "engine": "unknown", + "evidence": "Proposal memory-derived table not yet PDF-authoritatively verified" + } + ], + "unavailable_primary_sources": [ + { + "source_id": "ROSTEC_T80BVM", + "url": "https://rostec.ru/media/news/t-80bvm-letayushchiy-tank-snova-v-stroyu/", + "retrieved_at": "2026-10-11T21:33:56.605674+00:00", + "mode": "HTML_only_no_associated_media", + "status": "failed", + "error": "URLError: ", + "sha256": "unknown", + "snapshot_ref": "unknown" + }, + { + "source_id": "ROSTEC_DIESEL", + "url": "https://rostec.ru/media/news/tanki-vybirayut-dizel/", + "retrieved_at": "2026-10-11T21:33:56.639551+00:00", + "mode": "HTML_only_no_associated_media", + "status": "failed", + "error": "URLError: ", + "sha256": "unknown", + "snapshot_ref": "unknown" + }, + { + "source_id": "ODIN_T80UD", + "url": "https://odin.t2com.army.mil/WEG/Asset/T-80UD_Ukrainian_Main_Battle_Tank_(MBT)", + "status": "web tool returned empty page; searched excerpt not treated as verified evidence" + }, + { + "source_id": "ARMOR_1998", + "url": "https://www.moore.army.mil/armor/EArmor/content/issues/1998/SEP_OCT/ArmorSeptemberOctober1998web.pdf", + "status": "web tool inaccessible" + } + ], + "analyst_identifiability_argument": { + "statement": "Source-free x=Hs+n background samples constrain n, not H or s separately.", + "status": "algebraic reasoning, not an empirical or paper-author claim" + }, + "unknowns": [ + "No directly reviewed study validates this proposed cross-session military model task", + "No verified candidate-session independence or usable audio", + "No established L3 engine-order ground truth for these candidates", + "No published justification for the ten-group sufficiency threshold" + ] +} diff --git a/experiments/military_signature/source_evidence_supplement.json b/experiments/military_signature/source_evidence_supplement.json new file mode 100644 index 0000000..d179ef9 --- /dev/null +++ b/experiments/military_signature/source_evidence_supplement.json @@ -0,0 +1,259 @@ +{ + "supplement_id": "military_signature_p0_source_followup_20261011", + "protocol_id": "military_signature_p0_v1", + "scope": "P0 public literature only; no real media, features or fitting", + "relationship_to_frozen_ledger": "Additive dated evidence. The original source_evidence.json and 55-file freeze remain unchanged.", + "sources": [ + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "CUMMINS_DEFENSE", + "title": "Power Solutions for Defense", + "publisher": "Cummins", + "publication_date": "unknown", + "url": "https://mart.cummins.com/imagelibrary/data/assetfiles/0075965.pdf", + "pdf_sha256": "7f1c7aadb100c27f0c9f58223fb2ecfb5a0508dc9a23c4079425d943a0a0cc53", + "markdown_sha256": "7721ae97dc290b319f1720a662bb7954152d966d247b4f2e145999ada6441400", + "claims": [ + { + "pdf_page": 13, + "printed_page": "11", + "section": "Distinguished service at every milestone; 1981 timeline entry", + "author_statement": "The brochure associates the start of Bradley production in 1981 with VTA903-500 engines.", + "interpretation": "Historical association only. It does not identify the engine in a 2020 Bradley metadata candidate.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_DEFENSE_p13.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_DEFENSE.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_DEFENSE.md" + }, + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "CUMMINS_V903", + "title": "V903", + "publisher": "Cummins", + "publication_date": "2024-09 revision", + "url": "https://mart.cummins.com/imagelibrary/data/assetfiles/0063416.pdf", + "pdf_sha256": "c814c4f85a3b905d8731787accc5bbf30e912513b0fa91a3d9aa9d2e987f8f5d", + "markdown_sha256": "4dc76686d4ff0ca5be79dc9ab49afc13a072e0f55a73919d75ddfe3461587e30", + "claims": [ + { + "pdf_page": 2, + "printed_page": "unknown", + "section": "General specifications; Engine Design; Bulletin 5600645 Rev. 9/24", + "author_statement": "The V903 specification describes a 90-degree V8, four-stroke diesel architecture.", + "interpretation": "Supports the described V903 family architecture. Linking it to the historical Bradley association is a family-level interpretation, not proof of any recorded Bradley configuration.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_V903_p2.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_V903.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/CUMMINS_V903.md" + }, + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "ARMY_M109_1992", + "title": "Army Research, Development & Acquisition Bulletin, September–October 1992", + "publisher": "U.S. Army", + "publication_date": "1992-09/10", + "url": "https://asc.army.mil/docs/pubs/alt/archives/1992/Sep-Oct_1992.PDF", + "pdf_sha256": "ce1eb1aca7695b827effd2f1a8e6299ef1eb8d3afe198b53979fa68f23c97ed7", + "markdown_sha256": "b9eb13d586a43a299f3658fcbca64b277fa35d6b291ec736e112a2d318972b07", + "claims": [ + { + "pdf_page": 57, + "printed_page": "55", + "section": "RD&A News Briefs: Howitzer and FAASV Get Improved Engine", + "author_statement": "The brief describes a modified eight-cylinder Detroit Diesel 8V71T for M109A6 and M992A1, designated 8V71T-LHR.", + "interpretation": "Supports the reported M109A6 diesel association. Stroke count is not established by this page, and the association does not cover every M109 variant.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_M109_1992_p57.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_M109_1992.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_M109_1992.md" + }, + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "ARMY_STRYKER_2020", + "title": "Tracks, January 23, 2020", + "publisher": "Anniston Army Depot, U.S. Army", + "publication_date": "2020-01-23", + "url": "https://anad.army.mil/docs/librariesprovider2/default-document-library/tr012320_v2.pdf?sfvrsn=7a536430_0", + "pdf_sha256": "2c5f00e93f57271921caa6df5c49cebe2437f7336183f2f7bc17ef5c14ac6d94", + "markdown_sha256": "60aa67bdcf866a8710ae84612f12ee4a4ea3207c9516036ae241a7ac2e147594", + "claims": [ + { + "pdf_page": 1, + "printed_page": "1", + "section": "First Stryker FMS program begins at ANAD; Jennifer Bacchus, ANAD PAO", + "author_statement": "Thirty Strykers were being reset as Infantry Carrier Vehicles for the Royal Thai Army.", + "interpretation": "Defines the reported reset population, not a universal Stryker specification.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_STRYKER_2020_p1.png" + }, + { + "pdf_page": 4, + "printed_page": "4", + "section": "From STRYKER, page 1", + "author_statement": "Vehicles in the reset program could arrive with Caterpillar C7 or 3126 engines and leave with serviced C7 engines.", + "interpretation": "A family caption alone cannot identify an engine. Recording-specific engine labels remain unknown without further evidence.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_STRYKER_2020_p4.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_STRYKER_2020.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/ARMY_STRYKER_2020.md" + }, + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "CAT_DIESEL_FLUIDS_2025", + "title": "Cat Commercial Diesel Engine Fluids Recommendations, SEBU6251-32", + "publisher": "Caterpillar", + "publication_date": "2025-02", + "url": "https://caterpillar.scene7.com/is/content/Caterpillar/CM20250210-d820f-61dca", + "pdf_sha256": "25bebbaa87b976b4891426952d429f88f6601e45e0dbb3f9874718a18f1a1d7c", + "markdown_sha256": "217345cab40d27c1c55fdb727f7599faa975255f33334ba7a86f7daabf60c1ed", + "claims": [ + { + "pdf_page": 49, + "printed_page": "49", + "section": "Engine Oil Selection; Diesel Engine Oil (DEO) Characteristics; viscosity note", + "author_statement": "An oil-viscosity note explicitly includes 3126 and C7 among diesel engine families.", + "interpretation": "Combined with the Army article, this supports a coarse diesel association for the two reported Stryker engine families. It provides no recording-specific engine identification, cylinder count or stroke count.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/CAT_DIESEL_FLUIDS_2025_p49.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/CAT_DIESEL_FLUIDS_2025.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/CAT_DIESEL_FLUIDS_2025.md" + }, + { + "evidence_status": "selected original PDF pages verified", + "workflow": "Extracted Markdown read first; original PDF text and rendered pages checked", + "candidate_recording_engine": "unknown", + "source_id": "AMGENERAL_M1165", + "title": "M1100 Series HMMWV: M1165A1 Special Ops", + "publisher": "AM General", + "publication_date": "unknown", + "document_identifier": "AMP05072018DOM", + "url": "https://www.amgeneral.com/wp-content/uploads/2019/12/M1165-Special-Ops.pdf", + "pdf_sha256": "21c010d2314fa48f73b51efd37f2c60ede55e27653e0a9e8a9bca71cc2f4c056", + "markdown_sha256": "79dc33716608406e1bfaa4f23d0e7d94373d0f9a7c17723715aa5709ee768358", + "claims": [ + { + "pdf_page": 1, + "printed_page": "unknown", + "section": "M1165A1 Special Ops product description", + "author_statement": "The brochure identifies the subject as M1165A1 Special Ops in the HMMWV family.", + "interpretation": "Do not combine HMMWV and JLTV into one model identity.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/AMGENERAL_M1165_p1.png" + }, + { + "pdf_page": 2, + "printed_page": "unknown", + "section": "M1165A1 Special Ops specifications; Engine", + "author_statement": "The engine entry specifies a GEP 6.5-litre V8 turbocharged diesel.", + "interpretation": "Supports the described M1165A1 configuration. Other HMMWV variants, JLTV configurations and recordings remain unverified; stroke count stays unknown.", + "render_ref": "artifact:military_signature/p0/literature-followup-20261011/AMGENERAL_M1165_p2.png" + } + ], + "pdf_ref": "artifact:military_signature/p0/literature-followup-20261011/AMGENERAL_M1165.pdf", + "markdown_ref": "artifact:military_signature/p0/literature-followup-20261011/AMGENERAL_M1165.md" + } + ], + "powerplant_updates": [ + { + "vehicle_scope": "Bradley production beginning in 1981", + "engine": "VTA903-500", + "powerplant": "diesel at V903 family level", + "configuration": "V903 family V8 four-stroke; historical installation details unknown", + "evidence": [ + "CUMMINS_DEFENSE PDF p13 / printed p11", + "CUMMINS_V903 PDF p2" + ], + "status": "historical association plus family specification; not candidate-specific" + }, + { + "vehicle_scope": "M109A6 in the 1992 Army report", + "engine": "8V71T-LHR", + "powerplant": "diesel", + "cylinders": 8, + "stroke_count": "unknown", + "evidence": [ + "ARMY_M109_1992 PDF p57 / printed p55" + ], + "status": "reported variant association; not all M109 variants" + }, + { + "vehicle_scope": "Strykers entering the documented 2020 Anniston reset program", + "engine_options": [ + "Caterpillar C7", + "Caterpillar 3126" + ], + "powerplant": "diesel", + "cylinders": "unknown", + "stroke_count": "unknown", + "evidence": [ + "ARMY_STRYKER_2020 PDF p1, p4", + "CAT_DIESEL_FLUIDS_2025 PDF p49" + ], + "status": "reported reset population; exact candidate engine unknown" + }, + { + "vehicle_scope": "M1165A1 Special Ops described by AM General", + "engine": "GEP 6.5-litre turbocharged V8", + "powerplant": "diesel", + "cylinders": 8, + "stroke_count": "unknown", + "evidence": [ + "AMGENERAL_M1165 PDF p1–2" + ], + "status": "brochure configuration; not all HMMWV or JLTV variants" + } + ], + "unknowns": [ + "T-72/T-80/T-80UD/T-90 associations remain unverified under the PDF-authoritative contract.", + "JLTV is not established by the M1165A1 brochure.", + "No source here identifies the engine or firing/order label of any proposed recording.", + "No new verified independent recording groups or usable-audio observations were established." + ], + "retrieval_records": [ + "artifact:military_signature/p0/literature-followup-20261011/retrieval.json", + "artifact:military_signature/p0/literature-followup-20261011/retrieval_curl.json", + "artifact:military_signature/p0/literature-followup-20261011/retrieval_additional.json" + ], + "unverified_primary_leads": [ + { + "source_id": "ARMOR_2023_SUMMER", + "url": "https://www.benning.army.mil/armor/eARMOR/content/issues/2023/Summer/ARMOR_Summer_2023_edition.pdf", + "status": "local retrieval timed out; web PDF excerpt did not yield verifiable pages", + "sha256": "unknown", + "verified_claims": [] + }, + { + "source_id": "ARMOR_1998_SEP", + "url": "https://www.benning.army.mil/Armor/eARMOR/content/issues/1998/SEP_OCT/ArmorSeptemberOctober1998web.pdf", + "status": "retrieval timed out; no original-page verification", + "sha256": "unknown", + "verified_claims": [] + }, + { + "source_id": "ARMOR_2001_SEP", + "url": "https://www.benning.army.mil/Armor/eARMOR/content/issues/2001/SEP_OCT/ArmorSeptemberOctober2001web.pdf", + "status": "retrieval timed out; no original-page verification", + "sha256": "unknown", + "verified_claims": [] + } + ], + "method_notes": [ + "Initial Python TLS failures are retained. System curl succeeded with default TLS validation; certificate checks were not disabled.", + "Selected renders and PDF extraction checks are retained in pdf_page_checks.json.", + "Public literature documents with empty-password PDF encryption were opened using the standard reader; no credentials or media were used." + ], + "P1_authorized": false +} diff --git a/experiments/military_signature/verify_freeze.py b/experiments/military_signature/verify_freeze.py new file mode 100644 index 0000000..26a072b --- /dev/null +++ b/experiments/military_signature/verify_freeze.py @@ -0,0 +1,7 @@ +"""Thin read-only verification of the P0 draft freeze.""" +from abvid.signature.freeze import verify + + +if __name__ == "__main__": + import json + print(json.dumps(verify(), indent=2)) diff --git a/reports/military_signature_P0.md b/reports/military_signature_P0.md new file mode 100644 index 0000000..bb99c8e --- /dev/null +++ b/reports/military_signature_P0.md @@ -0,0 +1,166 @@ +# Military acoustic signature P0 research and preparation + +2026-10-11; protocol `military_signature_p0_v1`; base commit +`1937c8992da93f33dba7f4a228e4c2812d0535c5`, with a dirty working tree. + +P0 produced a frozen draft protocol, explicit manifest schema, offline caption-hidden +trimming/import/export tools, conservative provenance/reuse grouping, and artificial +validation evidence. The tooling passed its focused checks. This establishes +preparation mechanics only. No real audio/video was accessed, no representations +were extracted and no classifier or CAST model was fitted. The retired military +benchmark and protected-source roles were not resumed or changed. + +The scientific question remains open: can the proposed descriptors retain vehicle +model information across independent sessions while reducing nuisance information? +Source-model dominance over propagation mismatch remains a hypothesis. Existing +papers motivate components but do not validate this proposed study population. + +## Reviewable deliverables + +- [Frozen protocol draft](../experiments/military_signature/PROTOCOL_DRAFT.md), + [canonical configuration](../experiments/military_signature/p0_protocol.json), + [manifest schema](../experiments/military_signature/manifest_schema.json), and + [freeze ledger](../experiments/military_signature/p0_freeze.json). +- [Reusable implementation](../src/abvid/signature/__main__.py): synthetic fixtures, + strict manifest/config admission, blind packet construction, interval import, + native-frame export, label-blind waveform reuse audit and group/readiness checks. +- [Source evidence](../experiments/military_signature/source_evidence.json): paper + IDs, PDF byte hashes/page/section locators, original Markdown references, author + statements, interpretations, unknowns and failed source access. +- [Public candidate metadata](../experiments/military_signature/candidate_metadata.json): + seven HTML-only leads, retrieval times, snapshot hashes, conflicts and rights + quarantine. This is deliberately not an admitted media/segment manifest. +- [Synthetic regression checks](../tests/test_signature_p0.py) and the existing + overlap-audit regression. The new test file requires human review before merge. + +## What the literature supports + +Read the four local Markdown representations first, then verified the selected +PDF pages, including rendered tables. These are a focused feasibility review, +not an exhaustive literature search. + +| Source and verified locator | Evidence | Implication for this project | +|---|---|---| +| P04, Altmann 2004, PDF p16 / printed p728, §5.1; PDF p21 / p733, Table 5 | Engine and track periodicities; diesel Leopard 1/2 examples; spectral peak/order ambiguity. | Periodic descriptors are plausible. Dominant peaks alone do not identify RPM or clean source structure. | +| P05, Altmann et al. 2002, PDF p20 / p1104, Table 3 footnote; p21–22 / p1105–1106, conclusions | Training/testing vectors share passes; harmonic powers depend on state/angle; engine types can unite different vehicle types. | Published recognition is not evidence for this project's independent-session model claim. | +| P11, Göksu 2018, PDF p2–3 / p95–96, §II | Four physical cars, with one speed-sweep episode training and another testing. | Speed variation was studied, but independent-session/model-population transfer remains unestablished. | +| P12, Guo et al., journal issue 2012, PDF p9–11 / p37–39, §6/Table 1/Fig. 3 | Five anonymous classes; six runs/class; three runs training, three testing. | Run holdout is useful precedent; named models, individual identities and independent sessions remain unknown. | +| HW_M1_AGT1500, Honeywell brochure, PDF p1, product description | M1 Abrams uses the AGT1500 gas turbine. | Verifies a coarse powerplant association, not a recording-specific acoustic cue. | + +Full claim locators and PDF hashes are in the source ledger. The proposal's +T-72/T-80/T-90/T-80UD engine table remains incompletely verified. Rostec pages +failed retrieval, ODIN returned no readable content, and the official ARMOR PDF +was inaccessible through the web tool. Search excerpts were retained as leads, +not upgraded into verified facts. Variant-specific candidate powerplants and L3 +engine-order labels remain unknown where unsupported. + +## Public metadata feasibility + +HTML snapshots were fetched without loading scripts or associated media. The +seven leads cover captions for M1, Leopard, Bradley and Stryker, not the six-model +pilot quota. They provide **zero verified independent recording groups** and no +usable-audio/background evidence. + +| Candidate | Metadata lead | Unresolved issue | +|---|---|---| +| [Abrams 1754](https://www.dvidshub.net/video/1754/abrams-tank-b-roll) | 2005; AFN Iraq; 83 s | Location, variant, recording identity and audio quality unknown. | +| [Abrams 572786](https://www.dvidshub.net/video/572786/taa-abrams-set-up-b-roll-package) | 2017; 1st Marine Division; 505 s; M1A1 tag | Tag is not verified interval identity; backgrounds and audibility unknown. | +| [Leopard 855729](https://www.dvidshub.net/video/855729/us-soldiers-explore-finnish-leopard-battle-tank) | 2022; 5th MPAD; 38 s; caption 2A4 | Cultural-exchange scene may lack clean running audio; credits differ between byline/caption. | +| [Bradley 776583](https://www.dvidshub.net/video/776583/b-roll-stallions-conduct-bradley-live-fire) | 2020; 319th MPAD; 43 s | Firing described; contamination not auditioned. | +| [Stryker 386317](https://www.dvidshub.net/video/386317/m1126-stryker-combat-vehicle-convoy-ramstein-air-base) | 2015; DIMOC; 126 s; M1126 | Convoy caption does not establish one audible vehicle. | +| [Stryker 937004](https://www.dvidshub.net/video/937004/pinon-canyon-maneuver-site-b-roll-package) | 2024; 4th Infantry Division PAO; 263 s | Caption Sept 3 versus page Sept 4; recording/session continuity unknown. | +| [Leopard/NATO 879491](https://www.dvidshub.net/video/879491/nato-allies-train-ukrainian-soldiers-leopard-tanks-poland-b-roll) | 2023; Natochannel; 283 s | Date taken/posting conflict; editorial-use restriction despite public-domain banner. Rights quarantine. | + +The [DVIDS FAQ](https://www.dvidshub.net/about/faq) describes exceptions to general +public-domain availability. Per-asset restrictions must be resolved before any +future collection; hosting on DVIDS alone is not a rights clearance or an outlet +control. Audio suitability, permissions for the intended research purpose and +independence were not inferred from public metadata. + +## Design corrections and negative findings + +1. **Background does not identify a channel inverse.** In `x = Hs + n`, a + source-free interval constrains noise `n`; it does not identify `H` or separate + `H` from `s`. This is algebraic reasoning. The draft permits a noise-reference + ablation, not a claim of microphone/propagation removal or clean CAST source + recovery. Calibration and physical identification remain new study inputs. +2. **Eight groups is feasibility; ten is only a minimum.** The pilot target is + six models × eight independently verified components, not eight asset IDs per + model. Ten groups/model is a proposal threshold, not a power calculation. + New unexposed groups and evidence across physical vehicles are needed for a + model-wide claim. Adding two groups does not make exposed pilot results blind. +3. **DVIDS-only support limits outlet conclusions.** A constant outlet probe is + undefined. Originator/producer/camera confounds remain. With only M1 turbines, + a turbine/diesel contrast is also model-associated. Russian models appearing + only in new outlets do not separate outlet and model effects. +4. **Labels and effective fits have limits.** Unknown engine orders yield no L3 + AP. Fine separation without coarse hierarchy/null support does not pass the + claim gate. CAST fits to recorded observations remain effective descriptors; + their stability is not recovered physical source truth. Representation learning + and hyperparameter selection do not use vehicle labels. Development phase + gates expose labels/results and must be disclosed as development evidence. +5. **No-match is not independence.** The reuse audit catches the tested crop, + gain/polarity and resampling cases, but leaves short/silent inputs unresolved. + Codec/time-stretch/edit/mixture behavior, real false matches and aggregate + pilot-scale memory/runtime remain unverified. All known provenance, suspected + reuse and repeated known individuals are conservatively grouped before roles. + +These are limitations of the current evidence and study design, not evidence that +the eventual experiment will succeed or fail. + +## Validation evidence + +The final artificial workflow is at +`artifact:military_signature/p0/validation-20261011-final/validation.json`. +It records Python/library versions, the base commit, dirty-tree status, +implementation/config/schema hashes, seeds 17/29 and every raw fixture hash. + +| Check | Observed result | What it establishes | +|---|---|---| +| Focused signature + existing overlap tests | 37 passed: 32 new signature cases, 5 overlap cases | Authorization/type/provenance guards, contamination/overlap handling, grouping, packet commitments, native rounding and tested reuse cases. | +| Artificial end-to-end workflow | 6 assets; 8 native clips; 2 logged rejections; 3 components | Fixture generation → blind packet → import → grouping → native export, with exact sample comparisons. | +| External JSON Schema validator | Draft 2020-12 schema and resulting manifest valid | Interoperable structural schema; runtime additionally checks scientific constraints. | +| Offline viewer DOM events (`jsdom@26.1.0`) | 6 cards; 7 saved intervals; import/export produced 2 clips and retained 5 rejections | Add/remove, invalid bounds/overlap/contamination guards, reviewer/completeness checks and JSON serialization. Audio/play/download APIs were mocked. | +| Native Firefox access | Computer-use approval rejected access | Actual decoding, audible playback, layout and native download remain unverified. Human browser handoff required. | +| CAST harness canary | 707 collected vs 467 expected; 1 failed, 2 passed | Count drift remains visible; baseline-file/frozen-file checks passed. | + +DOM evidence is at +`artifact:military_signature/p0/validation-20261011-r2/dom-check/validation.json`. +The reproducible DOM script and package lock are retained there. It is a functional +JavaScript check, not a real browser or blind scientific review. The final +artificial workflow uses the same viewer source bytes. No generative vehicle +simulation, trained representation or scientific metric was tested by this work. + +The harness already collected 661 against 467 before P0. This work legitimately +adds 32 cases; other work also changed the shared suite. Do not attribute the +entire 707 count to this project. The human-only baseline and existing tests, +lockfiles and frozen experiment configs were not edited to make the canary pass. +Four `experiments/joint_confirmation/*DRAFT.md`/`PLAN.md` files changed elsewhere +during this work; they were not edited or reverted by P0. + +## Remaining decisions and exact handoffs + +P0 preparation can be reviewed using the freeze check and reproduction commands +in the protocol. Prior versions remain in external storage; reruns require fresh +paths. No commit or merge was performed. + +- **Browser handoff:** open the final `blind/index.html` locally, play an artificial + recording, review all cards, save JSON and exercise import. Supply approved + Firefox access or perform this human check; current DOM evidence cannot prove + decoding, rendering or audible playback. +- **Harness handoff:** review the new test file line by line; account for all + legitimate suite changes in both environments; have a human refresh the baseline + in the same commit before merge. No agent baseline refresh is authorized. +- **Research handoff:** provide explicit bounded P1 collection/processing/fitting + authorization, exact frozen checkpoint/extraction/CAST budgets, confirmed + per-asset rights, independently supported session groups and a six-model roster. + Missing or ambiguous powerplants stay unknown unless an accessible authoritative + PDF or primary specification resolves them. +- **Confirmation design:** before model-wide claims, freeze new unexposed groups, + physical-vehicle support, final allocation/power analysis, noise-reference + assumptions, and synthetic recovery/negative-control criteria. Preserve all + failed gates and inconclusive/null results; do not collect until a result turns + positive. + +The next scientific action is a separately authorized P1 admission/pilot protocol. +The P0 artifacts do not authorize that action or provide its data. diff --git a/reports/military_signature_P0_followup.md b/reports/military_signature_P0_followup.md new file mode 100644 index 0000000..82592e2 --- /dev/null +++ b/reports/military_signature_P0_followup.md @@ -0,0 +1,64 @@ +# Military acoustic signature P0 evidence follow-up + +2026-10-11 local date; protocol `military_signature_p0_v1`. +This is an additive supplement to the [frozen P0 report](military_signature_P0.md). +The original report, source ledger and 55-file freeze remain the dated P0 snapshot. +This follow-up covers public literature and the artificial browser round trip. + +The supplied annotation passed import, connected-group validation and native +export. The user has now confirmed that Chrome audio playback, waveform, +spectrogram and interval controls displayed correctly. The [round-trip record](../experiments/military_signature/ANNOTATION_ROUNDTRIP.md) +links the explicit confirmation receipt and sample-exact export evidence. This +closes the human browser handoff listed in the original report. It is run-level +user evidence; there is no instrumented or per-recording playback trace. + +## Additional powerplant evidence + +Six additional original public PDFs were retrieved. Their extracted Markdown +was read before the selected original pages were rendered and visually checked. +The [supplementary source ledger](../experiments/military_signature/source_evidence_supplement.json) +records PDF/Markdown hashes, page and section locators, author statements, +interpretations, rendered pages and failed access attempts. Documents and +renders are outside the checkout at +`artifact:military_signature/p0/literature-followup-20261011/`. + +| Verified scope | Author evidence and locator | Interpretation and remaining unknowns | +|---|---|---| +| Bradley production beginning in 1981 | Cummins associates its start with VTA903-500; `CUMMINS_DEFENSE`, PDF p13 / printed p11. `CUMMINS_V903`, PDF p2, specifies V903-family V8 four-stroke diesel architecture. | Historical association plus a later family specification. The proposed Bradley recording's variant and engine remain unknown. | +| M109A6 in a 1992 Army report | `ARMY_M109_1992`, PDF p57 / printed p55, describes an eight-cylinder diesel modification designated 8V71T-LHR. | Supports this reported variant. Stroke count and engines in other M109 variants or recordings remain unknown. | +| Strykers in a documented 2020 reset program | `ARMY_STRYKER_2020`, PDF p1 and p4, reports arrivals with C7 or 3126 engines and serviced C7 departures. `CAT_DIESEL_FLUIDS_2025`, PDF/printed p49, names both families as diesel. | Establishes engine heterogeneity in this population. A Stryker caption cannot identify the engine of a recording. Cylinder/stroke counts are not established here. | +| M1165A1 Special Ops brochure configuration | `AMGENERAL_M1165`, PDF p1–2, identifies the HMMWV variant and specifies a GEP 6.5-litre turbocharged V8 diesel. | Applies to the brochure configuration. It does not establish other HMMWV variants, JLTV, recording identity or stroke count. | + +The source PDFs are [Cummins defense](https://mart.cummins.com/imagelibrary/data/assetfiles/0075965.pdf), +[Cummins V903](https://mart.cummins.com/imagelibrary/data/assetfiles/0063416.pdf), +[Army RD&A September–October 1992](https://asc.army.mil/docs/pubs/alt/archives/1992/Sep-Oct_1992.PDF), +[Anniston Tracks January 23, 2020](https://anad.army.mil/docs/librariesprovider2/default-document-library/tr012320_v2.pdf?sfvrsn=7a536430_0), +[Caterpillar SEBU6251-32](https://caterpillar.scene7.com/is/content/Caterpillar/CM20250210-d820f-61dca) +and [AM General M1165A1](https://www.amgeneral.com/wp-content/uploads/2019/12/M1165-Special-Ops.pdf). + +Initial Python certificate failures are retained in the retrieval record. +System curl retrieved the affected documents using its default TLS validation; +certificate verification was not disabled. Three ARMOR originals still timed +out or lacked usable original-page verification. T-72/T-80/T-80UD/T-90 entries +therefore remain unverified under this project's evidence contract. Search +snippets do not supply verified powerplant labels. + +## Consequences for a future pilot + +Engine family and vehicle model need separate evidence fields. The documented +Stryker reset population supplies a concrete reason to retain within-model +engine heterogeneity. Manufacturer architecture cannot identify a recording's +firing rate, engine order or dominant acoustic mechanism without additional +ground truth. An engine specification provides no independent-session evidence. + +The six-model roster remains a future pilot decision. “HMMWV/JLTV” cannot serve +as one model identity. An M1-only turbine population still cannot separate a +generic turbine effect from the M1 model effect. The existing distinction +between eight development groups and the proposal's ten-group minimum remains; +neither is a power analysis or a sufficient model-generalization guarantee. + +The new documents add no admitted media, usable-audio observations or verified +independent recording groups. They support protocol preparation only. The +remaining handoffs are a human harness/test review before merge and separately +scoped P1 authorization with the exact population, assets, representation, +budgets and evaluation allocation frozen before execution. diff --git a/reports/military_signature_P1_admission.md b/reports/military_signature_P1_admission.md new file mode 100644 index 0000000..7bc79a3 --- /dev/null +++ b/reports/military_signature_P1_admission.md @@ -0,0 +1,240 @@ +# Military acoustic signature P1 collection and review preparation + +2026-10-11–12 local dates; contract `military_signature_p1_admission_5gb_v1`. +The user authorized more data, retained the 20-minute per-asset limit, then +reduced the starting storage allocation to 5 GB. The separately versioned +[authorization record](../experiments/military_signature/P1_AUTHORIZATION.md) +and [canonical contract](../experiments/military_signature/p1_admission_5gb.json) +record this scope. The original P0 protocol and 55-file freeze remain unchanged. + +The collection contains **54 recordings, 8,735.906 seconds (2 h 25 min 35.906 s)**. +The longest recording is 505.706 seconds, below the 1,200-second asset limit. +Data, metadata and seven review packets currently occupy about **4.88 GB** of +the **5,000,000,000-byte** ceiling. The 64-asset first-batch target was not reached +under the storage/reserve limit. Storage is a ceiling, not a fill requirement. + +These are collected playable recordings pending human blind review, not an +admitted segment dataset. There are **zero verified independent recording groups**, +zero admitted foreground segments and no experimental role allocation. No +representation, classifier or CAST model was fitted or evaluated by this run. + +## Collected caption support + +| Candidate caption class | Collected assets containing the caption cue | +|---|---:| +| M1 Abrams family | 14 | +| Bradley family | 11 | +| Stryker family | 11 | +| M109 / Paladin | 10 | +| HMMWV / Humvee | 6 | +| Leopard family | 5 | + +Mixed-model captions contribute to more than one row; the rows do not sum to +the 54 unique assets. These counts establish neither audible model identity nor +session independence. The caption classes still mix family and variant granularity. +Engine identities, physical vehicles, foreground/background support and audio +quality remain unknown until the relevant evidence is reviewed. + +The metadata names 38 originators. This is candidate diversity, not 38 independent +sessions or outlets. All assets are hosted on DVIDS, so outlet invariance remains +unestablished. Native audio is 48 kHz for 51 recordings, 44.1 kHz for two and +22.05 kHz for one; 51 recordings are stereo and three mono. Retain these recording +and codec differences as possible confounds in any separately authorized pilot. + +The metadata-only inventory `metadata_confound_inventory.json` binds all 54 +collection records and resolves the format counts by caption class: + +| Caption class | 48 kHz | 44.1 kHz | 22.05 kHz | Mono assets | +|---|---:|---:|---:|---:| +| Abrams | 13 | 0 | 1 | 1 | +| Leopard | 5 | 0 | 0 | 0 | +| Bradley | 11 | 0 | 0 | 0 | +| Stryker | 11 | 0 | 0 | 0 | +| M109 | 10 | 0 | 0 | 0 | +| HMMWV | 4 | 2 | 0 | 2 | + +All 54 retained audio streams use AAC. The only HLS acquisition is in the Abrams +caption class; the other 53 use explicit page MP4 downloads. There are 51 +single-class caption packages and three mixed-class packages. Known date-taken +metadata spans 2005-01-27 through 2026-09-28. These are observed acquisition and +metadata differences, not measured classifier decodability or evidence of an +acoustic mechanism. A later frozen pilot must retain them as confounds and +declare any common preprocessing using training data alone. This inventory does +not assign labels to human-selected intervals or certify independent sessions. + +## Provenance, restrictions and failures + +The discovery queue contains 109 asset IDs. The run recorded outcomes for 74 +unique IDs across 76 attempts: 54 collected IDs and 20 IDs still quarantined. +Both prior and later attempts are retained, including the recovered older HLS +asset. Failure records include rights restrictions, historical-source exclusions, +irrelevant caption hits, rendition byte limits and older-playlist retrieval errors. + +Existing archival catalogs were read only as metadata. Eight known DVIDS IDs +and seven historical source-filename tokens were excluded from acquisition. +No historical or protected audio was loaded for fingerprint comparisons, and +their roles and files were unchanged. Asset-level rights decisions were recorded +before payload access. The [DVIDS public-use notice](https://www.dvidshub.net/about/copyright) +was retained alongside source pages; additional restrictions or missing usage +evidence caused quarantine. Metadata-based admission does not guarantee absence +of unmarked third-party content. Media publication remains outside this stage. + +Some old public HLS playlists used `application/octet-stream`. The collector's +second version accepts this only for bounded `.m3u8` responses containing an +actual HLS header. It does not treat arbitrary binary responses as playlists. +Earlier errors and both collector versions remain available for review. + +One retry reached its output byte bound and was rejected as possibly truncated. +Its failure record, payload SHA256 and byte count were retained before discarding +the 79,072,864-byte incomplete payload. No valid collected rendition was removed. +Other deferred candidates were not replaced by silently trimming overlong media. + +## Mechanical verification and completed reuse audit + +- All 54 retained source renditions and extracted audio files match their recorded + SHA256 values. Encoded audio-packet hashes agree between each source and its + extracted M4A: no audio re-encoding was introduced by extraction. +- A separate cross-asset comparison found no identical whole encoded-audio packet + sequences among the 54 assets. Current native-file hashes were rechecked against + the acquisition ledger; `encoded_packet_reuse.json` binds those hashes and + records the negative result. Distinct whole-stream hashes do not rule out + partial, edited or transcoded waveform reuse and do not certify independence. +- Every source obeys the 20-minute cap. Caption/source names were absent from + native audio metadata; all experimental roles remain `unassigned`. +- Native-channel signal integrity was checked for all 54 recordings with no + resampling or channel downmixing. One retained stream decodes entirely to zeros + in every channel; 53 contain nonzero decoded samples. No asset has an entirely + zero mono display while retaining nonzero native-channel audio. The per-asset + counts and hashes are in `native_signal_integrity.json`. Nonzero samples do not + establish audible vehicles, usable foreground intervals or source-model labels. + The zero-stream finding is retained as mechanical evidence; no human rejection + or accepted interval was fabricated and no collection record was rewritten. +- Seven packets contain 54 caption-hidden review cards. A local server verified + packet retrieval, native audio byte-range responses and rejection of private + mapping/traversal paths. Browser audibility and human interval decisions remain + the reviewer's observations; HTTP checks do not establish them. +- The production native importer/exporter passed a synthetic round trip with + frames 16–32016 at 16 kHz, exact decoded-sample equality and rejection of a + changed blind-source identity. No real recording received synthetic annotations. +- Focused regression: 50 checks passed (13 P1 boundary/review/parity checks, 32 P0 checks + and five existing overlap checks). The CAST harness canary remains red at + 720 collected versus 467 expected; baseline-file and frozen-file checks passed. + Baseline refresh is human-only, and the new test file needs line-by-line human + review before merge. No baseline, existing tests, lockfile or frozen configuration + was changed to make the canary pass. +- The full label-blind waveform reuse audit completed at 2026-10-12 07:10:02 UTC. + All 54 assets are accounted for: 53 fingerprinted recordings with 4,175 indexed + windows and one entirely zero stream marked `short_or_silent_unresolved`. + Its authoritative result is + `artifact:military_signature/p1_admission/20261011_5gb/reuse_audit.json`. + One four-second reuse suggestion links assets 897659 and 897660: reference + start 100.000 seconds, query start 64.384 seconds, 265 landmark votes and + correlation 0.988650272254228. The frozen original library reproduced the exact + reported correlation. This agrees with the retained metadata dependence lead; + it does not establish a verified session or physical-vehicle identity. No-match + results do not establish independence. + +The initial reuse run was intentionally interrupted while incomplete, after 43 +recordings had been observed as processed. Its source and interruption record +remain available. A packed-integer vote counter replaces the costly two-column +sort while preserving reference IDs, signed offsets and exact counts. One million +synthetic hit pairs produced identical keys/counts; the controlled positive-match +fixture also produced identical full match rows. The reference library, correlation +calculation, thresholds and full 54-recording population remain unchanged. +The third run's completed result now supplies the full-population evidence; +neither interrupted run is treated as a completed audit. + +A subsequent native process profile showed repeated waveform dot products +dominated the remaining runtime. The third implementation nominates possible +winning lags with FFT correlation, then returns scores computed with the original +direct-dot arithmetic. Ill-conditioned cases use the original full lag scan. +Scores were identical in 25 synthetic and 16 newly collected-media cases; all +thresholds and the full population remain unchanged. The second incomplete run, +its source and native stack profile are retained. This is a numerical processing +change, not evidence about an acoustic mechanism or generalization. + +A separate workload diagnostic for asset 44 (921247) rebuilt the same first +43-asset reference index without running another full audit. It found 280,765 +candidate offsets across 49 reference windows, including 273,110 offsets with +valid query bounds. Twenty-five sampled correlations and their observed runtimes +are retained in `reuse_v3_workload_diagnostic.json`. This large candidate workload +explains why an asset-level progress counter can remain unchanged while the job +is active. The diagnostic did not restart or reduce the full audit, and does not +itself certify completion or session independence. + +A small independent dot-product benchmark compared the default BLAS thread +setting with `VECLIB_MAXIMUM_THREADS=1`; both produced identical scalar values +and similar timings. This negative check did not support changing the thread +setting. Additional display-waveform checks found long exact-zero regions in the +two expensive queries. Their constant centered windows explain near-zero lag +scores with many ambiguous candidates; this is a numerical workload observation, +not an acoustic mechanism. The audit's inputs, lag search and criteria remain +unchanged. + +An isolated constant-query shortcut was also rejected. Replacing the full lag +scan with one lag for a numerically constant query region failed exact-score +parity on synthetic case 19 (seed 43), after 19 prior cases passed. The original +and shortcut scores differed by 8.673617379884035e-18. The unused source, +`flat_query_correlation_prototype_check.json` and compressed synthetic reproducer +are retained outside the checkout. The attempted check stopped before its planned +new-media cases and timing benchmark; neither is reported as passed. No production +code, thresholds or tests were changed to accept the shortcut. This is numerical +implementation evidence, with no physical or recognition interpretation. + +Metadata dependence checks identify the two Bradley packages, 897659 and 897660, +through their shared creator/day VIRIN stem and matching date/originator/producer. +The record `metadata_dependence_leads.json` retains this possible common-shoot +link. It neither assigns session identities nor treats unmatched assets as +independent. Human and audio reuse evidence remain part of admission. + +The provisional admission ledger `provisional_admission_groups.json` binds all +54 unchanged collection records and retains these metadata links. Following the +frozen protocol's unknown-session rule, all assets currently share one quarantine +component and cannot enter held-out roles. This component does not assert a known +common shoot or physical vehicle. Its verified independent-group count is zero; +it was not an experimental admission freeze. The completed automated ledger +`admission_groups.json` now incorporates the waveform suggestion, metadata +links and zero-stream flag, while preserving the same quarantine constraint. +All 54 source/native hash pairs, seven review-packet commitments and 11 protected +repository witnesses passed the closing verification. The ledger remains pending +human admission and is not frozen for a fitting/evaluation experiment. No +collection records or prior evidence snapshots were rewritten. + +The external run is rooted at +`data:military_signature/p1_admission/20261011_5gb/` and +`artifact:military_signature/p1_admission/20261011_5gb/`. +Key records are `contract.json`, `collection_facts.json`, +`acquisition_verification.json`, `review_packet_receipt.json`, per-asset +`record.json`/rights/FFprobe files, resume/source snapshots and +`synthetic-native-validation/verification.json`. + +## Human review and reproducible commands + +Open [the local review index](http://127.0.0.1:8766/) in Chrome. Start with packet 1; +record accepted foreground/background intervals or logged rejections for every +recording, then save that packet's annotations. Each packet has a different batch +commitment. Source captions and private mappings are not served by the review UI. +The server requires the external run; it does not copy audio again or contact +external media services during playback. + +```sh +PYTHONPATH=src .venv/bin/python -m abvid.signature.admission status +PYTHONPATH=src .venv/bin/python -m abvid.signature.admission_review serve --port 8766 +PYTHONPATH=src .venv/bin/python -m abvid.signature.admission_annotations import \ + --packet packet-01 --input /path/to/annotations.json \ + --output artifact:military_signature/p1_admission/20261011_5gb/annotations/packet-01.json +PYTHONPATH=src .venv/bin/python -m abvid.signature.admission_annotations export \ + --input artifact:military_signature/p1_admission/20261011_5gb/annotations/packet-01.json \ + --output artifact:military_signature/p1_admission/20261011_5gb/clips/packet-01 +``` + +Imports bind to the original human JSON, opaque mapping, batch and unchanged +collection. Exports preserve floor-start/ceil-end native frames and retain every +rejection. Exports that cannot fit the remaining storage allowance are explicitly +deferred. The recorded native validations do not substitute for human review of +real material or independently supported shoot/individual provenance. + +The next scientific pilot still needs an admitted population, frozen identity +granularity, supported recording groups and a separate fitting/evaluation scope. +The eight-group feasibility target and ten-group proposal minimum remain distinct; +neither can be established from these raw asset counts. diff --git a/src/abvid/signature/__init__.py b/src/abvid/signature/__init__.py new file mode 100644 index 0000000..21f69bb --- /dev/null +++ b/src/abvid/signature/__init__.py @@ -0,0 +1,8 @@ +"""Preparation tools for the separately scoped acoustic-signature study. + +These tools do not fit representations, CAST parameters, or classifiers. +The P0 command surface admits synthetic fixtures only. +""" + +PROTOCOL_ID = "military_signature_p0_v1" +TOOL_VERSION = "signature-preparation-1" diff --git a/src/abvid/signature/__main__.py b/src/abvid/signature/__main__.py new file mode 100644 index 0000000..162914b --- /dev/null +++ b/src/abvid/signature/__main__.py @@ -0,0 +1,82 @@ +"""P0 CLI: validate, make fixtures, blind review, trim, and audit reuse.""" +from __future__ import annotations + +import argparse +import json + +from abvid.paths import resolve + +from .config import DEFAULT_CONFIG, external_path, load +from .dedup import audit_manifest, with_reuse_links +from .fixtures import create_bundle +from .manifest import readiness, validate +from .trim import _write_json_new, export_segments, import_annotations, make_packet + + +def _read(reference): + return json.loads(resolve(reference).read_text()) + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Synthetic-only military-signature P0 preparation") + parser.add_argument("--config", default=DEFAULT_CONFIG) + subs = parser.add_subparsers(dest="command", required=True) + p = subs.add_parser("fixtures", help="Generate artificial chirp/noise fixtures outside Git") + p.add_argument("--output", required=True) + p = subs.add_parser("validate-p0", help="Run an artificial end-to-end preparation check") + p.add_argument("--fixtures", required=True) + p.add_argument("--output", required=True) + for name in ["validate", "readiness", "dedup", "review", "import-annotations", "export"]: + p = subs.add_parser(name) + p.add_argument("--manifest", required=True) + if name in {"dedup", "review", "import-annotations", "export"}: + p.add_argument("--output", required=True) + if name == "review": + p.add_argument("--private-mapping", required=True) + p.add_argument("--seed", type=int, default=17) + if name == "import-annotations": + p.add_argument("--batch", required=True) + p.add_argument("--private-mapping", required=True) + p.add_argument("--annotations", required=True) + p.add_argument("--role", choices=["development", "validation", "evaluation"], default="development") + p = subs.add_parser("group") + p.add_argument("--manifest", required=True) + p.add_argument("--audit", required=True) + p.add_argument("--output", required=True) + args = parser.parse_args(argv) + config = load(args.config) + if args.command == "validate-p0": + from .validation import run + result = run(args.fixtures, args.output, config_reference=args.config) + elif args.command == "fixtures": + result = create_bundle(args.output, config) + result = {"manifest_ref": args.output + "/manifest.json", "assets": len(result["assets"])} + else: + manifest = _read(args.manifest) + validate(manifest, config) + if args.command == "validate": + result = {"valid": True, "scope": "synthetic_fixture_only", "assets": len(manifest["assets"]), + "segments": len(manifest["segments"])} + elif args.command == "readiness": + result = readiness(manifest, config) + elif args.command == "review": + result = make_packet(manifest, config, args.output, args.private_mapping, seed=args.seed) + elif args.command == "export": + result = export_segments(manifest, config, args.output) + result = {"output_ref": args.output, "clips": len(result["clips"]), "rejections": len(result["rejections"])} + else: + if args.command == "dedup": + result = audit_manifest(manifest, config) + elif args.command == "group": + result = with_reuse_links(manifest, _read(args.audit), config) + else: + result = import_annotations(manifest, config, _read(args.batch), _read(args.private_mapping), + _read(args.annotations), role=args.role) + _write_json_new(external_path(args.output), result) + result = {"output_ref": args.output, "command": args.command} + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/abvid/signature/admission.py b/src/abvid/signature/admission.py new file mode 100644 index 0000000..550d6b0 --- /dev/null +++ b/src/abvid/signature/admission.py @@ -0,0 +1,576 @@ +"""Bounded, separately authorized DVIDS media acquisition for P1 admission. + +The immutable P0 contract is never relaxed. Collected recordings remain pending +human blind review and have no experimental role or verified session identity. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +import hashlib +from html import unescape +from html.parser import HTMLParser +import json +import math +from pathlib import Path +import re +import shutil +import subprocess +import time +from urllib.parse import urljoin, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from abvid.paths import resolve +from abvid.provenance.hashing import sha256 +from .config import external_path + +CONFIG_REF = "repo:experiments/military_signature/p1_admission_5gb.json" +CONTRACT_HASH = "07c86a7a64dce473847e1a8928b1bde50b7375ac44194d4fcd8e5fc8e1cdea27" +TOOL_VERSION = "signature-admission-2" +MEDIA_HOST = "d34w7g4gy10iej.cloudfront.net" + + +def now(): + return datetime.now(timezone.utc).isoformat() + + +def load_config(reference=CONFIG_REF): + cfg = json.loads(resolve(reference).read_text()) + encoded = json.dumps(cfg, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + if hashlib.sha256(encoded).hexdigest() != CONTRACT_HASH: + raise ValueError("Changed P1 admission contract; version a new authorization/configuration") + for key in ("data_root", "artifact_root"): + external_path(cfg["storage"][key]) + return cfg + + +def write_new(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as stream: + json.dump(value, stream, indent=2, allow_nan=False) + stream.write("\n") + + +def storage_used(cfg): + # Counting every path is conservative if a future packet uses hard links. + total = 0 + for name in ("data_root", "artifact_root"): + root = external_path(cfg["storage"][name]) + if root.exists(): + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError("Symlinks are forbidden within the new collection") + if path.is_file(): + total += path.stat().st_size + return total + + +def available_bytes(cfg): + root = external_path(cfg["storage"]["data_root"]) + existing = next(p for p in [root, *root.parents] if p.exists()) + free = shutil.disk_usage(existing).free + if free < cfg["storage"]["minimum_free_disk_bytes"]: + raise OSError("Free-disk reserve reached") + remaining = cfg["storage"]["maximum_total_bytes"] - storage_used(cfg) + return min(remaining, free - cfg["storage"]["minimum_free_disk_bytes"]) + + +def check_url(url, *, media=False): + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.username or parsed.password or parsed.port: + raise ValueError("Only plain HTTPS metadata/media URLs are allowed") + if media: + if parsed.hostname != MEDIA_HOST or not parsed.path.startswith("/video/"): + raise ValueError("Media URL is not the explicit DVIDS video CDN") + elif parsed.hostname != "www.dvidshub.net": + raise ValueError("Metadata URL left DVIDS") + return url + + +class _Redirect(HTTPRedirectHandler): + def __init__(self, media): + self.media = media + super().__init__() + + def redirect_request(self, req, fp, code, msg, headers, newurl): + check_url(newurl, media=self.media) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def response(url, cfg, *, media=False): + check_url(url, media=media) + opener = build_opener(_Redirect(media)) + return opener.open(Request(url, headers={"User-Agent": "ABVID-P1-Research-Admission/1.0"}), + timeout=cfg["collection"]["request_timeout_seconds"]) + + +def fetch_text(url, cfg, destination, *, media=False): + with response(url, cfg, media=media) as stream: + content_type = stream.headers.get_content_type() + binary_playlist = (content_type == "application/octet-stream" and + urlsplit(url).path.endswith(".m3u8")) + if not (content_type.startswith("text/") or "mpegurl" in content_type or binary_playlist): + raise ValueError("Unexpected metadata/playlist content type: " + content_type) + limit = cfg["collection"]["maximum_metadata_bytes"] + data = stream.read(limit + 1) + if len(data) > limit or len(data) > available_bytes(cfg): + raise ValueError("Metadata bound exceeded") + if binary_playlist and not data.lstrip().startswith(b"#EXTM3U"): + raise ValueError("Octet-stream playlist did not contain an HLS manifest") + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as handle: + handle.write(data) + return data.decode("utf-8", "replace") + + +class _Text(HTMLParser): + def __init__(self): + super().__init__() + self.parts = [] + self.ignored = 0 + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style"): + self.ignored += 1 + if tag in ("br", "p", "div", "tr"): + self.parts.append(" ") + + def handle_endtag(self, tag): + if tag in ("script", "style"): + self.ignored = max(0, self.ignored - 1) + self.parts.append(" ") + + def handle_data(self, data): + if not self.ignored: + self.parts.append(data) + + +def plain(fragment): + parser = _Text() + parser.feed(fragment) + return " ".join("".join(parser.parts).split()) + + +def _element(html, tag, css_class): + match = re.search(rf'<{tag}\b[^>]*class="[^"]*\b{re.escape(css_class)}\b[^"]*"[^>]*>(.*?)', + html, flags=re.S | re.I) + return match.group(1) if match else "" + + +def caption_classes(text): + lower = text.lower() + found = [] + if re.search(r"\bm1a[12]\b", lower) or ( + "abrams" in lower and re.search(r"\btanks?\b|\bmain battle\b", lower)): + found.append("abrams") + if re.search(r"\bleopard(?:[- ]?[12])?\b", lower): + found.append("leopard") + if "bradley" in lower and re.search(r"fighting|vehicle|convoy|maneuver|\bm2\b", lower): + found.append("bradley") + if re.search(r"\bstryker\b", lower): + found.append("stryker") + if re.search(r"\bm[- ]?109(?:a[67]|[- ]a[- ]?[67])?\b|\bpaladin\b", lower): + found.append("m109") + if re.search(r"\bhumvees?\b|\bhmmwvs?\b|high mobility multipurpose", lower): + found.append("hmmwv") + return found + + +def page_metadata(html, asset_id, url): + title = plain(_element(html, "h1", "asset-title")) + table = _element(html, "table", "video_info_table") + fields = {} + for row in re.findall(r"]*>(.*?)", table, re.S | re.I): + cells = re.findall(r"]*>(.*?)", row, re.S | re.I) + if len(cells) == 2: + fields[plain(cells[0]).rstrip(":")] = plain(cells[1]) or "unknown" + caption_match = re.search( + r'
\s*

(.*?)

', + html, re.S | re.I) + caption = plain(caption_match.group(1)) if caption_match else "" + copyright_info = plain(_element(html, "p", "copyright_info")) + domains = [plain(v) for v in re.findall( + r']*class="[^"]*\bpublic-domain\b[^"]*"[^>]*>(.*?)', html, re.S | re.I)] + duration_text = fields.get("Length", "") + match = re.fullmatch(r"(\d{2}):(\d{2}):(\d{2})", duration_text) + duration = sum(int(v) * factor for v, factor in zip(match.groups(), [3600, 60, 1])) if match else None + media_urls = [] + for attrs in re.findall(r"]+)>", html, re.S | re.I): + src = re.search(r'\bsrc=["\']([^"\']+)["\']', attrs) + if src and urlsplit(unescape(src.group(1))).path.endswith(".mp4"): + media_urls.append(unescape(src.group(1))) + h3s = [plain(s) for s in re.findall(r"]*>(.*?)", html, re.S | re.I)] + producer = next((s.removeprefix("Video by ") for s in h3s if s.startswith("Video by ")), "unknown") + unit = plain(_element(html, "h3", "the_unit")) or "unknown" + if not title or not caption or fields.get("Video ID") != str(asset_id): + raise ValueError("Incomplete/mismatched DVIDS title, caption or asset table") + return { + "asset_id": str(asset_id), "url": url, "title": title, "caption": caption, + "caption_classes": caption_classes(title + " " + caption), "caption_confidence": "caption", + "producer": producer, "unit": unit, "outlet": "DVIDS", + "date_taken": fields.get("Date Taken", "unknown"), + "date_posted": fields.get("Date Posted", "unknown"), + "location": fields.get("Location", "unknown"), "VIRIN": fields.get("VIRIN", "unknown"), + "source_filename": fields.get("Filename", "unknown"), + "duration_seconds_from_page": duration, + "public_domain_banner": any(s == "PUBLIC DOMAIN" for s in domains), + "copyright_info": copyright_info, "mp4_urls": media_urls, + "recording_session": "unknown", "session_evidence": "unknown", + "physical_vehicle_id": "unknown", "independence_verified": False, + "experimental_role": "unassigned", "engine_identity": "unknown", + "audio_quality": "unknown", "foreground_seconds": "unknown", + "background_support": "unknown", "blind_review_status": "pending", + } + + +def rights_decision(row): + text = " ".join([row["caption"], row["copyright_info"], row["unit"]]).lower() + restricted = [ + r"editorial use only", r"no commercial", r"no archive", r"not for sale", + r"not for redistribut", r"rights reserved", r"copyrighted", + r"reuters", r"associated press", r"natochannel", r"nato channel", + r"courtesy of", r"used with permission", r"\(c\)|©", + ] + reasons = [pattern for pattern in restricted if re.search(pattern, text)] + if not row["public_domain_banner"] or not row["copyright_info"]: + reasons.append("missing public-domain banner or asset usage statement") + if row["unit"] == "unknown": + reasons.append("originator unknown") + return { + "decision": "quarantine" if reasons else "collect_for_local_noncommercial_research", + "basis": "Per-asset banner, originator and caption/usage restriction review plus DVIDS public-use notice", + "policy_url": "https://www.dvidshub.net/about/copyright", + "flags": reasons, + "limitations": "Metadata-based admission, not a guarantee about unmarked third-party content; human audio review remains required", + "media_redistribution_authorized": False, + } + + +def historical_exclusions(cfg): + ids, filenames, witnesses = set(), set(), [] + for ref in cfg["protected_catalogs"]: + path = resolve(ref) + text = path.read_text() + ids.update(re.findall(r"dvidshub\.net/video/(\d+)", text)) + filenames.update(re.findall(r"/(DOD_[A-Za-z0-9_]+)(?:/|-|\.)", text)) + witnesses.append({"reference": ref, "sha256": sha256(path)}) + return ids, filenames, witnesses + + +def validate_playlist(text, base_url, maximum_seconds): + if "#EXT-X-ENDLIST" not in text or "#EXTINF:" not in text: + raise ValueError("Only finite media playlists are accepted") + if "#EXT-X-KEY" in text or "#EXT-X-MAP" in text or "#EXT-X-BYTERANGE" in text: + raise ValueError("Encrypted or unsupported media playlist") + durations = [float(x) for x in re.findall(r"#EXTINF:([^,]+)", text)] + if any(not math.isfinite(x) or x <= 0 for x in durations): + raise ValueError("Invalid playlist segment durations") + duration = sum(durations) + if duration > maximum_seconds: + raise ValueError("Playlist exceeds per-asset duration; long assets are not trimmed to fit") + urls = [urljoin(base_url, line.strip()) for line in text.splitlines() + if line.strip() and not line.startswith("#")] + if len(urls) != len(durations): + raise ValueError("Playlist segment/duration count differs") + for url in urls: + check_url(url, media=True) + return duration + + +def select_hls(master): + choices = [] + lines = master.splitlines() + for i, line in enumerate(lines): + if line.startswith("#EXT-X-STREAM-INF:"): + attrs = dict(re.findall(r"([A-Z-]+)=([^,]+)", line)) + bandwidth = int(attrs.get("BANDWIDTH", "0")) + resolution = attrs.get("RESOLUTION", "unknown") + height = int(resolution.split("x")[-1]) if "x" in resolution else 0 + if i + 1 < len(lines) and height <= 720: + url = lines[i + 1].strip() + check_url(url, media=True) + choices.append((bandwidth, url, resolution)) + if not choices: + raise ValueError("No allowed finite HLS rendition") + bandwidth, url, resolution = max(choices) + return {"url": url, "bandwidth": bandwidth, "resolution": resolution} + + +def probe(path): + result = subprocess.run( + ["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(path)], + capture_output=True, text=True, timeout=30, check=True) + metadata = json.loads(result.stdout) + duration = float(metadata["format"]["duration"]) + audio = next((s for s in metadata["streams"] if s.get("codec_type") == "audio"), None) + if not audio or not math.isfinite(duration) or duration <= 0: + raise ValueError("Media has no valid audio stream/duration") + return metadata, duration, audio + + +def run_ffmpeg(args, destination, cfg, *, timeout=600): + limit = min(cfg["collection"]["maximum_encoded_asset_bytes"], + available_bytes(cfg) - cfg["storage"]["reserved_workspace_bytes"]) + if limit <= 0: + raise OSError("Storage budget reached") + command = ["ffmpeg", "-nostdin", "-v", "error", "-n", *args, "-fs", str(limit), str(destination)] + log = destination.with_suffix(destination.suffix + ".ffmpeg.log") + with log.open("x") as handle: + process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=handle) + deadline = time.monotonic() + timeout + try: + while process.poll() is None: + if time.monotonic() > deadline: + raise TimeoutError("Bounded ffmpeg operation timed out") + if available_bytes(cfg) < cfg["storage"]["reserved_workspace_bytes"]: + raise OSError("Storage reserve reached during ffmpeg") + time.sleep(0.2) + if process.returncode: + raise ValueError(f"ffmpeg exit {process.returncode}; see retained log") + if destination.stat().st_size >= limit: + raise OSError("Output hit byte bound and may be truncated") + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return command + + +def download_file(url, destination, cfg): + check_url(url, media=True) + limit = min(cfg["collection"]["maximum_encoded_asset_bytes"], + available_bytes(cfg) - cfg["storage"]["reserved_workspace_bytes"]) + if limit <= 0: + raise OSError("Storage budget reached") + digest = hashlib.sha256() + size = 0 + with response(url, cfg, media=True) as stream: + expected = stream.headers.get("Content-Length") + if expected is not None and int(expected) > limit: + raise OSError("Advertised media exceeds available asset/storage budget") + if stream.headers.get_content_type().startswith("text/"): + raise ValueError("Media request returned text") + with destination.open("xb") as output: + while True: + chunk = stream.read(262144) + if not chunk: + break + if size + len(chunk) > limit or len(chunk) > available_bytes(cfg): + raise OSError("Media byte/storage budget reached") + output.write(chunk) + output.flush() + digest.update(chunk) + size += len(chunk) + if not size or (expected is not None and size != int(expected)): + raise ValueError("Incomplete or empty media download") + return {"url": url, "bytes": size, "sha256": digest.hexdigest(), + "method": "unchanged explicit page MP4 download"} + + +def acquire(row, cfg, directory, excluded_filenames): + maximum = cfg["collection"]["maximum_asset_seconds"] + if row["duration_seconds_from_page"] is None or not 0 < row["duration_seconds_from_page"] <= maximum: + raise ValueError("Missing/over-limit page duration") + if row["source_filename"] in excluded_filenames: + raise ValueError("Historical source filename exclusion") + media = directory / "source.mp4" + if row["mp4_urls"]: + url = row["mp4_urls"][0] + if any(filename in url for filename in excluded_filenames): + raise ValueError("Historical media URL filename exclusion") + acquisition = download_file(url, media, cfg) + expected_duration = row["duration_seconds_from_page"] + else: + master_url = f"https://www.dvidshub.net/video/{row['asset_id']}.m3u8" + master = fetch_text(master_url, cfg, directory / "master.m3u8") + choice = select_hls(master) + if any(filename in choice["url"] for filename in excluded_filenames): + raise ValueError("Historical HLS filename exclusion") + playlist = fetch_text(choice["url"], cfg, directory / "rendition.m3u8", media=True) + expected_duration = validate_playlist(playlist, choice["url"], maximum) + # A validated remote finite playlist is remuxed, with no codec encoding. + command = run_ffmpeg( + ["-protocol_whitelist", "https,tls,tcp", "-i", choice["url"], + "-map", "0:v:0?", "-map", "0:a:0", "-c", "copy", + "-t", str(maximum + 1), "-map_metadata", "-1"], media, cfg) + acquisition = {**choice, "method": "validated finite HLS stream-copy remux", + "master_url": master_url, "command": command, + "sha256": sha256(media), "bytes": media.stat().st_size} + metadata, duration, audio = probe(media) + if duration > maximum or abs(duration - expected_duration) > 2.0: + raise ValueError("Native duration exceeds limit or differs from full source") + native = directory / "audio.m4a" + command = run_ffmpeg( + ["-i", str(media), "-map", "0:a:0", "-vn", "-c:a", "copy", "-map_metadata", "-1"], + native, cfg, timeout=90) + audio_metadata, audio_duration, native_stream = probe(native) + if abs(audio_duration - duration) > 1.0: + raise ValueError("Extracted audio duration differs from source") + write_new(directory / "ffprobe_source.json", metadata) + write_new(directory / "ffprobe_audio.json", audio_metadata) + return { + "source": acquisition, "source_ref": _reference(directory / "source.mp4", cfg), + "source_sha256": sha256(media), "source_duration_seconds": duration, + "audio_ref": _reference(native, cfg), "audio_sha256": sha256(native), + "audio_duration_seconds": audio_duration, + "audio_codec": native_stream["codec_name"], + "native_sample_rate_hz": int(native_stream["sample_rate"]), + "channels": native_stream["channels"], + "source_audio_codec": audio["codec_name"], + "source_audio_bit_rate": audio.get("bit_rate", "unknown"), + "native_audio_command": command, "audio_reencoding_performed": False, + } + + +def _reference(path, cfg): + root = external_path(cfg["storage"]["data_root"]) + return cfg["storage"]["data_root"] + "/" + str(path.relative_to(root)) + + +def collect(candidates_ref, config_ref=CONFIG_REF, *, retry_metadata_failures=False): + cfg = load_config(config_ref) + data = external_path(cfg["storage"]["data_root"]) + artifacts = external_path(cfg["storage"]["artifact_root"]) + data.mkdir(parents=True, exist_ok=True) + artifacts.mkdir(parents=True, exist_ok=True) + excluded_ids, excluded_filenames, catalog_witnesses = historical_exclusions(cfg) + identity = artifacts / "contract.json" + if not identity.exists(): + write_new(identity, {"configuration": cfg, "config_sha256": sha256(resolve(config_ref)), + "tool_version": TOOL_VERSION, "catalog_witnesses": catalog_witnesses, + "started_at": now(), "source_code_sha256": sha256(Path(__file__))}) + for name, url in [("copyright", "https://www.dvidshub.net/about/copyright"), + ("faq", "https://www.dvidshub.net/about/faq")]: + fetch_text(url, cfg, artifacts / (name + ".html")) + else: + existing = json.loads(identity.read_text()) + if existing["configuration"] != cfg or existing["catalog_witnesses"] != catalog_witnesses: + raise ValueError("Resume contract/catalog witnesses changed") + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + source_copy = artifacts / ("collector-source-" + TOOL_VERSION + "-" + stamp + ".py") + with source_copy.open("xb") as handle: + handle.write(Path(__file__).read_bytes()) + write_new(artifacts / ("resume-" + stamp + ".json"), { + "recorded_at": now(), "tool_version": TOOL_VERSION, + "source_code_sha256": sha256(Path(__file__)), + "retained_source_copy": str(source_copy), + "retry_metadata_failures": retry_metadata_failures, + "prior_contract_source_code_sha256": existing["source_code_sha256"], + }) + candidates = json.loads(resolve(candidates_ref).read_text())["candidates"] + all_rows = [json.loads(p.read_text()) for p in sorted(data.glob("*/record.json"))] + done_ids = {r["asset_id"] for r in all_rows} + eligible = sum(r["status"] == "collected_pending_blind_review" for r in all_rows) + for candidate in candidates[:cfg["collection"]["maximum_candidate_assets"]]: + asset_id, url = str(candidate["asset_id"]), candidate["url"] + if not re.fullmatch(r"\d+", asset_id): + raise ValueError("Candidate ID must be numeric") + prior = [r for r in all_rows if r["asset_id"] == asset_id] + retryable = (retry_metadata_failures and prior and + not any(r["status"] == "collected_pending_blind_review" for r in prior) and + "Unexpected metadata/playlist content type: application/octet-stream" in + prior[-1].get("error", "")) + if asset_id in done_ids and not retryable: + continue + if eligible >= cfg["collection"]["target_collected_assets"]: + break + if available_bytes(cfg) < cfg["storage"]["reserved_workspace_bytes"] + 20_000_000: + break + directory = data / (f"{asset_id}-attempt-{len(prior)+1}" if prior else asset_id) + directory.mkdir(exist_ok=True) + if list(directory.iterdir()): + # An interruption may leave uncommitted bytes. Preserve them rather + # than silently overwrite or treat partial downloads as complete. + row = {"asset_id": asset_id, "url": url, "status": "interrupted_partial_quarantine", + "error": "Existing partial asset directory has no terminal record", + "recorded_at": now(), "media_access_performed": "unknown"} + else: + row = {"asset_id": asset_id, "url": url, "recorded_at": now(), + "discovery_classes": candidate.get("discovery_classes", []), + "media_access_performed": False, "attempt": len(prior) + 1, + "tool_version": TOOL_VERSION, "source_code_sha256": sha256(Path(__file__))} + try: + if asset_id in excluded_ids: + raise ValueError("Historical asset-ID exclusion; metadata/media not fetched") + html = fetch_text(url, cfg, directory / "page.html") + row.update(page_metadata(html, asset_id, url)) + row["page_sha256"] = sha256(directory / "page.html") + if not row["caption_classes"]: + raise ValueError("No proposed caption class in title/caption") + row["rights"] = rights_decision(row) + if row["rights"]["decision"] == "quarantine": + raise ValueError("Rights quarantine: " + "; ".join(row["rights"]["flags"])) + # The rights decision is persisted before any media request. + write_new(directory / "rights_decision.json", row["rights"]) + row["media_access_attempted"] = True + row["acquisition"] = acquire(row, cfg, directory, excluded_filenames) + row["media_access_performed"] = True + row["status"] = "collected_pending_blind_review" + eligible += 1 + except Exception as error: + row["status"] = "quarantine" + row["error"] = f"{type(error).__name__}: {error}" + row["media_access_performed"] = ((directory / "source.mp4").is_file() and + (directory / "source.mp4").stat().st_size > 0) + row["completed_at"] = now() + row["stored_bytes"] = sum(p.stat().st_size for p in directory.rglob("*") if p.is_file()) + write_new(directory / "record.json", row) + all_rows.append(row) + done_ids.add(asset_id) + print(json.dumps({"asset_id": asset_id, "status": row["status"], + "classes": row.get("caption_classes", []), + "error": row.get("error"), + "collected": eligible, "storage_bytes": storage_used(cfg)}), flush=True) + summary = summarize(cfg, all_rows) + checkpoint = artifacts / ("summary-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + ".json") + write_new(checkpoint, summary) + return summary + + +def summarize(cfg, rows): + collected = [r for r in rows if r["status"] == "collected_pending_blind_review"] + return { + "recorded_at": now(), "protocol_id": cfg["id"], "tool_version": TOOL_VERSION, + "candidate_outcomes": len(rows), "collected_assets": len(collected), + "unique_candidate_assets": len({r["asset_id"] for r in rows}), + "status_counts": dict(Counter(r["status"] for r in rows)), + "caption_class_asset_counts": dict(Counter(c for r in collected for c in r["caption_classes"])), + "total_source_seconds": sum(r["acquisition"]["source_duration_seconds"] for r in collected), + "storage_used_bytes": storage_used(cfg), "storage_limit_bytes": cfg["storage"]["maximum_total_bytes"], + "maximum_asset_seconds": cfg["collection"]["maximum_asset_seconds"], + "verified_independent_recording_groups": 0, "admitted_foreground_segments": 0, + "human_blind_review": "pending", "model_or_CAST_fitting_performed": False, + "experimental_roles": "unassigned", + "limitations": ["Caption class counts are not session counts or interval labels", + "Collected playable media are pending human audio/contamination and provenance admission", + "No source/model recognition or generalization result is established"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["collect", "status"]) + parser.add_argument("--config", default=CONFIG_REF) + parser.add_argument("--candidates") + parser.add_argument("--retry-metadata-failures", action="store_true") + args = parser.parse_args() + if args.action == "collect": + if not args.candidates: + parser.error("collect requires --candidates") + result = collect(args.candidates, args.config, + retry_metadata_failures=args.retry_metadata_failures) + else: + cfg = load_config(args.config) + data = external_path(cfg["storage"]["data_root"]) + rows = [json.loads(p.read_text()) for p in sorted(data.glob("*/record.json"))] + result = summarize(cfg, rows) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/abvid/signature/admission_annotations.py b/src/abvid/signature/admission_annotations.py new file mode 100644 index 0000000..2650c51 --- /dev/null +++ b/src/abvid/signature/admission_annotations.py @@ -0,0 +1,269 @@ +"""Import human P1 blind reviews and export bounded native-frame audio clips.""" +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +import re +import subprocess + +import numpy as np +import soundfile as sf + +from abvid.paths import resolve +from abvid.provenance.hashing import sha256 +from . import admission, admission_review +from .config import configuration, external_path +from .manifest import CONTAMINANTS, MOTION, REASONS + + +def validate_intervals(intervals, recordings): + fields = {"blind_id", "start_seconds", "end_seconds", "kind", + "audible_vehicle_count", "motion_state", "contaminants", "rejection_reasons"} + if not isinstance(intervals, list): + raise ValueError("Intervals must be an array") + by_id = {r["blind_id"]: r for r in recordings} + reviewed, accepted = set(), {} + trimming = configuration()["trimming"] + for interval in intervals: + if not isinstance(interval, dict) or set(interval) != fields: + raise ValueError("Unexpected interval fields") + key = interval["blind_id"] + if key not in by_id: + raise ValueError("Unknown blind recording") + lo, hi = interval["start_seconds"], interval["end_seconds"] + if any(type(x) not in (int, float) or not math.isfinite(x) for x in (lo, hi)): + raise ValueError("Interval times must be finite JSON numbers") + if not 0 <= lo < hi <= by_id[key]["duration_seconds"]: + raise ValueError("Interval exceeds loaded audio bounds") + kind, count = interval["kind"], interval["audible_vehicle_count"] + if kind not in {"foreground", "background", "rejected"}: + raise ValueError("Unknown interval decision") + if not (count == "unknown" or (type(count) is int and count in {0, 1, 2})): + raise ValueError("Invalid audible vehicle count") + if interval["motion_state"] not in MOTION: + raise ValueError("Invalid motion state") + for name, allowed in [("contaminants", CONTAMINANTS), ("rejection_reasons", REASONS)]: + values = interval[name] + if (not isinstance(values, list) or any(v not in allowed for v in values) or + len(set(values)) != len(values)): + raise ValueError("Invalid " + name) + if kind == "rejected": + if not interval["rejection_reasons"]: + raise ValueError("Rejected intervals require a reason") + else: + if interval["rejection_reasons"]: + raise ValueError("Accepted intervals cannot have rejection reasons") + if count != (1 if kind == "foreground" else 0): + raise ValueError("Foreground requires one audible vehicle; background requires zero") + if not trimming["minimum_" + kind + "_seconds"] <= hi - lo <= trimming["maximum_segment_seconds"]: + raise ValueError("Accepted duration outside frozen trimming rule") + if kind == "foreground" and set(interval["contaminants"]) & set(trimming["forbidden_foreground_contaminants"]): + raise ValueError("Contaminated foreground must be rejected") + for a, b in accepted.get(key, []): + if a < hi and lo < b: + raise ValueError("Accepted intervals overlap") + accepted.setdefault(key, []).append((lo, hi)) + reviewed.add(key) + if reviewed != set(by_id): + raise ValueError("Every recording requires an interval or logged rejection") + return intervals + + +def import_packet(annotation_path, packet_name, output_ref, config_ref=admission.CONFIG_REF): + cfg = admission.load_config(config_ref) + if not re.fullmatch(r"packet-\d{2}", packet_name): + raise ValueError("Invalid packet name") + artifacts = external_path(cfg["storage"]["artifact_root"]) + batch = json.loads((artifacts / "blind" / packet_name / "review_batch.json").read_text()) + private = json.loads((artifacts / "private" / (packet_name + ".json")).read_text()) + annotations = json.loads(Path(annotation_path).read_text()) + if set(annotations) != {"batch_sha256", "tool_version", "trimmer_id", "intervals"}: + raise ValueError("Unexpected annotation fields") + if (annotations["tool_version"] != admission_review.VERSION or + batch["tool_version"] != admission_review.VERSION or batch["protocol_id"] != cfg["id"]): + raise ValueError("Wrong annotation protocol/tool version") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", annotations["trimmer_id"]): + raise ValueError("Invalid reviewer ID") + commitment = {k: v for k, v in private.items() if k != "batch_sha256"} + batch_hash = admission_review.digest(batch) + if (batch["private_commitment_sha256"] != admission_review.digest(commitment) or + private["config_sha256"] != admission_review.digest(cfg) or + private["batch_sha256"] != batch_hash or annotations["batch_sha256"] != batch_hash): + raise ValueError("Annotation/private mapping/batch commitment mismatch") + rows = admission_review.records(cfg) + if private["input_records_sha256"] != admission_review.digest(rows): + raise ValueError("Collection changed after review packet was frozen") + public = {r["blind_id"]: r for r in batch["recordings"]} + if set(public) != set(private["mapping"]): + raise ValueError("Public/private recording identities differ") + by_id = {r["asset_id"]: r for r in rows} + for key, source in private["mapping"].items(): + row = by_id[source["asset_id"]] + if (public[key]["audio_sha256"] != source["audio_sha256"] or + row["acquisition"]["audio_sha256"] != source["audio_sha256"]): + raise ValueError("Source audio binding changed") + admission_review.validate_review_audio(row) + validate_intervals(annotations["intervals"], batch["recordings"]) + segments = [] + for interval in annotations["intervals"]: + source = private["mapping"][interval["blind_id"]] + segment_id = "segment-" + admission_review.digest( + [packet_name, annotations["trimmer_id"], interval])[:24] + segment = { + **interval, "segment_id": segment_id, "asset_id": source["asset_id"], + "audio_ref": source["audio_ref"], "audio_sha256": source["audio_sha256"], + "source_sha256": source["source_sha256"], + "native_sample_rate_hz": source["native_sample_rate_hz"], + "channels": source["channels"], "trimmer_id": annotations["trimmer_id"], + "experimental_role": "unassigned", "recording_session": "unknown", + "physical_vehicle_id": "unknown", "independence_verified": False, + "model_label": "unknown", "model_label_confidence": "unknown", + "distance_m": "unknown", "mic_type": "unknown", + "caption_class_candidates": by_id[source["asset_id"]]["caption_classes"], + "blind_review": True, + } + segments.append(segment) + result = { + "protocol_id": cfg["id"], "tool_version": admission_review.VERSION, + "recorded_at": admission.now(), "packet": packet_name, "batch_sha256": batch_hash, + "annotation_sha256": sha256(Path(annotation_path)), + "config_sha256": admission_review.digest(cfg), + "input_records_sha256": private["input_records_sha256"], "segments": segments, + "note": "Human interval review; source captions are separate candidates, not verified model/engine identities", + } + output = external_path(output_ref) + if not output.is_relative_to(artifacts): + raise ValueError("Annotation outputs must stay inside the allocated P1 artifact root") + preserved_input = output.with_name(output.name + ".input.json") + result["annotation_input_ref"] = output_ref + ".input.json" + encoded = json.dumps(result, indent=2).encode() + original_input = Path(annotation_path).read_bytes() + if len(encoded) + len(original_input) > admission.available_bytes(cfg): + raise OSError("Storage budget cannot fit imported annotations") + preserved_input.parent.mkdir(parents=True, exist_ok=True) + with preserved_input.open("xb") as handle: + handle.write(original_input) + admission.write_new(output, result) + return result + + +def export_native(imported_ref, output_ref, config_ref=admission.CONFIG_REF): + cfg = admission.load_config(config_ref) + imported = json.loads(resolve(imported_ref).read_text()) + if imported["protocol_id"] != cfg["id"] or imported["config_sha256"] != admission_review.digest(cfg): + raise ValueError("Wrong imported review contract") + source_rows = admission_review.records(cfg) + if imported["input_records_sha256"] != admission_review.digest(source_rows): + raise ValueError("Source collection changed") + by_id = {r["asset_id"]: r for r in source_rows} + original_input = resolve(imported["annotation_input_ref"]) + if sha256(original_input) != imported["annotation_sha256"]: + raise ValueError("Original human annotation input changed") + original_annotations = json.loads(original_input.read_text()) + if (original_annotations["batch_sha256"] != imported["batch_sha256"] or + original_annotations["tool_version"] != admission_review.VERSION or + len(original_annotations["intervals"]) != len(imported["segments"])): + raise ValueError("Original annotation binding/count changed") + packet = imported["packet"] + if not re.fullmatch(r"packet-\d{2}", packet): + raise ValueError("Invalid imported packet identity") + artifacts = external_path(cfg["storage"]["artifact_root"]) + batch = json.loads((artifacts / "blind" / packet / "review_batch.json").read_text()) + private = json.loads((artifacts / "private" / (packet + ".json")).read_text()) + commitment = {k: v for k, v in private.items() if k != "batch_sha256"} + if (admission_review.digest(batch) != imported["batch_sha256"] or + private["batch_sha256"] != imported["batch_sha256"] or + batch["private_commitment_sha256"] != admission_review.digest(commitment)): + raise ValueError("Imported review packet/private commitment changed") + interval_fields = {"blind_id", "start_seconds", "end_seconds", "kind", + "audible_vehicle_count", "motion_state", "contaminants", "rejection_reasons"} + for segment, original in zip(imported["segments"], original_annotations["intervals"]): + if {k: segment[k] for k in interval_fields} != original: + raise ValueError("Imported interval differs from the retained human annotation") + row = by_id.get(segment["asset_id"]) + mapped = private["mapping"].get(segment["blind_id"]) + if not mapped or any(segment[k] != mapped[k] for k in + ("asset_id", "audio_ref", "audio_sha256", "native_sample_rate_hz", "channels")): + raise ValueError("Imported interval changed its original blind source identity") + if not row or any(segment[k] != row["acquisition"][k] for k in + ("audio_ref", "audio_sha256", "native_sample_rate_hz", "channels")): + raise ValueError("Clip source differs from the admitted collection binding") + if segment["experimental_role"] != "unassigned": + raise ValueError("P1 admission cannot allocate experimental roles") + output = external_path(output_ref) + if not output.is_relative_to(artifacts) or output.exists(): + raise ValueError("Choose a new output directory inside the allocated P1 artifact root") + output.mkdir(parents=True) + clip_records, rejected, budget_deferred = [], [], [] + for segment in imported["segments"]: + if segment["kind"] == "rejected": + rejected.append(segment) + continue + source = resolve(segment["audio_ref"]) + if sha256(source) != segment["audio_sha256"]: + raise ValueError("Native source audio changed") + rate, channels = segment["native_sample_rate_hz"], segment["channels"] + lo = math.floor(segment["start_seconds"] * rate) + hi = math.ceil(segment["end_seconds"] * rate) + predicted_bytes = (hi - lo) * channels * 4 + 1024 + if predicted_bytes + 16384 > admission.available_bytes(cfg): + budget_deferred.append(segment["segment_id"]) + continue + target = output / (segment["segment_id"] + ".wav") + command = [ + "ffmpeg", "-nostdin", "-v", "error", "-n", "-i", str(source), "-map", "0:a:0", + "-vn", "-af", f"atrim=start_sample={lo}:end_sample={hi},asetpts=PTS-STARTPTS", + "-c:a", "pcm_f32le", "-map_metadata", "-1", "-fs", str(predicted_bytes), str(target), + ] + subprocess.run(command, capture_output=True, timeout=120, check=True) + info = sf.info(target) + if info.samplerate != rate or info.channels != channels or info.frames != hi - lo: + raise ValueError("Native-frame export metadata mismatch") + decoded = subprocess.run( + ["ffmpeg", "-nostdin", "-v", "error", "-i", str(source), "-map", "0:a:0", + "-f", "f32le", "-"], capture_output=True, timeout=120, check=True) + original = np.frombuffer(decoded.stdout, dtype=" len(original) or not np.array_equal(actual, original[lo:hi]): + raise ValueError("Export did not preserve decoded native samples") + if admission.storage_used(cfg) > cfg["storage"]["maximum_total_bytes"]: + raise OSError("Storage budget exceeded during export") + clip_records.append({ + "segment_id": segment["segment_id"], "asset_id": segment["asset_id"], + "source_audio_sha256": segment["audio_sha256"], + "start_frame": lo, "end_frame": hi, "sample_rate_hz": rate, "channels": channels, + "native_samples_equal": True, "clip_sha256": sha256(target), + "clip_ref": output_ref + "/" + target.name, + }) + result = { + "protocol_id": cfg["id"], "recorded_at": admission.now(), + "imported_review_sha256": sha256(resolve(imported_ref)), + "clips": clip_records, "rejections": rejected, "budget_deferred_segments": budget_deferred, + "experimental_roles": "unassigned", "model_or_CAST_fitting_performed": False, + } + admission.write_new(output / "export.json", result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["import", "export"]) + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--packet") + args = parser.parse_args() + if args.action == "import": + if not args.packet: + parser.error("import requires --packet") + result = import_packet(args.input, args.packet, args.output) + else: + result = export_native(args.input, args.output) + print(json.dumps({"segments": len(result.get("segments", [])), + "clips": len(result.get("clips", [])), + "budget_deferred": len(result.get("budget_deferred_segments", []))}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/abvid/signature/admission_reuse.py b/src/abvid/signature/admission_reuse.py new file mode 100644 index 0000000..991f0ac --- /dev/null +++ b/src/abvid/signature/admission_reuse.py @@ -0,0 +1,110 @@ +"""P1 label-blind audio reuse audit; no historical/protected audio is loaded.""" +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path + +import numpy as np + +from abvid.data.overlap import HOP, RATE, Index, landmarks +from abvid.paths import resolve +from abvid.provenance.hashing import sha256 +from . import admission, admission_review +from .config import configuration, external_path +from .admission_reuse_votes import match + + +def audit(config_ref=admission.CONFIG_REF): + cfg = admission.load_config(config_ref) + rows = admission_review.records(cfg) + spec = configuration()["fingerprint"] + settings = spec["settings"] + index = Index(settings) + windows, raw, statuses, best = {}, {}, [], {} + width = round(spec["window_seconds"] * RATE) + stride = round(spec["hop_seconds"] * RATE) + for row in sorted(rows, key=lambda r: r["asset_id"]): + aid = row["asset_id"] + audio = row["acquisition"] + path = admission_review.validate_review_audio(row) + key = audio["audio_sha256"] + if key in raw: + pair = tuple(sorted((raw[key], aid))) + best[pair] = {"left": pair[0], "right": pair[1], "kind": "exact_retained_audio_hash", + "correlation": 1.0, "overlap_seconds": None} + else: + raw[key] = aid + x = admission_review.display_audio(path, cfg["collection"]["maximum_asset_seconds"]).ravel() + if len(x) < width or not np.any(x): + statuses.append({"asset_id": aid, "status": "short_or_silent_unresolved", "windows": 0}) + continue + if index.parts: + index.freeze() + for result in match(index, aid, x, RATE): + if not result["accepted"]: + continue + previous, previous_start = windows[result["reference_id"]] + pair = tuple(sorted((previous, aid))) + candidate = { + "left": pair[0], "right": pair[1], "kind": "waveform_reuse_suggestion", + "correlation": result["correlation"], "overlap_seconds": spec["window_seconds"], + "reference_asset_id": previous, "reference_start_seconds": previous_start, + "query_asset_id": aid, "query_start_seconds": result["offset_s"], + "landmark_votes": result["votes"], + } + if pair not in best or (best[pair]["kind"] != "exact_retained_audio_hash" and + candidate["correlation"] > best[pair]["correlation"]): + best[pair] = candidate + starts = list(range(0, len(x) - width + 1, stride)) + if len(starts) > spec["maximum_windows_per_asset"]: + raise ValueError("Fingerprint window budget exceeded") + count = 0 + for start in starts: + window = x[start:start + width] + hashes, _ = landmarks(window, settings) + if not len(hashes): + continue + key = f"{aid}::{start}" + index.add(key, window, RATE) + windows[key] = (aid, start / RATE) + count += 1 + statuses.append({"asset_id": aid, "status": "fingerprinted" if count else "no_landmarks_unresolved", + "windows": count, "unindexed_windows": len(starts) - count}) + print(json.dumps({"fingerprinted_assets": len(statuses), "total_assets": len(rows), + "indexed_windows": len(windows), "reuse_suggestions": len(best)}), flush=True) + result = { + "recorded_at": admission.now(), "protocol_id": cfg["id"], + "scope": "P1 label-blind reuse suggestions only; no model or CAST fitting", + "tool_version": "signature-admission-reuse-3", + "input_audio_sha256": {r["asset_id"]: r["acquisition"]["audio_sha256"] for r in rows}, + "pairs": [best[k] for k in sorted(best)], "assets": statuses, + "settings": deepcopy(spec), "implementation_rate_hz": RATE, "implementation_hop": HOP, + "resampling": "ffmpeg display/fingerprint route to mono 8000 Hz float32; native audio retained", + "source_code_sha256": sha256(Path(__file__)), + "overlap_implementation_sha256": sha256(resolve("repo:src/abvid/data/overlap.py")), + "vote_implementation_sha256": sha256(resolve("repo:src/abvid/signature/admission_reuse_votes.py")), + "correlation_implementation_sha256": sha256(resolve("repo:src/abvid/signature/admission_reuse_correlation.py")), + "voting": "exact packed integer counts; parity with the original reference-ID/offset sort verified", + "correlation": "FFT winner nomination with original direct-dot scores; exact-score parity verified; ill-conditioned cases use original full lag scan", + "independence_certified": False, "verified_independent_recording_groups": 0, + "protected_audio_accessed": False, + "limitations": [ + "No-match is not evidence of independent recording sessions or physical vehicles", + "Periodic signals may conservatively link distinct recordings", + "Codec, edits, mixtures, time stretch and overlaps shorter than four seconds may be missed", + "Suggested reuse must be retained in admission grouping before any experimental role allocation", + ], + } + output = external_path(cfg["storage"]["artifact_root"]) / "reuse_audit.json" + encoded = json.dumps(result, indent=2, allow_nan=False).encode() + if len(encoded) > admission.available_bytes(cfg): + raise OSError("Storage budget cannot fit reuse record") + admission.write_new(output, result) + return result + + +if __name__ == "__main__": + result = audit() + print(json.dumps({"assets": len(result["assets"]), "reuse_suggestions": len(result["pairs"]), + "verified_independent_recording_groups": 0}, indent=2)) diff --git a/src/abvid/signature/admission_reuse_correlation.py b/src/abvid/signature/admission_reuse_correlation.py new file mode 100644 index 0000000..68f3981 --- /dev/null +++ b/src/abvid/signature/admission_reuse_correlation.py @@ -0,0 +1,43 @@ +"""Accelerate local lag search while returning original direct-dot scores. + +FFT scores nominate possible winning lags. Ambiguous/ill-conditioned cases use +the original full search; nominated lags are scored with its exact arithmetic. +""" +import numpy as np +from scipy.signal import correlate + +from abvid.data.overlap import correlation as original + + +def correlation(reference, query, start, lag): + n = len(reference) + first = max(0, start - lag) + last = min(len(query) - n, start + lag) + if not n or first > last: + return 0.0 + ref = np.asarray(reference, dtype=np.float64) + window = np.asarray(query[first:last + n], dtype=np.float64) + reference_energy = np.dot(ref, ref) + if not reference_energy > 0: + return original(reference, query, start, lag) + cumulative = np.concatenate(([0.0], np.cumsum(window * window, dtype=np.float64))) + energy = cumulative[n:] - cumulative[:-n] + # A conservative cancellation guard sends quiet windows next to strong + # transients back through the original arithmetic. + error_bound = np.finfo(np.float64).eps * len(window) * cumulative[-1] + if (not np.isfinite(cumulative[-1]) or + np.any(energy <= max(error_bound * 1e8, np.finfo(np.float64).tiny))): + return original(reference, query, start, lag) + numerator = correlate(window, ref, mode="valid", method="fft") + score_estimates = np.abs(numerator) / np.sqrt(reference_energy * energy) + if not np.isfinite(score_estimates).all(): + return original(reference, query, start, lag) + candidates = np.flatnonzero(score_estimates >= score_estimates.max() - 1e-7) + best = 0.0 + for offset in candidates: + a = first + int(offset) + q = query[a:a + n] + denominator = np.sqrt(np.dot(reference, reference) * np.dot(q, q)) + if denominator > 0: + best = max(best, abs(float(np.dot(reference, q) / denominator))) + return best diff --git a/src/abvid/signature/admission_reuse_votes.py b/src/abvid/signature/admission_reuse_votes.py new file mode 100644 index 0000000..4b47839 --- /dev/null +++ b/src/abvid/signature/admission_reuse_votes.py @@ -0,0 +1,56 @@ +"""Exact packed-integer voting for the unchanged landmark reuse criterion.""" +import numpy as np + +from abvid.data.overlap import HOP, RATE, landmarks, mono8k +from .admission_reuse_correlation import correlation + + +def votes(ref, reference_time, lo, hi, query_time): + if not len(reference_time) or not len(query_time): + return np.zeros((0, 2), np.int64), np.zeros(0, np.int64) + minimum = int(reference_time.min()) - int(query_time.max()) + maximum = int(reference_time.max()) - int(query_time.min()) + span = maximum - minimum + 1 + largest = int(ref.max(initial=0)) * span + span - 1 + dtype = np.uint32 if largest <= np.iinfo(np.uint32).max else np.uint64 + if largest > np.iinfo(np.int64).max: + raise ValueError("Packed vote key would exceed exact integer range") + parts = [ + (ref[a:b] * span + reference_time[a:b] - int(t) - minimum).astype(dtype) + for a, b, t in zip(lo, hi, query_time) if b > a + ] + if not parts: + return np.zeros((0, 2), np.int64), np.zeros(0, np.int64) + packed, counts = np.unique(np.concatenate(parts), return_counts=True) + packed = packed.astype(np.int64) + keys = np.column_stack((packed // span, packed % span + minimum)) + return keys, counts + + +def match(index, query_id, audio, rate): + settings = index.settings + x = mono8k(audio, rate) + hashes, times = landmarks(x, settings) + lo = np.searchsorted(index.hash, hashes, "left") + hi = np.searchsorted(index.hash, hashes, "right") + keys, counts = votes(index.ref, index.time, lo, hi, times) + if not len(keys): + return [] + # This is the same integer/float criterion as the original per-key loop. + fractions = counts / np.maximum(index.counts[keys[:, 0]], 1) + selected = (counts >= settings["min_votes"]) & (fractions >= settings["min_vote_fraction"]) + rows = [] + for (r, offset), count, fraction in zip(keys[selected], counts[selected], fractions[selected]): + start = int(-offset) * HOP + score = correlation(index.audio[r], x, start, settings["lag_search_samples"]) + rows.append({ + "query_id": query_id, "reference_id": index.ids[r], "offset_s": start / RATE, + "votes": int(count), "vote_fraction": float(fraction), "correlation": score, + "accepted": score >= settings["min_correlation"], + }) + best = {} + for row in rows: + key = row["reference_id"] + if key not in best or (row["correlation"], row["votes"]) > (best[key]["correlation"], best[key]["votes"]): + best[key] = row + return sorted(best.values(), key=lambda r: (-r["correlation"], r["reference_id"])) diff --git a/src/abvid/signature/admission_review.py b/src/abvid/signature/admission_review.py new file mode 100644 index 0000000..598a720 --- /dev/null +++ b/src/abvid/signature/admission_review.py @@ -0,0 +1,273 @@ +"""Caption-hidden P1 review packets served offline without duplicating audio.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import hashlib +import json +import math +from pathlib import Path +import re +import secrets +import subprocess +from urllib.parse import unquote, urlsplit + +import numpy as np + +from abvid.paths import resolve +from abvid.provenance.hashing import canonical, sha256 +from . import admission +from .config import external_path, configuration +from .trim import _spectrum_svg, _wave_svg +from . import review_ui + +VERSION = "signature-admission-review-1" + + +def digest(value): + return hashlib.sha256(canonical(value)).hexdigest() + + +def display_audio(path, maximum_seconds): + result = subprocess.run( + ["ffmpeg", "-nostdin", "-v", "error", "-i", str(path), "-map", "0:a:0", + "-vn", "-ac", "1", "-ar", "8000", "-f", "f32le", "-"], + capture_output=True, timeout=60, check=True) + samples = np.frombuffer(result.stdout, dtype=" (maximum_seconds + 1) * 8000 or not np.isfinite(samples).all(): + raise ValueError("Invalid or overlong display waveform") + return samples.reshape(-1, 1) + + +def records(cfg): + root = external_path(cfg["storage"]["data_root"]) + rows = [json.loads(p.read_text()) for p in sorted(root.glob("*/record.json"))] + result = [r for r in rows if r["status"] == "collected_pending_blind_review"] + if len({r["asset_id"] for r in result}) != len(result): + raise ValueError("Duplicate collected identities") + return result + + +def validate_review_audio(row): + audio = row["acquisition"] + path = resolve(audio["audio_ref"]) + if sha256(path) != audio["audio_sha256"]: + raise ValueError("Collected native audio hash changed") + metadata, duration, stream = admission.probe(path) + # The collector strips source tags. Verify that no title/artist/source names + # were carried into either container or stream metadata before blind review. + allowed = { + "major_brand": {"M4A "}, "minor_version": {"512"}, + "compatible_brands": {"M4A isomiso2"}, "language": {"und"}, + "handler_name": {"SoundHandler"}, + } + for section in [metadata["format"], *metadata["streams"]]: + for key, value in section.get("tags", {}).items(): + if key == "encoder" and re.fullmatch(r"Lavf[0-9.]+", value): + continue + if key not in allowed or value not in allowed[key]: + raise ValueError("Non-generic embedded metadata would unblind audio") + if (int(stream["sample_rate"]) != audio["native_sample_rate_hz"] or + stream["channels"] != audio["channels"] or + abs(duration - audio["audio_duration_seconds"]) > 0.1): + raise ValueError("Native review audio metadata changed") + return path + + +def card(key, ordinal, duration, x): + # Controls deliberately match the Chrome-confirmed P0 viewer. P1's different + # protocol/version/batch binding is supplied separately. + if not re.fullmatch(r"review-[0-9a-f]{24}", key): + raise ValueError("Invalid opaque review identity") + section = (review_ui.CARD.replace("__BLIND_ID__", key) + .replace("__AUDIO_ROUTE__", f"/audio/{key}.m4a") + .replace("__ORDINAL__", str(ordinal)) + .replace("__DURATION__", f"{duration:.3f}")) + insertion = section.index('
') + return section[:insertion] + _wave_svg(x) + _spectrum_svg(x, 8000) + section[insertion:] + + +def make_packets(config_ref=admission.CONFIG_REF, batch_size=8): + cfg = admission.load_config(config_ref) + if type(batch_size) is not int or not 1 <= batch_size <= 12: + raise ValueError("Review batches must contain one to twelve recordings") + root = external_path(cfg["storage"]["artifact_root"]) + blind = root / "blind" + private = root / "private" + if blind.exists() or private.exists(): + raise FileExistsError("Review packets are immutable; preserve existing handoff") + rows = records(cfg) + for row in rows: + validate_review_audio(row) + ordering = np.random.default_rng(17).permutation(len(rows)) + viewer_hash = review_ui.SOURCE_TEMPLATE_SHA256 + blind.mkdir() + private.mkdir() + manifest_hash = digest(rows) + public_summary, private_summary = [], [] + audio_routes = {} + for start in range(0, len(rows), batch_size): + name = f"packet-{start // batch_size + 1:02d}" + packet = blind / name + packet.mkdir() + public, mapping, cards = [], {}, [] + for offset, index in enumerate(ordering[start:start + batch_size]): + row = rows[int(index)] + audio = row["acquisition"] + key = "review-" + secrets.token_hex(12) + path = validate_review_audio(row) + x = display_audio(path, cfg["collection"]["maximum_asset_seconds"]) + cards.append(card(key, offset + 1, audio["audio_duration_seconds"], x)) + public.append({ + "blind_id": key, "audio_file": "/audio/" + key + ".m4a", + "duration_seconds": audio["audio_duration_seconds"], + "audio_sha256": audio["audio_sha256"], + }) + mapping[key] = { + "asset_id": row["asset_id"], "audio_ref": audio["audio_ref"], + "audio_sha256": audio["audio_sha256"], "source_sha256": audio["source_sha256"], + "native_sample_rate_hz": audio["native_sample_rate_hz"], + "channels": audio["channels"], + } + audio_routes[key] = mapping[key] + private_data = { + "protocol_id": cfg["id"], "config_sha256": digest(cfg), + "input_records_sha256": manifest_hash, "mapping": mapping, + "nonce": secrets.token_hex(16), + } + batch = { + "protocol_id": cfg["id"], "tool_version": VERSION, "recordings": public, + "private_commitment_sha256": digest(private_data), + } + batch_hash = digest(batch) + meta = {"batch_sha256": batch_hash, "tool_version": VERSION} + script = (review_ui.SCRIPT.replace("__META_JSON__", json.dumps(meta)) + .replace("__TRIMMING_JSON__", json.dumps(configuration()["trimming"]))) + document = review_ui.HEAD + "".join(cards) + script + disclaimer = ("The appearance of U.S. Department of War (DoW) visual information " + "does not imply or constitute DoW endorsement.") + document = document.replace("" diff --git a/src/abvid/signature/trim.py b/src/abvid/signature/trim.py new file mode 100644 index 0000000..d644c68 --- /dev/null +++ b/src/abvid/signature/trim.py @@ -0,0 +1,226 @@ +"""Offline caption-hidden review packets and native-frame interval export.""" +from __future__ import annotations + +import hashlib +import json +import math +import secrets +from copy import deepcopy +from html import escape + +import numpy as np +import soundfile as sf + +from abvid.data.overlap import mono8k, spectrogram +from abvid.provenance.hashing import canonical, sha256 + +from . import PROTOCOL_ID, TOOL_VERSION +from .audio import read_asset +from .config import external_path +from .manifest import CONTAMINANTS, MOTION, REASONS, validate + + +def _digest(value): + return hashlib.sha256(canonical(value)).hexdigest() + + +def _write_json_new(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as stream: + stream.write(json.dumps(value, indent=2, allow_nan=False) + "\n") + + +def _wave_svg(x): + mono = x.mean(axis=1) + bounds = np.linspace(0, len(mono), min(800, len(mono)) + 1, dtype=int) + peak = max(float(np.max(np.abs(mono))), 1e-12) + strokes = [] + for i, (lo, hi) in enumerate(zip(bounds[:-1], bounds[1:])): + part = mono[lo:hi] + a, b = 50 - 47 * part.max() / peak, 50 - 47 * part.min() / peak + strokes.append(f"M{i},{a:.2f}V{b:.2f}") + return (f'' + f'') + + +def _spectrum_svg(x, rate): + s = spectrogram(mono8k(x, rate)) + if not len(s): + return "

Insufficient duration for a spectrogram.

" + # Decimation affects the display only. Audio, trim times and deduplication + # use their specified native/fingerprint routes without these display bins. + ti = np.linspace(0, len(s) - 1, min(256, len(s)), dtype=int) + fi = np.linspace(0, s.shape[1] - 1, 64, dtype=int) + pixels = s[np.ix_(ti, fi)] + top = float(s.max()) + shades = np.clip((pixels - (top - 80)) / 80, 0, 1) + rects = [] + for t in range(len(ti)): + for f in range(len(fi)): + c = round(255 * shades[t, f]) + rects.append(f'') + return (f'' + + "".join(rects) + "") + + +def make_packet(manifest, config, blind_reference, private_reference, *, seed=17): + validate(manifest, config) + blind = external_path(blind_reference) + private = external_path(private_reference) + if private == blind or private.is_relative_to(blind) or blind.is_relative_to(private): + raise ValueError("Private mapping must be outside the blind review directory") + if blind.exists() or private.exists(): + raise FileExistsError("Review outputs are immutable; choose new output paths") + # Verify all inputs before publishing any packet. + loaded = [(a, *read_asset(a, config)) for a in manifest["assets"]] + ordering = np.random.default_rng(seed).permutation(len(loaded)) + public, mapping, cards = [], {}, [] + blind.mkdir(parents=True) + for ordinal, i in enumerate(ordering): + asset, x, rate = loaded[int(i)] + key = "review-" + _digest([seed, ordinal, asset["raw_sha256"]])[:16] + name = key + ".wav" + # Re-encoding drops source filenames and embedded title/artist tags. + sf.write(blind / name, x, rate, subtype="FLOAT") + public.append({"blind_id": key, "audio_file": name, "duration_seconds": len(x) / rate, + "audio_sha256": sha256(blind / name)}) + mapping[key] = asset["asset_id"] + cards.append(f'

Recording {ordinal+1}

' + f'' + f'

Duration {len(x)/rate:.3f} s. Waveform uses relative display amplitude. ' + 'Spectrogram spans 0 to 4 kHz and 80 dB below the recording maximum.

' + + _wave_svg(x) + _spectrum_svg(x, rate) + + '
' + '' + '' + '
' + '' + '' + '' + '
Contaminants' + "".join(f'' for c in CONTAMINANTS) + '
' + '' + '
    ') + private_data = {"manifest_sha256": _digest(manifest), "protocol_sha256": _digest(config), + "mapping": mapping, "nonce": secrets.token_hex(16)} + batch = {"protocol_id": PROTOCOL_ID, "tool_version": TOOL_VERSION, "recordings": public, + "private_commitment_sha256": _digest(private_data)} + batch_hash = _digest(batch) + _write_json_new(blind / "review_batch.json", batch) + _write_json_new(private, {**private_data, "batch_sha256": batch_hash}) + payload = json.dumps({"batch_sha256": batch_hash, "tool_version": TOOL_VERSION}, allow_nan=False).replace("<", "\\u003c") + document = ''' +Acoustic recording review +

    Acoustic recording review

    Review audio without captions. Record separate vehicle and background intervals, and retain rejected intervals with reasons. Save annotations when every recording has been reviewed.

    +

    +''' + "".join(cards) + '''' + (blind / "index.html").write_text(document) + return {"blind_reference": blind_reference, "private_reference": private_reference, + "batch_sha256": batch_hash, "recordings": len(public)} + + +def import_annotations(manifest, config, batch, private, annotations, *, role="development"): + validate(manifest, config) + if private.get("manifest_sha256") != _digest(manifest) or private.get("protocol_sha256") != _digest(config): + raise ValueError("Private mapping belongs to different inputs") + commitment = {key: value for key, value in private.items() if key != "batch_sha256"} + if batch.get("private_commitment_sha256") != _digest(commitment): + raise ValueError("Private mapping commitment mismatch") + if private.get("batch_sha256") != _digest(batch) or annotations.get("batch_sha256") != _digest(batch): + raise ValueError("Annotation/review batch hash mismatch") + if set(annotations) != {"batch_sha256", "tool_version", "trimmer_id", "intervals"}: + raise ValueError("Unexpected annotation fields") + if annotations["tool_version"] != TOOL_VERSION: + raise ValueError("Changed annotation tool version") + public_ids = {r["blind_id"] for r in batch["recordings"]} + if batch.get("protocol_id") != PROTOCOL_ID or batch.get("tool_version") != TOOL_VERSION: + raise ValueError("Changed review protocol/tool version") + if public_ids != set(private["mapping"]): + raise ValueError("Public/private recording identities differ") + result = deepcopy(manifest) + interval_keys = {"blind_id", "start_seconds", "end_seconds", "kind", "audible_vehicle_count", + "motion_state", "contaminants", "rejection_reasons"} + reviewed = set() + if not isinstance(annotations["intervals"], list): + raise ValueError("Annotation intervals must be an array") + for interval in annotations["intervals"]: + if not isinstance(interval, dict) or set(interval) != interval_keys or interval["blind_id"] not in public_ids: + raise ValueError("Invalid blind interval identity or fields") + reviewed.add(interval["blind_id"]) + row = {key: deepcopy(value) for key, value in interval.items() if key != "blind_id"} + row.update({"segment_id": "segment-" + _digest([interval, annotations["trimmer_id"]])[:20], + "asset_id": private["mapping"][interval["blind_id"]], + "role": "quarantine" if interval["kind"] == "rejected" else role, + "trimmer_id": annotations["trimmer_id"], "tool_version": TOOL_VERSION, + "blind_review": True, "distance_m": "unknown", "mic_type": "unknown"}) + result["segments"].append(row) + if reviewed != public_ids: + raise ValueError("Every recording requires a reviewed interval or a logged rejection") + validate(result, config) + return result + + +def export_segments(manifest, config, output_reference): + validate(manifest, config) + output = external_path(output_reference) + if output.exists(): + raise FileExistsError("Segment output is immutable; choose a new directory") + assets = {a["asset_id"]: a for a in manifest["assets"]} + loaded = {aid: read_asset(a, config) for aid, a in assets.items()} + records = [] + output.mkdir(parents=True) + for segment in manifest["segments"]: + if segment["kind"] == "rejected": + continue + aid = segment["asset_id"] + x, rate = loaded[aid] + start = math.floor(segment["start_seconds"] * rate) + end = math.ceil(segment["end_seconds"] * rate) + if not 0 <= start < end <= len(x): + raise ValueError("Rounded interval exceeds raw audio") + path = output / (segment["segment_id"] + ".wav") + sf.write(path, x[start:end], rate, subtype="FLOAT") + records.append({"segment_id": segment["segment_id"], "asset_id": aid, + "raw_sha256": assets[aid]["raw_sha256"], "start_frame": start, + "end_frame": end, "sample_rate_hz": rate, "channels": x.shape[1], + "clip_sha256": sha256(path), "clip_ref": output_reference + "/" + path.name}) + result = {"manifest_sha256": _digest(manifest), "protocol_sha256": _digest(config), + "clips": records, "rejections": [s for s in manifest["segments"] if s["kind"] == "rejected"]} + _write_json_new(output / "export.json", result) + return result diff --git a/src/abvid/signature/validation.py b/src/abvid/signature/validation.py new file mode 100644 index 0000000..6b54b56 --- /dev/null +++ b/src/abvid/signature/validation.py @@ -0,0 +1,143 @@ +"""Reproducible P0 preparation checks, using artificial fixtures only.""" +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import math +import platform +import subprocess +from copy import deepcopy +from datetime import datetime, timezone + +import numpy as np +import soundfile as sf + +from abvid.paths import resolve, roots +from abvid.provenance.hashing import canonical, sha256 + +from . import PROTOCOL_ID, TOOL_VERSION +from .audio import read_asset +from .config import DEFAULT_CONFIG, external_path, load +from .dedup import audit_manifest, with_reuse_links +from .fixtures import create_bundle +from .manifest import connected_groups, readiness, validate +from .trim import _write_json_new, export_segments, import_annotations, make_packet + + +def _digest(value): + return hashlib.sha256(canonical(value)).hexdigest() + + +def run(fixture_reference, output_reference, *, config_reference=DEFAULT_CONFIG): + """Run the synthetic oracle through the same public preparation operations. + + Interval locations are known from fixture generation, not an audio detector + or a human-reviewed scientific observation. No representation is fitted. + """ + config = load(config_reference) + output = external_path(output_reference) + if output.exists(): + raise FileExistsError("Validation output is immutable; choose a new directory") + if external_path(fixture_reference).exists(): + raise FileExistsError("Validation fixtures are immutable; choose a new directory") + manifest = create_bundle(fixture_reference, config) + packet = make_packet(manifest, config, output_reference + "/blind", + output_reference + "/private/mapping.json", seed=17) + batch = json.loads((output / "blind/review_batch.json").read_text()) + private = json.loads((output / "private/mapping.json").read_text()) + recipes = { + "fixture-a": [(2.00003, 6.00003, "foreground", None), (0, 2, "background", None)], + "fixture-b": [(2.00003, 6.00003, "foreground", None), (0, 2, "background", None)], + "fixture-crop": [(1.00003, 5.00003, "foreground", None), (7, 9, "background", None)], + "fixture-resampled": [(2.00003, 6.00003, "foreground", None), (0, 2, "background", None)], + "fixture-silence": [(0, 5, "rejected", "no_vehicle")], + "fixture-short": [(0, 1, "rejected", "too_short")], + } + intervals = [] + for blind_id, aid in private["mapping"].items(): + for start, end, kind, reason in recipes[aid]: + intervals.append({"blind_id": blind_id, "start_seconds": start, "end_seconds": end, + "kind": kind, "audible_vehicle_count": "unknown" if reason else int(kind == "foreground"), + "motion_state": "unknown", "contaminants": [], + "rejection_reasons": [reason] if reason else []}) + annotations = {"batch_sha256": packet["batch_sha256"], "tool_version": TOOL_VERSION, + "trimmer_id": "synthetic-oracle", "intervals": intervals} + _write_json_new(output / "oracle_annotations.json", annotations) + imported = import_annotations(manifest, config, batch, private, annotations) + # These arbitrary names test manifest bookkeeping, not vehicle identity. + for segment in imported["segments"]: + if segment["kind"] == "foreground": + label = "artificial-b" if segment["asset_id"] == "fixture-b" else "artificial-a" + imported["labels"].append({"segment_id": segment["segment_id"], "caption_model": label, + "caption_variant": "unknown", "confidence": "confirmed", "L1": "unknown", + "L2": "unknown", "L3": "unknown", "label_evidence": "Fixture generator seed; no vehicle claim"}) + validate(imported, config) + reuse = audit_manifest(imported, config) + grouped = with_reuse_links(imported, reuse, config) + _write_json_new(output / "reuse_audit.json", reuse) + _write_json_new(output / "manifest.json", grouped) + exported = export_segments(grouped, config, output_reference + "/native_clips") + by_id = {a["asset_id"]: a for a in grouped["assets"]} + by_segment = {s["segment_id"]: s for s in grouped["segments"]} + frame_checks = [] + for clip in exported["clips"]: + raw, rate = read_asset(by_id[clip["asset_id"]], config) + segment = by_segment[clip["segment_id"]] + expected_start = math.floor(segment["start_seconds"] * rate) + expected_end = math.ceil(segment["end_seconds"] * rate) + actual, actual_rate = sf.read(resolve(clip["clip_ref"]), dtype="float32", always_2d=True) + passed = (actual_rate == rate and clip["start_frame"] == expected_start and + clip["end_frame"] == expected_end and np.array_equal(actual, raw[expected_start:expected_end])) + if not passed: + raise AssertionError("Native samples/rounding were not preserved") + frame_checks.append({"segment_id": clip["segment_id"], "start_frame": expected_start, + "end_frame": expected_end, "native_samples_equal": passed}) + pairs = {frozenset((p["left"], p["right"])) for p in reuse["pairs"]} + required = [frozenset(("fixture-a", "fixture-crop")), frozenset(("fixture-a", "fixture-resampled"))] + if not all(p in pairs for p in required) or any("fixture-b" in p for p in pairs): + raise AssertionError("Synthetic reuse positive or independent negative failed") + changed = deepcopy(imported) + for asset in changed["assets"]: + asset["title"] = "Deliberately changed hidden model name" + if audit_manifest(changed, config)["pairs"] != reuse["pairs"]: + raise AssertionError("Caption changed label-blind reuse matching") + groups = connected_groups(grouped) + reused_ids = ["fixture-a", "fixture-crop", "fixture-resampled", "fixture-short"] + if len({groups[aid] for aid in reused_ids}) != 1: + raise AssertionError("Declared or detected reuse was split") + if len(exported["rejections"]) != 2: + raise AssertionError("Rejected intervals were lost") + blind_text = (output / "blind/index.html").read_text() + (output / "blind/review_batch.json").read_text() + if any(a[k] in blind_text for a in manifest["assets"] for k in ["asset_id", "title", "recording_session"]): + raise AssertionError("Source metadata leaked into the reviewer packet") + version = subprocess.run(["git", "rev-parse", "HEAD"], cwd=roots()["repo"], + check=True, capture_output=True, text=True).stdout.strip() + status = subprocess.run(["git", "status", "--porcelain"], cwd=roots()["repo"], + check=True, capture_output=True, text=True).stdout + code_hashes = {"repo:" + str(p.relative_to(roots()["repo"])): sha256(p) + for p in sorted((roots()["repo"] / "src/abvid/signature").glob("*.py"))} + report = {"protocol_id": PROTOCOL_ID, "tool_version": TOOL_VERSION, + "created_at": datetime.now(timezone.utc).isoformat(), "scope": "synthetic_preparation_validation_only", + "fixture_ref": fixture_reference, "output_ref": output_reference, + "seed": 17, "independent_seed": 29, "commit": version, "working_tree_dirty": bool(status), + "environment": {"python": platform.python_version(), "platform": platform.platform(), + **{name: importlib.metadata.version(name) for name in ["numpy", "scipy", "soundfile"]}}, + "configuration": config, "config_file_sha256": sha256(resolve(config_reference)), + "schema_file_sha256": sha256(resolve(config["storage"]["manifest_schema"])), + "implementation_sha256": code_hashes, + "raw_hashes": {a["asset_id"]: a["raw_sha256"] for a in manifest["assets"]}, + "manifest_sha256": _digest(grouped), "native_export_checks": frame_checks, + "checks": {"native_samples_and_rounding": True, "caption_hidden": True, + "label_blind_reuse": True, "crop_gain_polarity_reuse": True, + "resampling_reuse": True, "independent_negative": True, + "rejection_log_retained": True, "shared_origin_group_preserved": True}, + "readiness": readiness(grouped, config), + "limitations": ["Synthetic oracle intervals do not validate human trimming accuracy", + "No real audio, representations, model fits or scientific evaluation", + "Browser interaction is a separately recorded functional check", + "Negative reuse detection does not certify recording independence"]} + _write_json_new(output / "validation.json", report) + return {"validation_ref": output_reference + "/validation.json", "passed": True, + "assets": len(manifest["assets"]), "exported_clips": len(exported["clips"]), + "rejections": len(exported["rejections"]), "groups": len(set(groups.values()))} diff --git a/tests/test_signature_admission.py b/tests/test_signature_admission.py new file mode 100644 index 0000000..505b86e --- /dev/null +++ b/tests/test_signature_admission.py @@ -0,0 +1,227 @@ +"""Offline boundary checks for the separately scoped real-media collector.""" +import json + +import pytest +import numpy as np + +from abvid.signature import admission +from abvid.signature.admission_annotations import validate_intervals +from abvid.signature import admission_review +from abvid.signature.admission_reuse_votes import votes as packed_votes +from abvid.signature.admission_reuse_correlation import correlation as accelerated_correlation +from abvid.data.overlap import correlation as original_correlation + + +def test_canonical_authorization_rejects_budget_drift(tmp_path, monkeypatch): + original = admission.load_config() + assert original["storage"]["maximum_total_bytes"] == 5_000_000_000 + assert original["collection"]["maximum_asset_seconds"] == 1200 + original["authorization"]["model_fitting"] = True + changed = tmp_path / "changed.json" + changed.write_text(json.dumps(original)) + monkeypatch.setattr(admission, "resolve", lambda reference: changed) + with pytest.raises(ValueError, match="Changed P1"): + admission.load_config() + + +def test_network_guard_refuses_private_or_unrelated_urls(): + for url in [ + "http://www.dvidshub.net/video/1", + "https://127.0.0.1/video/1", + "https://www.dvidshub.net@evil.test/video/1", + "https://d34w7g4gy10iej.cloudfront.net:443/video/example.mp4", + "file:///etc/passwd", + ]: + with pytest.raises(ValueError): + admission.check_url(url, media=True) + admission.check_url("https://d34w7g4gy10iej.cloudfront.net/video/example.mp4", media=True) + with pytest.raises(ValueError): + admission.check_url("https://d34w7g4gy10iej.cloudfront.net/private/example.mp4", media=True) + + +def test_finite_playlist_does_not_trim_an_overlong_asset(): + playlist = "#EXTM3U\n#EXTINF:1199,\na.ts\n#EXTINF:2,\nb.ts\n#EXT-X-ENDLIST\n" + with pytest.raises(ValueError, match="exceeds"): + admission.validate_playlist(playlist, "https://d34w7g4gy10iej.cloudfront.net/video/a/index.m3u8", 1200) + for unsafe in [ + playlist.replace("1199", "nan"), + playlist.replace("1199", "2").replace("a.ts", "https://evil.test/a.ts"), + playlist.replace("1199", "2").replace("#EXT-X-ENDLIST", "#EXT-X-KEY:METHOD=AES-128"), + playlist.replace("1199", "2").replace("#EXT-X-ENDLIST", ""), + ]: + with pytest.raises(ValueError): + admission.validate_playlist(unsafe, "https://d34w7g4gy10iej.cloudfront.net/video/a/index.m3u8", 1200) + assert admission.validate_playlist( + playlist.replace("1199", "1198"), + "https://d34w7g4gy10iej.cloudfront.net/video/a/index.m3u8", 1200 + ) == 1200 + + +def test_rendition_selection_avoids_lower_quality_audio_only_and_above_720p(): + base = "https://d34w7g4gy10iej.cloudfront.net/video/a/" + master = ( + "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=64000,CODECS=mp4a\n" + base + "audio.m3u8\n" + "#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=1280x720\n" + base + "720.m3u8\n" + "#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080\n" + base + "1080.m3u8\n" + ) + assert admission.select_hls(master)["url"] == base + "720.m3u8" + + +def test_rights_banner_does_not_override_asset_restriction(): + row = { + "caption": "Editorial use only. Leopard tank training.", + "copyright_info": "DVIDS copyright notice", + "unit": "Natochannel", + "public_domain_banner": True, + } + assert admission.rights_decision(row)["decision"] == "quarantine" + row.update(caption="M1151 driver training.", unit="128th Mobile Public Affairs Detachment") + assert admission.rights_decision(row)["decision"] == "collect_for_local_noncommercial_research" + row["public_domain_banner"] = False + assert admission.rights_decision(row)["decision"] == "quarantine" + + +def test_caption_names_do_not_invent_tank_identity_or_sessions(): + assert admission.caption_classes("M1 heavy wrecker truck at a parade") == [] + assert admission.caption_classes("General Robert Abrams visits a clinic") == [] + assert admission.caption_classes("M1A2 Abrams tanks and M109A6 Paladin convoy") == ["abrams", "m109"] + + +def test_storage_budget_counts_data_and_artifacts_and_rejects_symlink(tmp_path, monkeypatch): + data, artifacts = tmp_path / "data", tmp_path / "artifacts" + data.mkdir() + artifacts.mkdir() + (data / "source.mp4").write_bytes(b"a" * 13) + (artifacts / "viewer.html").write_bytes(b"b" * 17) + cfg = {"storage": {"data_root": "data:test", "artifact_root": "artifact:test"}} + monkeypatch.setattr(admission, "external_path", lambda ref: data if ref.startswith("data:") else artifacts) + assert admission.storage_used(cfg) == 30 + (data / "alias").symlink_to(artifacts / "viewer.html") + with pytest.raises(ValueError, match="Symlinks"): + admission.storage_used(cfg) + + +def test_oversize_download_is_rejected_before_any_payload(tmp_path, monkeypatch): + class Response: + headers = {"Content-Length": "1001"} + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def read(self, *args): + raise AssertionError("Oversize media payload must not be read") + + monkeypatch.setattr(admission, "response", lambda *args, **kwargs: Response()) + monkeypatch.setattr(admission, "available_bytes", lambda cfg: 1100) + cfg = {"collection": {"maximum_encoded_asset_bytes": 1000}, + "storage": {"reserved_workspace_bytes": 100}} + destination = tmp_path / "source.mp4" + with pytest.raises(OSError, match="Advertised"): + admission.download_file("https://d34w7g4gy10iej.cloudfront.net/video/a.mp4", destination, cfg) + assert not destination.exists() + + +def test_octet_stream_playlist_requires_an_actual_hls_header(tmp_path, monkeypatch): + class Headers: + def get_content_type(self): + return "application/octet-stream" + + class Response: + headers = Headers() + payload = b"#EXTM3U\n#EXTINF:2,\na.ts\n#EXT-X-ENDLIST\n" + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def read(self, limit): + return self.payload + + mock = Response() + monkeypatch.setattr(admission, "response", lambda *args, **kwargs: mock) + monkeypatch.setattr(admission, "available_bytes", lambda cfg: 10000) + cfg = {"collection": {"maximum_metadata_bytes": 1000}} + url = "https://d34w7g4gy10iej.cloudfront.net/video/a.m3u8" + assert admission.fetch_text(url, cfg, tmp_path / "playlist.m3u8", media=True).startswith("#EXTM3U") + mock.payload = b"not a playlist" + with pytest.raises(ValueError, match="HLS manifest"): + admission.fetch_text(url, cfg, tmp_path / "wrong.m3u8", media=True) + assert not (tmp_path / "wrong.m3u8").exists() + + +def test_review_controls_have_no_caption_identity_or_fixture_dependency(): + key = "review-" + "a" * 24 + html = admission_review.card(key, 1, 4.0, np.ones((32000, 1), dtype=np.float32)) + assert f'/audio/{key}.m4a' in html + assert "Recording 1" in html and html.count(" a + ]) + expected_keys, expected_counts = np.unique(pairs, axis=0, return_counts=True) + keys, counts = packed_votes(refs, times, starts, ends, query_times) + assert np.array_equal(keys, expected_keys) + assert np.array_equal(counts, expected_counts) + keys, counts = packed_votes(refs, times, np.zeros(0, np.int64), + np.zeros(0, np.int64), np.zeros(0, np.int64)) + assert keys.shape == (0, 2) and len(counts) == 0 + + +def test_accelerated_lag_search_returns_original_scores_for_edges_and_dynamic_range(): + rng = np.random.default_rng(29) + for n in (64, 512, 32000): + reference = rng.normal(size=n) + query = np.r_[rng.normal(size=256), -.45 * reference, rng.normal(size=256)] + for start in (-300, 0, 128, 256, len(query), len(query) - n): + assert accelerated_correlation(reference, query, start, 256) == original_correlation( + reference, query, start, 256) + quiet = query * 1e-12 + quiet[0] = 1.0 + assert accelerated_correlation(reference, quiet, 256, 256) == original_correlation( + reference, quiet, 256, 256) + for reference, query in [ + (np.zeros(32000), np.zeros(33000)), + (np.ones(32000), np.ones(33000)), + (np.sin(np.arange(32000) / 7), np.sin(np.arange(33000) / 7)), + ]: + assert accelerated_correlation(reference, query, 400, 256) == original_correlation( + reference, query, 400, 256) diff --git a/tests/test_signature_p0.py b/tests/test_signature_p0.py new file mode 100644 index 0000000..a624f0c --- /dev/null +++ b/tests/test_signature_p0.py @@ -0,0 +1,299 @@ +"""Synthetic-only P0 contracts; no real media, CAST fits, or learned models.""" +import hashlib +import json +from copy import deepcopy + +import numpy as np +import pytest +import soundfile as sf + +from abvid.provenance.hashing import canonical +from abvid.signature.audio import read_asset +from abvid.signature.config import configuration, external_path, validate as validate_config +from abvid.signature.dedup import audit_manifest, with_reuse_links +from abvid.signature.fixtures import create_bundle, sample_annotations, sample_segment +from abvid.signature.manifest import connected_groups, readiness, schema, validate +from abvid.signature.trim import export_segments, import_annotations, make_packet + + +@pytest.fixture +def bundle(tmp_path, monkeypatch): + monkeypatch.setenv("ABVID_DATA_ROOT", str(tmp_path / "data")) + monkeypatch.setenv("ABVID_ARTIFACT_ROOT", str(tmp_path / "artifacts")) + cfg = configuration() + manifest = create_bundle("data:military_signature/p0_fixtures/validation", cfg) + return manifest, cfg + + +def _asset(manifest, aid="fixture-a"): + return next(a for a in manifest["assets"] if a["asset_id"] == aid) + + +def _packet(manifest, config): + result = make_packet(manifest, config, "artifact:military_signature/blind", + "artifact:military_signature/private/map.json") + blind = external_path(result["blind_reference"]) + batch = json.loads((blind / "review_batch.json").read_text()) + private = json.loads(external_path(result["private_reference"]).read_text()) + annotations = sample_annotations(batch, result["batch_sha256"]) + return result, batch, private, annotations + + +def test_canonical_config_and_schema_artifacts_match_runtime(): + from abvid.paths import resolve + assert json.loads(resolve("repo:experiments/military_signature/p0_protocol.json").read_text()) == json.loads(json.dumps(configuration())) + assert json.loads(resolve("repo:experiments/military_signature/manifest_schema.json").read_text()) == schema() + + +@pytest.mark.parametrize("field", ["real_media_collection", "real_media_processing", "model_fitting", "protected_source_evaluation"]) +def test_p0_cannot_expand_authorization(field): + cfg = configuration() + cfg["authorization"][field] = True + with pytest.raises(ValueError, match="Changed P0"): + validate_config(cfg) + + +def test_strict_manifest_rejects_extra_fields_and_boolean_durations(bundle): + m, cfg = bundle + bad = deepcopy(m) + bad["assets"][0]["invented_session"] = "session-999" + with pytest.raises(ValueError, match="unexpected fields"): + validate(bad, cfg) + bad = deepcopy(m) + bad["assets"][0]["duration_seconds"] = True + with pytest.raises(ValueError, match="expected number"): + validate(bad, cfg) + + +def test_unknown_session_preserved_and_not_counted_as_independent(bundle): + m, cfg = bundle + for a in m["assets"][:2]: + a.update(recording_session="unknown", session_evidence="unknown", independence_verified=False) + m["segments"] = [sample_segment("fixture-a"), sample_segment("fixture-b", segment_id="fixture-segment-b")] + validate(m, cfg) + assert connected_groups(m)["fixture-a"] == connected_groups(m)["fixture-b"] + assert all(a["recording_session"] == "unknown" for a in m["assets"][:2]) + assert readiness(m, cfg)["verified_groups_per_fixture_label"] == {} + + +def test_asset_id_does_not_authorize_independence(bundle): + m, cfg = bundle + m["assets"][0].update(recording_session="unknown", session_evidence="unknown") + with pytest.raises(ValueError, match="Independence requires"): + validate(m, cfg) + + +def test_same_session_and_transitive_reuse_cannot_cross_roles(bundle): + m, cfg = bundle + a = sample_segment("fixture-a") + b = sample_segment("fixture-b", segment_id="fixture-segment-b") + b["role"] = "evaluation" + m["segments"] = [a, b] + validate(m, cfg) + m["reuse_links"] = [ + {"left": "fixture-a", "right": "fixture-short", "kind": "suspected_reuse", "evidence": "fixture linkage"}, + {"left": "fixture-short", "right": "fixture-b", "kind": "provenance", "evidence": "fixture linkage"}, + ] + with pytest.raises(ValueError, match="group split across roles"): + validate(m, cfg) + + +def test_exact_raw_hash_groups_assets_even_if_session_claims_differ(bundle): + m, cfg = bundle + duplicate = deepcopy(m["assets"][0]) + duplicate.update(asset_id="fixture-copy", recording_session="different-claimed-session") + m["assets"].append(duplicate) + validate(m, cfg) + groups = connected_groups(m) + assert groups["fixture-copy"] == groups["fixture-a"] + + +@pytest.mark.parametrize("kind,field,value", [ + ("foreground", "audible_vehicle_count", 2), + ("background", "audible_vehicle_count", 1), + ("foreground", "contaminants", ["music"]), + ("foreground", "start_seconds", float("nan")), + ("foreground", "end_seconds", 99), + ("foreground", "blind_review", False), + ("rejected", "rejection_reasons", []), +]) +def test_invalid_trim_decisions_are_rejected(bundle, kind, field, value): + m, cfg = bundle + row = sample_segment("fixture-a", kind=kind) + row[field] = value + m["segments"] = [row] + with pytest.raises(ValueError): + validate(m, cfg) + + +def test_overlapping_accepted_background_and_foreground_fail(bundle): + m, cfg = bundle + m["segments"] = [sample_segment("fixture-a"), + sample_segment("fixture-a", segment_id="background", kind="background", start=3, end=5)] + with pytest.raises(ValueError, match="overlap"): + validate(m, cfg) + + +def test_nonindependent_source_cannot_enter_evaluation(bundle): + m, cfg = bundle + m["assets"][0]["independence_verified"] = False + m["segments"] = [sample_segment("fixture-a")] + m["segments"][0]["role"] = "evaluation" + with pytest.raises(ValueError, match="Unverified sessions"): + validate(m, cfg) + + +def test_missing_background_is_readiness_failure(bundle): + m, cfg = bundle + m["segments"] = [sample_segment("fixture-a")] + result = readiness(m, cfg) + assert "fixture-a: no reviewed same-asset background" in result["unresolved"] + assert not result["model_claim_eligible"] and not result["real_collection_authorized"] + + +def test_dedup_finds_crop_gain_polarity_and_resampling_without_class_labels(bundle): + m, cfg = bundle + result = audit_manifest(m, cfg) + pairs = {frozenset((p["left"], p["right"])) for p in result["pairs"]} + assert frozenset(("fixture-a", "fixture-crop")) in pairs + assert frozenset(("fixture-a", "fixture-resampled")) in pairs + assert frozenset(("fixture-a", "fixture-b")) not in pairs + statuses = {r["asset_id"]: r["status"] for r in result["assets"]} + assert statuses["fixture-short"] == "too_short_for_fingerprint" + assert statuses["fixture-silence"] == "silent_unresolved" + assert result["independence_certified"] is False + altered = deepcopy(m) + for a in altered["assets"]: + a["title"] = "Conflicting hidden label" + again = audit_manifest(altered, cfg) + assert again["pairs"] == result["pairs"] + + +def test_stale_dedup_audit_and_role_leakage_fail(bundle): + m, cfg = bundle + result = audit_manifest(m, cfg) + stale = deepcopy(m) + stale["assets"][0]["title"] = "Changed metadata" + with pytest.raises(ValueError, match="different manifest"): + with_reuse_links(stale, result, cfg) + # Deliberately conflicting synthetic claims: detected waveform reuse must + # override the claimed independent session IDs. + m["assets"][2]["recording_session"] = "claimed-independent-crop" + m["segments"] = [sample_segment("fixture-a"), sample_segment("fixture-crop", segment_id="crop")] + m["segments"][1]["role"] = "evaluation" + validate(m, cfg) + result = audit_manifest(m, cfg) + with pytest.raises(ValueError, match="group split across roles"): + with_reuse_links(m, result, cfg) + + +def test_corrupt_reuse_scores_cannot_create_links(bundle): + m, cfg = bundle + result = audit_manifest(m, cfg) + result["pairs"][0]["correlation"] = 0.1 + with pytest.raises(ValueError, match="acceptance rule"): + with_reuse_links(m, result, cfg) + + +def test_known_physical_vehicle_cannot_cross_roles(bundle): + m, cfg = bundle + for asset in m["assets"][:2]: + asset["physical_vehicle_id"] = "known-artificial-individual" + asset["vehicle_identity_evidence"] = "Synthetic fixture identity declaration" + m["segments"] = [sample_segment("fixture-a"), sample_segment("fixture-b", segment_id="b")] + m["segments"][1]["role"] = "evaluation" + with pytest.raises(ValueError, match="group split across roles"): + validate(m, cfg) + + +def test_preparation_workflow_retains_rejections_and_native_samples(bundle): + from abvid.signature.validation import run + result = run("data:military_signature/p0_fixtures/workflow", + "artifact:military_signature/p0/workflow") + evidence = json.loads(external_path(result["validation_ref"], must_exist=True).read_text()) + assert result["exported_clips"] == 8 and result["rejections"] == 2 + assert all(evidence["checks"].values()) + assert len(evidence["native_export_checks"]) == 8 + assert evidence["readiness"]["model_claim_eligible"] is False + + +def test_blind_packet_hides_metadata_and_original_filenames(bundle): + m, cfg = bundle + result, batch, private, annotations = _packet(m, cfg) + blind = external_path(result["blind_reference"]) + text = (blind / "index.html").read_text() + (blind / "review_batch.json").read_text() + for a in m["assets"]: + assert a["asset_id"] not in text + assert a["title"] not in text + assert a["recording_session"] not in text + assert len(batch["recordings"]) == len(m["assets"]) + assert not external_path(result["private_reference"]).is_relative_to(blind) + imported = import_annotations(m, cfg, batch, private, annotations) + assert len(imported["segments"]) == len(m["assets"]) + assert all(s["kind"] == "rejected" and s["rejection_reasons"] for s in imported["segments"]) + + +def test_private_mapping_cannot_be_inside_blind_directory(bundle): + m, cfg = bundle + with pytest.raises(ValueError, match="outside the blind"): + make_packet(m, cfg, "artifact:blind", "artifact:blind/private.json") + + +def test_wrong_batch_and_unreviewed_recordings_fail(bundle): + m, cfg = bundle + _, batch, private, annotations = _packet(m, cfg) + annotations["batch_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="batch hash mismatch"): + import_annotations(m, cfg, batch, private, annotations) + annotations["batch_sha256"] = hashlib.sha256(canonical(batch)).hexdigest() + annotations["intervals"].pop() + with pytest.raises(ValueError, match="Every recording"): + import_annotations(m, cfg, batch, private, annotations) + + +def test_altered_private_mapping_is_detected(bundle): + m, cfg = bundle + _, batch, private, annotations = _packet(m, cfg) + keys = list(private["mapping"]) + private["mapping"][keys[0]], private["mapping"][keys[1]] = private["mapping"][keys[1]], private["mapping"][keys[0]] + with pytest.raises(ValueError, match="commitment mismatch"): + import_annotations(m, cfg, batch, private, annotations) + + +def test_native_trim_rounding_preserves_samples_channels_and_rejections(bundle): + m, cfg = bundle + stereo, rate = read_asset(_asset(m), cfg) + stereo = np.column_stack((stereo[:, 0], 0.3 * stereo[:, 0])) + path = external_path(_asset(m)["media_ref"], must_exist=True) + sf.write(path, stereo, rate, subtype="FLOAT") + _asset(m).update(raw_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), channels=2) + m["segments"] = [sample_segment("fixture-a", start=2.00003, end=6.00003), + sample_segment("fixture-b", segment_id="rejected", kind="rejected")] + result = export_segments(m, cfg, "artifact:military_signature/trimmed") + record = result["clips"][0] + clip, clip_rate = sf.read(external_path(record["clip_ref"]), dtype="float64", always_2d=True) + expected, _ = read_asset(_asset(m), cfg) + assert record["start_frame"] == 32000 and record["end_frame"] == 96001 + assert clip_rate == rate and clip.shape[1] == 2 + np.testing.assert_array_equal(clip, expected[32000:96001]) + assert result["rejections"] == [m["segments"][1]] + with pytest.raises(FileExistsError): + export_segments(m, cfg, "artifact:military_signature/trimmed") + + +def test_raw_hash_mismatch_fails_before_audio_is_used(bundle): + m, cfg = bundle + _asset(m)["raw_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="SHA256 mismatch"): + read_asset(_asset(m), cfg) + + +def test_real_input_and_repository_outputs_are_blocked(bundle): + m, cfg = bundle + _asset(m)["media_kind"] = "real" + with pytest.raises(ValueError, match="synthetic-only"): + validate(m, cfg) + with pytest.raises(ValueError, match="data: or artifact:"): + external_path("repo:output/forbidden.wav") + with pytest.raises(ValueError): + external_path("data:../ABVID/forbidden.wav")