diff --git a/MANIFEST.in b/MANIFEST.in index 50b1373..b8c05de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,8 @@ recursive-include scripts *.py recursive-include tests *.py *.txt *.json recursive-include integrations *.py *.json *.md *.ts *.tsx recursive-include clients *.ts *.json *.mjs +prune clients/typescript/node_modules +prune clients/typescript/dist recursive-include evals *.json global-exclude *.pyc global-exclude *.pt *.pth *.safetensors *.onnx *.ckpt diff --git a/docs/RELEASE_READINESS.md b/docs/RELEASE_READINESS.md index cb3b709..8257e01 100644 --- a/docs/RELEASE_READINESS.md +++ b/docs/RELEASE_READINESS.md @@ -1,5 +1,23 @@ # Release readiness — intentionally incomplete +## Full-weight mechanics and graph lifetime supplement (September 29, 2026) + +The [graph lifetime repair and measured rank-0 limits](evidence/native/graph-lifetime-fullweight-limits-20260929.md) +permit the same-init BF16 first step and reload, with a passing CUDA ownership +regression and failing old-source control. Rank-0 FP32/BF16 resume backward still +exhausts the unchanged allocator cap on this configuration. These results do not +close #3/#4 or alter the source identity of earlier numerical/rank-8 receipts. + +## FP32 candidate numerical readback (September 29, 2026) + +The [canonical FP32 repair](evidence/native/short-fp32-canonical-v10-20260929.md) +passes the short independent HF adapter oracle, four generated cached/batched/full +cases, 8191/8192 state-prefix boundaries, and a 10520-token state numerical gate. +This is candidate source evidence; the exact-limit FP32 training resource probe exhausted the fixed allocator cap +during forward, leaving backward unverified. Delivery readback remains pending. Issues #3/#4 remain open; representative quality and Jev benefit +are unknown. Overwatch remains archived by user instruction. Older readbacks +below are historical and do not override these scoped receipts. + ## Current delivery readback (September 28, 2026) Layev main is `a6926f7b4585e840b36f1d9157c3af3907210ef5`, after protected diff --git a/docs/acceptance-status.json b/docs/acceptance-status.json index 9d6dacc..2848fcc 100644 --- a/docs/acceptance-status.json +++ b/docs/acceptance-status.json @@ -1,4 +1,23 @@ { + "fullweight_graph_lifetime_supplement_2026_09_29": { + "receipt": "docs/evidence/native/graph-lifetime-fullweight-limits-20260929.json", + "graph_lifetime": "CUDA positive passes; proper old-source negative fails; 14 BF16 tests pass", + "rank0_fp32": "first step and reload pass; resume backward CUDA OOM; allocator-option retry fails", + "rank0_bf16": "same-init graph-repaired first step and reload pass; resume short-attention backward CUDA OOM", + "scope": "Generated short mechanics; measured configuration only; no complete full-weight matrix or quality claim", + "issue_3": "open", + "issue_4": "open" + }, + "short_fp32_canonical_v10_candidate_2026_09_29": { + "scope": "candidate source short FP32 numerical repair only; no issue closure", + "receipt": "docs/evidence/native/short-fp32-canonical-v10-20260929.json", + "hf_adapter_gradients_passed": 96, + "generated_seam_cases_passed": 4, + "gradients_per_comparison": 100, + "long_validation": "fresh 8191/8192 prefix boundaries and 10520-token state numerical parity passed; exact-limit FP32 training forward OOM at unchanged 70% cap; backward unverified", + "issue_3": "open", + "issue_4": "open" + }, "schema_version": 1, "kind": "workstream-evidence-ledger-not-original-feature-matrix", "base_revision": "b4cb9df97be61729ebd68b8e0f5ee9bb2d193c73", diff --git a/docs/evidence/native/FP32_REPRODUCTION.md b/docs/evidence/native/FP32_REPRODUCTION.md new file mode 100644 index 0000000..6f71886 --- /dev/null +++ b/docs/evidence/native/FP32_REPRODUCTION.md @@ -0,0 +1,23 @@ +# Reproduce the scoped FP32 numerical evidence + +Use the CUDA model environment with PyTorch 2.10.0+cu128 and Transformers 4.57.1. +Supply the locally retained trained native LoRA checkpoint; weights are not +included in this repository. Both scripts record runtime source identity and +refuse overwriting result artifacts. Output paths below must be fresh. + +```shell +python scripts/verify_native_fp32_gradients.py --checkpoint CHECKPOINT.pt --expected-checkpoint-sha256 99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9 --out fp32-hf.json +python scripts/verify_native_fp32_seams.py --checkpoint CHECKPOINT.pt --expected-checkpoint-sha256 99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9 --out-dir fp32-seams +python scripts/verify_native_fp32_seams.py --checkpoint CHECKPOINT.pt --expected-checkpoint-sha256 99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9 --out-dir fp32-long --cases boundary_8191 boundary_8192 long_10520 +``` + +The first command compares hidden states and all Q/V LoRA parameter gradients +against independent eager HF parameters. The second runs four exact-token +cached/batched/full numerical cases. The third constructs the same generated +8191/8192 boundary and 10520-token state requests and applies the same loss and +three-mode backward protocol. It requires substantial GPU time. The 70% CUDA allocator cap and disabled +TF32 are fixed. HF hidden tolerance is 1e-4, logits/probabilities 1e-5, and +trainable gradients 2e-5 (combined absolute/relative). Generated requests +establish no representative quality, long-context or Jev benefit. The seams +protocol retains its explicit 16-GiB cache-policy ceiling; this is a protocol +planning ceiling, not allocator budget growth or a total-memory claim. diff --git a/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.json b/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.json new file mode 100644 index 0000000..76e261c --- /dev/null +++ b/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.json @@ -0,0 +1,544 @@ +{ + "evidence_class": "cuda-bf16-branch-graph-lifetime-repair-and-rank0-resource-limits", + "source_parent": "e4822909680080bc300371782f6b439ff26fc8a8", + "historical_training_sha256": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde", + "repaired_training_sha256": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c", + "numerical_model_sha256": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "numerical_native_gradients_sha256": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "graph_lifetime_regression": { + "test": "tests/test_parallel_questions.py::test_bf16_branch_activations_released_before_optimizer", + "test_source_sha256": "5ba0d7a79d29a829a7ac378708eef1b4e1e42bb8d62f48b119fe32b82e8cd15b", + "positive_cuda": { + "passed": 1, + "deselected": 80, + "elapsed_seconds": 6.18 + }, + "proper_old_source_negative": { + "expected_failed": 1, + "deselected": 80, + "elapsed_seconds": 3.87 + }, + "bf16_regressions": { + "passed": 14, + "deselected": 67, + "elapsed_seconds": 18.09 + }, + "private_ledger_result_line_sha256": "a57285a3034e4d1fa3676ebf0e9af523b45b028989650a01dfd04796af6ca805", + "scope": "Tiny CUDA graph-ownership regression and BF16 tests; not full-weight resource or numerical acceptance.", + "private_root_tool_observations_sha256": "f1f82dbba6a044f8c51361cc319e629f84f2e3d0595d8e32dd99deab33bc11bc", + "verification_scope": "Terminal observations recorded by GPU owner; independently read by reviewer, not rerun by reviewer.", + "count_overlap": "14 BF16 tests include the one lifetime case; counts are not additive." + }, + "rank0_cells": [ + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r0-fp32-2b0c3a8f3255", + "precision": "fp32", + "rank": 0, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "failed_or_unsupported", + "private_terminal_sha256": "991870e7c7ec9b024d16fd54c3b0bf293e843b6e64d0dc6c501ded4849bd228e", + "phases": [ + { + "private_receipt_sha256": "cd8c9236d5d0b4ac5b9d646238d4f8c555b1a6e3ab82a01245547bfd44b102d1", + "phase": "init", + "status": "completed", + "elapsed_seconds": 25.50128520000726, + "exit_code": 0, + "child_resources": null, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "f34b0a1dc2932a66fbd733d600c4e4c299e8a7adf0d4b3399fb8b33a7866f2bb" + }, + { + "private_receipt_sha256": "6306c617b12d2e63e57a6fff9d9eb24cd5f59a8db142811c4daa2e2dbb84c875", + "phase": "reload_step1", + "status": "completed", + "elapsed_seconds": 27.59219409999787, + "exit_code": 0, + "child_resources": null, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "private_receipt_sha256": "1b8d2f315d0cf05eee2474b65360b0456ff06ebb4cf60113a4cf59bf22e97728", + "phase": "resume_step2", + "status": "cuda_oom", + "elapsed_seconds": 54.52116990002105, + "exit_code": 1, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 615837696, + "matmul_precision": "highest", + "peak_allocated_bytes": 8380563968, + "peak_reserved_bytes": 8829009920, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "eba6b4aa5b0dfc2a4c18d94d1a6333b449eef940d4b09e8b6173ce171c4ed1e9", + "stdout_sha256": "145c1d65faff7450dd93bb30ae340a1351dccffb3fabc1ecb726852de9f3d03d" + }, + { + "private_receipt_sha256": "134df19faf858ab0c4e1918b434941d3e2f3fd6575d120cec250697437fab003", + "phase": "train_step1", + "status": "completed", + "elapsed_seconds": 51.43313920000219, + "exit_code": 0, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 969986048, + "matmul_precision": "highest", + "peak_allocated_bytes": 7943451648, + "peak_reserved_bytes": 8594128896, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "084cbc0cf9be2780d3bfe0dd4c3d536dd23d711e4fa91b228c41964b6068040a" + } + ], + "step1_reload": { + "private_receipt_sha256": "16a1cc1adbacc1fb47070d034cf718f430b9c758325f5c5e3f47e7dd935947a0", + "activation_checkpointing": false, + "checkpoint_sha256": "5020d31211f6d374a8582e48add1426ff801e83d7b74c718fc18eb40edf23235", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2931298816, + "cuda_peak_reserved_bytes": 3061841920, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 294, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 0, + "precision": "fp32", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 1 + }, + "protocol_runner_sha256": "b1574fcc46068a4348af397c220b492925ea0d120e83957ca0ce4c695778daae", + "replayed_init_sha256": null + }, + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r0-bf16-58b0b97ba9b2", + "precision": "bf16", + "rank": 0, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "failed_or_unsupported", + "private_terminal_sha256": "bc271f62e4f60752a8527dd80416026c82b672894333ed57421149e5f301a69c", + "phases": [ + { + "private_receipt_sha256": "6f96107b0964f8b18a0aaf49216ba202bda6a8c4e58704ac9c90a11deaa74e2f", + "phase": "init", + "status": "completed", + "elapsed_seconds": 25.524509299983038, + "exit_code": 0, + "child_resources": null, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "d9ef07881308cd6d286c33b0712d498f3cd2b5b819e61d59c324297e95816193" + }, + { + "private_receipt_sha256": "ace74a3e57057390412089a660575b3c3b62d53a223bbf7b6481813583242217", + "phase": "train_step1", + "status": "cuda_oom", + "elapsed_seconds": 16.892366099986248, + "exit_code": 1, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 246013952, + "matmul_precision": "highest", + "peak_allocated_bytes": 8242568704, + "peak_reserved_bytes": 9011462144, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "41f10cdb436417cd1f5aec988b9a4f4a2d1702ceb86ecc10e20eb7ae0ef1e386", + "stdout_sha256": "7fbec3da4bdd207a3176d52cb6796a8cc73dfeb00cfea7177ae00eecafa3cb60" + } + ], + "protocol_runner_sha256": "b1574fcc46068a4348af397c220b492925ea0d120e83957ca0ce4c695778daae", + "replayed_init_sha256": null + }, + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r0-bf16-199c0c84e7b7", + "precision": "bf16", + "rank": 0, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "failed_or_unsupported", + "private_terminal_sha256": "b6b6047c247cb48005d87471c95964f6bae392e688e3da1fb9eaae66f978327b", + "phases": [ + { + "private_receipt_sha256": "a56f923c3aed7638332fc04d13ce34656ace5ff03bc935447324894033bab265", + "phase": "train_step1", + "status": "failed", + "elapsed_seconds": 10.250507699995069, + "exit_code": 1, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 11799625728, + "matmul_precision": "highest", + "peak_allocated_bytes": 0, + "peak_reserved_bytes": 0, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "f7c01155072833786f37a8abc5c9dc94ec9d3375b7aa7e04b3692b98fc50b9a8", + "stdout_sha256": "165f18655732ce03edb4c7297c392408bae918d9dfd1ce220dc70769fb3b9d9d" + } + ], + "protocol_runner_sha256": "3078cfde99f4e2be19d9e36747df11c84afd5c6148fa3a88548ee4fd3f04e64a", + "replayed_init_sha256": "3439eeef30f4bcf9516672d5aec2b27e51ede6651e2334e150d43a5a9a934ad2" + }, + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r0-bf16-edc45eee2b52", + "precision": "bf16", + "rank": 0, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "failed_or_unsupported", + "private_terminal_sha256": "6004e5c929de1730b7ba5bfea175e5d908ecca73d94761a8396a15d3ce616385", + "phases": [ + { + "private_receipt_sha256": "e0cb5d1d01c6cde56166ff3faa995900f74bf09f0ba13fc50c88f61e0824c771", + "phase": "reload_step1", + "status": "completed", + "elapsed_seconds": 26.448426799994195, + "exit_code": 0, + "child_resources": null, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "private_receipt_sha256": "38de1c5a043607ad221cddc7242220b227f9eedb94a30a93bc5027a514446a7c", + "phase": "resume_step2", + "status": "cuda_oom", + "elapsed_seconds": 51.65265030000592, + "exit_code": 1, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 244879360, + "matmul_precision": "highest", + "peak_allocated_bytes": 8638769664, + "peak_reserved_bytes": 9017753600, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "6c4715d57245a0c0f574b27360e61a8955eb9f715eeb229acd9a6449e83f0a6b", + "stdout_sha256": "b58574aab245133f2d2035dd202f0c78f3425dabf2c4433c91b8da1621b2bee2" + }, + { + "private_receipt_sha256": "c757d926a0ea7ec965c8b1200ddfb254defb751a57ac5d9cc939273482e294a6", + "phase": "train_step1", + "status": "completed", + "elapsed_seconds": 39.36326710000867, + "exit_code": 0, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 639475712, + "matmul_precision": "highest", + "peak_allocated_bytes": 7971341824, + "peak_reserved_bytes": 8552185856, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "c8f44fc256a7c9193cc1d04e14c07af44bb2a23228846d98c40cadb32b937d2c" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "stdout_sha256": "63d952188d6df65372ef051cdf891974ea13ab217dc12c1256371df4a3807324" + } + ], + "step1_reload": { + "private_receipt_sha256": "1e2b078c5eecb86f056ee56aabecc90f304d58d329bd90c81915cfe9b7e21505", + "activation_checkpointing": false, + "checkpoint_sha256": "5c30c08a966653b7c3faea28b0c863c8567c24c7de0c1902c0e82a92b175f900", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2931298816, + "cuda_peak_reserved_bytes": 3068133376, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 294, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 0, + "precision": "bf16", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 1 + }, + "protocol_runner_sha256": "70172c85eeac490a721985ccea4f406a41e4b1dac86d1bbb19cab7493d354556", + "replayed_init_sha256": "3439eeef30f4bcf9516672d5aec2b27e51ede6651e2334e150d43a5a9a934ad2" + } + ], + "same_run_fp32_allocator_retry": { + "phase": "resume_step2_expandable_once", + "status": "cuda_oom", + "elapsed_seconds": 76.27579380001407, + "exit_code": 1, + "child_resources": { + "allocator_fraction": 0.7, + "allow_tf32": false, + "cudnn_allow_tf32": false, + "free_after_bytes": 735862784, + "matmul_precision": "highest", + "peak_allocated_bytes": 8380563968, + "peak_reserved_bytes": 8829009920, + "total_bytes": 12884377600 + }, + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "stderr_sha256": "d80c048a551f84d111a13ac3fdb3d0bc284c11ddf6b042dc74ee1c0e2c6b8ff4", + "stdout_sha256": "1888e3cdfc854e454950c7f624ab60ffe1d44a3ed70383a61086776fbe8c1289", + "private_phase_receipt_sha256": "66a464bb921b525ea48c9dc4deb956418fd1e714f252deee0024bbd90cc4cadf", + "private_terminal_sha256": "54aa0d0861b0ec770f7bc006d7f34737256f166e9b3851353ba9f3441c2b0f7e", + "allocator_option": "expandable_segments:True", + "allocator_option_effect": "PyTorch warned expandable_segments not supported on this platform; retry still CUDA OOM." + }, + "scope": "Generated short noncheckpoint rank-0 cells on RTX3060 under unchanged70%cap. C8 graph release repairs demonstrated optimizer graph lifetime and permits BF16 first step plus reload, while rank-0 FP32/BF16 step-2 backward remain CUDA OOM. No universal unsupported-hardware, installed serving, numerical equality, representative quality or issue closure claim." +} diff --git a/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.md b/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.md new file mode 100644 index 0000000..190eacd --- /dev/null +++ b/docs/evidence/native/graph-lifetime-fullweight-limits-20260929.md @@ -0,0 +1,45 @@ +# BF16 graph lifetime repair and full-weight resource limits + +Refs #3; incomplete acceptance. The [portable receipt](graph-lifetime-fullweight-limits-20260929.json) +separates historical numerical source from the subsequent training graph repair. +Previous numerical and rank-8 receipts retain their original source hashes. + +Earlier BF16 branch backward retained graph owners through optimizer allocation. +Deleting the completed local logit, gradient, list and loss owners after backward +releases those activations before AdamW. The CUDA regression checks weak references +at optimizer entry: one pass; the proper old-source control fails that assertion. +All 14 selected BF16 tests pass, including the lifetime case; these counts are +not additive. The GPU owner recorded these terminal results; the reviewer +verified the observation receipt and did not rerun the GPU tests. This demonstrates graph ownership repair without +changing gradient order, optimizer parameters, allocator cap or resume identity. + +| Generated noncheckpoint rank-0 cell | Step 1 / reload | Resume step 2 | +| --- | --- | --- | +| Historical FP32 | Passed / passed | Backward CUDA OOM | +| Historical BF16 | First AdamW moment allocation CUDA OOM | Not reached | +| BF16 same-init replay after graph repair | Passed / passed | Short attention backward CUDA OOM | + +The repaired BF16 replay uses the exact original initialization hash recorded in +the receipt. A preliminary replay failed tokenizer setup before GPU execution; +it is preserved separately. The corrected replay's first step takes 39.363267 +seconds, allocating/reserving peaks of 7,971,341,824 / 8,552,185,856 bytes. +Its resume fails after 51.652650 seconds with peaks of 8,638,769,664 / +9,017,753,600 bytes, at a 2-MiB short-attention gradient allocation. Historical +FP32 resume fails at a 520-MiB backward allocation. A single same-run FP32 +allocator-option retry also fails; PyTorch explicitly warns that expandable +segments are unsupported on this Windows runtime. This observation does not +establish that all hardware or supported execution configurations are unsuitable. + +The tested graph repair improves the observed BF16 first-step result. Complete +rank-0 train/resume mechanics remain unverified on this measured configuration. +The private mode orchestrator is not published here; no public runnable mode +command is claimed. The graph ownership test is reproducible with the CUDA model +environment: + +```shell +python -m pytest -q tests/test_parallel_questions.py::test_bf16_branch_activations_released_before_optimizer +``` + +These generated mechanics establish no installed serving, quality, Jev benefit +or closure of #3/#4. The separate exact-limit training resource failure remains +recorded in the [FP32 numerical evidence](short-fp32-canonical-v10-20260929.md). diff --git a/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.json b/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.json new file mode 100644 index 0000000..95db84f --- /dev/null +++ b/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.json @@ -0,0 +1,251 @@ +{ + "evidence_class": "actual-weight-noncheckpoint-rank8-short-train-resume-reload-v10", + "cells": [ + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r8-fp32-f0969f7dbcd4", + "config_sha256": "9ea5b040cc86a771f5e1b55d7ebdd72e78107c691a42ca7abe76d9a308273558", + "init_sha256": "e0257a4b48787ecb369364dd3e72edd9813f15f97ee7f1c854e6d6b3440d2f5d", + "limitations": "Generated short mechanics only; no quality, long-context or installed serving claim.", + "precision": "fp32", + "rank": 8, + "reload_step1_sha256": "1d6dd5562fdcde78c48c40b889f117e6a084737cdff65ae985c0811930eac4bb", + "reload_step2_sha256": "7708a966854cb0061fe4d4926cfef0e70e076c46f2117eb0a63c260fbec4489c", + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "passed_generated_short_mechanics", + "step1": { + "attempt_id": "7fe5a3a4808a483d9a5eef76289d1cbf", + "attempt_index": 0, + "gpu_peak_allocated_bytes": 2295416832, + "gradient_norm": 1564.74853515625, + "loss": 3.7636687755584717, + "manifest_sha256": "6eb220653e802c7365eb626bbc7ceb310a9022e0752ee9726cd1f747b8ce69b4", + "metrics_sha256": "ce7d4d3f80bbb5a519936a8a1151b459557f239b981f81a8e9c06fac064e1794", + "monitoring_export_failures": 0, + "parent_attempt_id": null, + "parent_sha256": "e0257a4b48787ecb369364dd3e72edd9813f15f97ee7f1c854e6d6b3440d2f5d", + "sha256": "10895f4245ec99d95c9a132aa2f6bee5a38815b3a8e87d3f6e8a21cafc491df4", + "size_bytes": 1983563799, + "step": 1, + "telemetry_sha256": "fd1053197ad181ae8ee76aaeaf55e91917838dadc7f9c9ac697491cd4141ba54" + }, + "step2": { + "attempt_id": "8d0a6c5345924b2b91a53399a5b404c7", + "attempt_index": 1, + "gpu_peak_allocated_bytes": 2300203008, + "gradient_norm": 692.789794921875, + "loss": 2.033273220062256, + "manifest_sha256": "2772abe092d2ec8c8e3ae177b7ac9803d0ac7ac9e9031c2860db9ac08b5c72de", + "metrics_sha256": "d6c40d02b0f64f7d1e94b209c00f7064e9724ef0a160c43da13a99bf57581f5b", + "monitoring_export_failures": 0, + "parent_attempt_id": "7fe5a3a4808a483d9a5eef76289d1cbf", + "parent_sha256": "10895f4245ec99d95c9a132aa2f6bee5a38815b3a8e87d3f6e8a21cafc491df4", + "sha256": "a8b832cc7857307b6ff57208ba8d0b4b3b82d2e268f8dfa1ae0104da66ac7d84", + "size_bytes": 1983564759, + "step": 2, + "telemetry_sha256": "b9c75a31d3afc5e5ab313657710407295a64bff8d70e2bd18e7b34c14446f1a9" + }, + "private_terminal_sha256": "5392ab66823a23c6e92a35f3b228568d9337c873441f3b065bd6f7859438f6ef", + "protocol_runner_sha256": "76a2e23a930f402d7b8bce948a6106af3506f55466c1b10983f689a473f890ed", + "cuda_allocator_fraction": 0.7, + "reloads": { + "step1": { + "activation_checkpointing": false, + "checkpoint_sha256": "10895f4245ec99d95c9a132aa2f6bee5a38815b3a8e87d3f6e8a21cafc491df4", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2933461504, + "cuda_peak_reserved_bytes": 3061841920, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 390, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 8, + "precision": "fp32", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 1 + }, + "step2": { + "activation_checkpointing": false, + "checkpoint_sha256": "a8b832cc7857307b6ff57208ba8d0b4b3b82d2e268f8dfa1ae0104da66ac7d84", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2933461504, + "cuda_peak_reserved_bytes": 3061841920, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 390, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 8, + "precision": "fp32", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 2 + } + } + }, + { + "activation_checkpointing": false, + "cell_id": "issue3-nockpt-r8-bf16-f37c14aee159", + "config_sha256": "252f7030161d660ae6436a155311d617c452c13cf3ac775f0cb68adb0768600c", + "init_sha256": "1de9cb9bab06e1c8e6a5ec917bdf40da948432fd35144252f4cd644cfdf90011", + "limitations": "Generated short mechanics only; no quality, long-context or installed serving claim.", + "precision": "bf16", + "rank": 8, + "reload_step1_sha256": "46d7398bc56f25f5efc91462289643f1d9ff529388138fc54a32ec39394ebf50", + "reload_step2_sha256": "ffc56d1d890c73968a0d6a4a3f125550fad1ca82d73cf91188c5b5ba27195ca0", + "source": { + "base_config_sha256": "f52eb9f1bdf1a1552b5edec727f21b975b83eb9b3a064aa7d848f413d2d0146c", + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "files": { + "checkpoint.py": "f7cd8cb15e130756751c53d6cb55ec2b7c3ef0c608da764d10b000552f1aab95", + "cli.py": "41a9567ed9791208bc01cdf8eeb979171d54dbf69fac8a66dc8a4caec09fa5c5", + "execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a", + "model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "suite_manifest_sha256": "29cbc2e7541cc0b8a9ec92f6b8583740b91fa33d1e4340e47b6ff9f0e17c84e0", + "tokenizer_sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "weights_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342" + }, + "status": "passed_generated_short_mechanics", + "step1": { + "attempt_id": "9eadb1fec2a8447cabe294d62ba41c95", + "attempt_index": 0, + "gpu_peak_allocated_bytes": 4246000640, + "gradient_norm": 7.916440963745117, + "loss": 0.03347368538379669, + "manifest_sha256": "4e9366d1704380c803b0f4e8459e0c29aa9c7428283443d5efbd9f12a170dad8", + "metrics_sha256": "eb1266d0c45cb7b832fae392e3bcfe5bd426b5142ee882b71846a4e482c80522", + "monitoring_export_failures": 0, + "parent_attempt_id": null, + "parent_sha256": "1de9cb9bab06e1c8e6a5ec917bdf40da948432fd35144252f4cd644cfdf90011", + "sha256": "97d836eb1100c7b2ddd94fcbb2020029af62a5f69d41815a4b1c030d5f1505c4", + "size_bytes": 1983563799, + "step": 1, + "telemetry_sha256": "551a57a8dc97110c7c5cb88d0d685563b89d8dc1f71c59f75de27c998cd227d8" + }, + "step2": { + "attempt_id": "d396a1609cbd4e10a248207df5203bd1", + "attempt_index": 1, + "gpu_peak_allocated_bytes": 4250786816, + "gradient_norm": 85.5900650024414, + "loss": 0.932081401348114, + "manifest_sha256": "796db6e232825cd7e5216e39ea5c35bce42fb6eeb24882cc183a4895f289d475", + "metrics_sha256": "90e4eacb0ea0ab07cede22c32dbcd37475f160cedb033c1883cad9b520e28ad8", + "monitoring_export_failures": 0, + "parent_attempt_id": "9eadb1fec2a8447cabe294d62ba41c95", + "parent_sha256": "97d836eb1100c7b2ddd94fcbb2020029af62a5f69d41815a4b1c030d5f1505c4", + "sha256": "cea1cb3a74a2022e9e53aa290bcd2cea2ab4aadfdbc84b045d62f8e0a3ad3aac", + "size_bytes": 1983564759, + "step": 2, + "telemetry_sha256": "ad50219ce264e0196574c0c4eb7c88d4c572ba8bbf2d2ca7412c343088df46b3" + }, + "private_terminal_sha256": "4f66ff86b3d966746fc974be67084255ae79f722964734db716328528c1e0182", + "protocol_runner_sha256": "b1574fcc46068a4348af397c220b492925ea0d120e83957ca0ce4c695778daae", + "cuda_allocator_fraction": 0.7, + "reloads": { + "step1": { + "activation_checkpointing": false, + "checkpoint_sha256": "97d836eb1100c7b2ddd94fcbb2020029af62a5f69d41815a4b1c030d5f1505c4", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2933461504, + "cuda_peak_reserved_bytes": 3059744768, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 390, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 8, + "precision": "bf16", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 1 + }, + "step2": { + "activation_checkpointing": false, + "checkpoint_sha256": "cea1cb3a74a2022e9e53aa290bcd2cea2ab4aadfdbc84b045d62f8e0a3ad3aac", + "cuda_allocator_fraction": 0.7, + "cuda_matmul_allow_tf32": false, + "cuda_peak_allocated_bytes": 2933461504, + "cuda_peak_reserved_bytes": 3059744768, + "cudnn_allow_tf32": false, + "finite_parameter_tensors": 390, + "float32_matmul_precision": "highest", + "forward_tokens": 230, + "loaded_backbone_sha256": "88c142557820ccad55bb59756bfcfcf891de9cc6202816bd346445188a0ed342", + "logical_tokens": 230, + "logit_shapes": [ + [ + 2 + ], + [ + 2 + ], + [ + 10 + ] + ], + "lora_rank": 8, + "precision": "bf16", + "tokenizer_identity": "qwen-tokenizer-sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539", + "training_steps": 2 + } + } + } + ], + "scope": "Two independently initialized precision cells, each one-step generated native LoRA training, continuation of the same run to step 2, and complete finite reload forward at both steps. This is not identical-initialization precision comparison, optimizer equality against uninterrupted training, independent HF numerical parity, installed service qualification, long-context quality, or issue closure.", + "public_reproduction": "The private v3 mode orchestrator is not published by this receipt. Runner/source/config/checkpoint/telemetry hashes preserve provenance; no public runnable orchestration command is claimed." +} diff --git a/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.md b/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.md new file mode 100644 index 0000000..b810415 --- /dev/null +++ b/docs/evidence/native/noncheckpoint-rank8-mechanics-v10-20260929.md @@ -0,0 +1,30 @@ +# Rank-8 noncheckpoint CUDA mechanics — September 29, 2026 + +Refs #3; partial evidence only. The [portable receipt](noncheckpoint-rank8-mechanics-v10-20260929.json) +records separate FP32 and BF16 cells with actual pinned Qwen backbone and +tokenizer provenance. Activation checkpointing is disabled in both cells. + +Each cell initializes its own pointer head, completes generated short training +step 1, reloads the checkpoint for a complete finite forward, resumes the same +run/attempt lineage through step 2, and reloads again. Step-2 parent checkpoint +and attempt IDs point to step 1. All 390 parameter tensors are finite on reload; +the generated three-question request has 230 logical tokens. No monitoring +export failures were recorded. The CUDA allocator cap remains 70%. + +| Precision | Step-1 peak allocation | Step-2 peak allocation | +| --- | ---: | ---: | +| FP32 | 2,295,416,832 bytes | 2,300,203,008 bytes | +| BF16 | 4,246,000,640 bytes | 4,250,786,816 bytes | + +These are independently initialized cells. Loss values and memory observations +are individual measurements, not a matched precision accuracy/performance +comparison. Passing resume mechanics does not establish equality with an +uninterrupted run, installed serving, representative quality, long-context +training, or completion of the applicable native matrix. Full-weight rank-0 +cells remain separate, unclaimed work at this readback. + +The mode orchestrator remains private. Its v3 runner hash, source hashes, +configuration identities and immutable checkpoint/reload/telemetry receipt +hashes are retained; this document provides no public runnable mode command. +The separately published [FP32 numerical reproductions](FP32_REPRODUCTION.md) +cover different protocols and must not be presented as mode orchestration. diff --git a/docs/evidence/native/short-fp32-canonical-v10-20260929.json b/docs/evidence/native/short-fp32-canonical-v10-20260929.json new file mode 100644 index 0000000..fabc82e --- /dev/null +++ b/docs/evidence/native/short-fp32-canonical-v10-20260929.json @@ -0,0 +1,677 @@ +{ + "evidence_class": "pinned-actual-weight-short-fp32-canonical-v10-numerical-repair", + "source_parent": "e4822909680080bc300371782f6b439ff26fc8a8", + "source_worktree_dirty": true, + "source_file_sha256": { + "src/kev_laya/model.py": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "src/kev_laya/native_gradients.py": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "src/kev_laya/training.py": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde", + "src/kev_laya/execution.py": "41d7eaa6b3bf33c6e59c30fd2b36c7e086b29116179109a75dcbdc7d24d1756a" + }, + "checkpoint_sha256": "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9", + "precision": "fp32", + "device": "NVIDIA GeForce RTX 3060", + "torch_version": "2.10.0+cu128", + "transformers_version": "4.57.1", + "cuda_allocator_fraction": 0.7, + "tf32": false, + "derivative_version": "cuda-fp32-canonical-v10", + "attention_operator_version": "cuda-fp32-absolute-query-bands-fixed256-cpurope-v3", + "cache_plan_version": "cuda-double-kv-gqa-storage-v2", + "tolerances": { + "hidden_atol_and_rtol": 0.0001, + "logits_and_probabilities_atol_and_rtol": 1e-05, + "gradients_atol_and_rtol": 2e-05 + }, + "hf_oracle": { + "private_receipt_sha256": "db4a3b968ad3bab6aa5343d4b2c9a2696d864435573e868cb6b25485e93dfc7c", + "protocol_script_sha256": "bc69046d9a30bae20ef124dccbf7534675afa8e1b7297fed83825ec8ed744f26", + "tokens": 62, + "gradient_count": 96, + "all_gradients_passed": true, + "hidden": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_tolerance_ratio": 0.7061729431152344 + }, + "seam_cases": [ + { + "case": "short_full", + "private_receipt_sha256": "eefb9d93594c6a1cd1040cd386644b58dca7947c5123d9fc884c86e017ff281c", + "protocol_script_sha256": "d280f74c4b7879cab48d114a84ba229a86a31ae1f0de223e908a4d093e46848a", + "state_tokens": 200, + "branch_tokens": [ + 256, + 252 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.009706847369670868, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.009706847369670868, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + } + }, + { + "case": "short_medium_255", + "private_receipt_sha256": "ce1862a58a509e501789831ac4c0d16de5cb2f7bd9fe56591b87d682d15d9b0e", + "protocol_script_sha256": "d280f74c4b7879cab48d114a84ba229a86a31ae1f0de223e908a4d093e46848a", + "state_tokens": 255, + "branch_tokens": [ + 311, + 307 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.014025088399648666, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.014025088399648666, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + } + }, + { + "case": "short_medium_256", + "private_receipt_sha256": "9d177db52bfd6fafbee4812b0f197e0d61812a7bf8af5c5a405fd8a2aedbe74d", + "protocol_script_sha256": "d280f74c4b7879cab48d114a84ba229a86a31ae1f0de223e908a4d093e46848a", + "state_tokens": 256, + "branch_tokens": [ + 312, + 308 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.012803155928850174, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.012803155928850174, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + } + }, + { + "case": "medium_257", + "private_receipt_sha256": "ba227aa181bb4d770c96843b962a108fe19ecea01c5b30f095ecf02de55a2d98", + "protocol_script_sha256": "d280f74c4b7879cab48d114a84ba229a86a31ae1f0de223e908a4d093e46848a", + "state_tokens": 257, + "branch_tokens": [ + 313, + 309 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.022406483069062233, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.9073486328125e-06, + "max_tolerance_ratio": 0.022406483069062233, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + } + } + ], + "scope": "Generated numerical probes on a pinned two-step checkpoint, including 8191/8192 state-prefix boundaries and 10520-token state. Not representative quality, current-source optimizer/resume or serving qualification, exact-limit resource qualification, or issue closure.", + "boundary_and_long_cases": [ + { + "case": "boundary_8191", + "private_receipt_sha256": "64883f3358e6e15be074ed6bfd69db81b6803fafe8678848bafb60f086ac2794", + "protocol_script_sha256": "3ea5c5c86d6f8e1cb1e735726c462a1f1c32876a516c0df9613dea32d60413c4", + "checkpoint_sha256": "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9", + "source": { + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "diff_sha256": "0892caf43486ba068a8171daa1bc8971e7da4e323c2c3b0690c085741401eb3f", + "model_sha256": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients_sha256": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f" + }, + "state_tokens": 8191, + "branch_tokens": [ + 8247, + 8243 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 3.0517578125e-05, + "max_tolerance_ratio": 0.02111618034541607, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 3.0517578125e-05, + "max_tolerance_ratio": 0.02111618034541607, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + }, + "runs": { + "batched": { + "elapsed_seconds": 203.7953961000021, + "peak_allocated_bytes": 4398753280 + }, + "full": { + "elapsed_seconds": 199.58722319998196, + "peak_allocated_bytes": 4428107264 + }, + "serial": { + "elapsed_seconds": 192.29716380001628, + "peak_allocated_bytes": 4398180352 + } + }, + "scope": "Generated two-question actual-weight native numerical parity only; no optimizer update, resume, serving or representative quality claim." + }, + { + "case": "boundary_8192", + "private_receipt_sha256": "fdd4db8617e94a04e1b8aca26866368963b0f677a1db01f50dab024b7c6be68d", + "protocol_script_sha256": "3ea5c5c86d6f8e1cb1e735726c462a1f1c32876a516c0df9613dea32d60413c4", + "checkpoint_sha256": "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9", + "source": { + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "diff_sha256": "0892caf43486ba068a8171daa1bc8971e7da4e323c2c3b0690c085741401eb3f", + "model_sha256": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients_sha256": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f" + }, + "state_tokens": 8192, + "branch_tokens": [ + 8248, + 8244 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 3.0517578125e-05, + "max_tolerance_ratio": 0.010387285612523556, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 3.0517578125e-05, + "max_tolerance_ratio": 0.010387285612523556, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + }, + "runs": { + "batched": { + "elapsed_seconds": 198.8323976000247, + "peak_allocated_bytes": 4399217152 + }, + "full": { + "elapsed_seconds": 191.71707380001317, + "peak_allocated_bytes": 4428767232 + }, + "serial": { + "elapsed_seconds": 187.30125089999638, + "peak_allocated_bytes": 4400606720 + } + }, + "scope": "Generated two-question actual-weight native numerical parity only; no optimizer update, resume, serving or representative quality claim." + }, + { + "case": "long_10520", + "private_receipt_sha256": "a38130488e8edfdcedb63d682ee3cf8b695586cc9b75937baa25819464af35e2", + "protocol_script_sha256": "d4091a2a2a8f9b42be93e8a87d8a8451aa187afa2ef3ba48aade1efc93c2d6d2", + "checkpoint_sha256": "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9", + "source": { + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "diff_sha256": "0892caf43486ba068a8171daa1bc8971e7da4e323c2c3b0690c085741401eb3f" + }, + "state_tokens": 10520, + "branch_tokens": [ + 10576, + 10572 + ], + "comparisons": { + "batched_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.52587890625e-05, + "max_tolerance_ratio": 0.044013213366270065, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + }, + "batched_serial": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.q_proj.a" + }, + "serial_full": { + "failed_gradient_count": 0, + "failed_gradients": {}, + "gradient_count": 100, + "logits": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "passed": true, + "probabilities": { + "max_absolute": 0.0, + "max_tolerance_ratio": 0.0, + "passed": true + }, + "worst_gradient": { + "max_absolute": 1.52587890625e-05, + "max_tolerance_ratio": 0.044013213366270065, + "passed": true + }, + "worst_gradient_name": "backbone.layers.0.self_attn.v_proj.b" + } + }, + "runs": { + "batched": { + "elapsed_seconds": 296.1061069999996, + "peak_allocated_bytes": 4900734976 + }, + "full": { + "elapsed_seconds": 333.6611523999891, + "peak_allocated_bytes": 4899299328 + }, + "serial": { + "elapsed_seconds": 306.1350345000101, + "peak_allocated_bytes": 4901821440 + } + }, + "scope": "Generated two-question actual-weight native numerical parity only; no optimizer update, resume, serving or representative quality claim." + } + ], + "exact_limit_resource_failure": { + "private_receipt_sha256": "13d981534818bbb84cfc97e94b6ebac4cb00210b4399570436f3ba590b460f42", + "protocol_script_sha256": "e2c76bb41cae1b4c6002cb542f568f9c3598394026c4d5aa7f3916d21f296026", + "checkpoint_sha256": "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9", + "source": { + "commit": "e4822909680080bc300371782f6b439ff26fc8a8", + "diff_sha256": "0892caf43486ba068a8171daa1bc8971e7da4e323c2c3b0690c085741401eb3f", + "model_sha256": "794c9a648d5f40cb461a9631204a5146d2791eea663eeae4abccdad8cf926898", + "native_gradients_sha256": "5c8f5b8876e04642d093b9878d3020b4e40438345540966cc1204cc81a820c4f", + "training_sha256": "0bdc29c53a98d9b5d54145c871d14ff24c0f2cee8c28a1e117d6930ab1210cde" + }, + "precision": "fp32", + "cuda_allocator_fraction": 0.7, + "max_branch_tokens": 32768, + "aggregate_tokens": 65536, + "passed_resource_probe": false, + "stage": "forward", + "error_type": "OutOfMemoryError", + "worker_elapsed_seconds": 148.09840970000369, + "peak_allocated_bytes": 8382291968, + "peak_reserved_bytes": 9017753600, + "scope": "Exact-limit training resource-only probe exhausted allocator cap during forward. Backward did not run. No inference numerical acceptance or quality claim." + } +} diff --git a/docs/evidence/native/short-fp32-canonical-v10-20260929.md b/docs/evidence/native/short-fp32-canonical-v10-20260929.md new file mode 100644 index 0000000..3a4e407 --- /dev/null +++ b/docs/evidence/native/short-fp32-canonical-v10-20260929.md @@ -0,0 +1,46 @@ +# FP32 canonical numerical repair — September 29, 2026 + +Refs #3 and #4. This is partial numerical evidence on pinned Qwen weights, +not closure of either acceptance issue. The portable [receipt](short-fp32-canonical-v10-20260929.json) +records checkpoint, source-file, protocol-script and private-result hashes. +[Public reproduction commands](FP32_REPRODUCTION.md) retain the generated +requests, fixed precision settings and numerical tolerances. +The source parent is recorded with a dirty candidate tree; delivery commit and +post-merge verification must be recorded separately by the delivery owner. + +CUDA FP32 projections use fixed 64-row calls. Attention dispatch follows absolute +query bands: positions 0–255 use fixed 256-row eager arithmetic, positions +256–8191 use SDPA, and later positions use grouped fused SDPA. Canonical CPU +RoPE frequencies are stored as nonpersistent integer bits to survive dtype moves. +Shared K/V and LoRA accumulation use double precision; the short query derivative +retains the rounded eager graph. CUDA cache planning accounts for eight-byte +materialized GQA storage, and its version participates in resume identity for +both FP32 and BF16. Checkpoint schema and telemetry v1 are preserved. + +The independent Transformers 4.57.1 eager oracle passes all 96 Q/V LoRA +parameter gradients at 62 tokens; hidden states are exact. Worst gradient +combined-tolerance ratio is 0.706173. Four generated two-question cases cover +short full rows, prefix lengths 255 and 256 crossing the short/medium band, and +prefix length 257. All 100 trainable gradients pass in each batched/serial/full +comparison; tolerances remain 1e-5 for logits/probabilities and 2e-5 for gradients. +These tests use the pinned two-step checkpoint, not a newly trained quality model. + +Earlier failed candidates remain preserved in the private execution ledger, +including v9's prefix-255 one-gradient failure (receipt SHA-256 +`302f96c18369879e2ae96ae975b6664e9f9351c406f209a6fa39866393d5146e`). +No passing historical long run is promoted to evidence for this candidate. +Fresh current-source 8191/8192 state-prefix boundary cases and the 10520-token +state case pass all 100 gradients in each comparison, with exact logits and +probabilities. Their checkpoint/source/protocol hashes, timings and peaks are +recorded separately in the receipt. These are generated forward/backward probes. +Exact-limit resources, optimizer/resume and serving qualification remain +separate gates. Generated +probes establish no representative quality or Jev benefit; issue #4's diagnostic +marker threshold remains unmet. Overwatch remains archived by user instruction. + +The separate FP32 exact-limit training resource probe failed with CUDA OOM during +forward on the RTX 3060 under the unchanged 70% allocator cap. The frozen row +has maximum branch length 32768 and aggregate length 65536. Worker elapsed time +was 148.098410 seconds; peak allocation/reservation was 8,382,291,968 / +9,017,753,600 bytes. Backward did not run. This measured hardware limitation is +not an inference numerical failure or a passing exact-limit training gate. diff --git a/docs/research/layev-feature-mapping.json b/docs/research/layev-feature-mapping.json index 5f2473e..20db32c 100644 --- a/docs/research/layev-feature-mapping.json +++ b/docs/research/layev-feature-mapping.json @@ -459,8 +459,8 @@ "type_safe_primary_source": "https://docs.typesafe.ai/introduction", "classification": "comparison_feature", "implemented": "source_present", - "tested": "CPU suite 857 passed on 2026-09-29; actual-weight two-question 10k BF16 batched/serial/full numerical gate passed", - "native_validated": "partial; actual-weight two-question 10k BF16 outputs and 100 gradients passed; full native matrix remains open", + "tested": "CPU suite 858 passed on 2026-09-29; 98 affected CUDA tests and five resume-version cases passed; actual-weight v10 FP32 short HF adapter and four cached/batched/full seam gates passed", + "native_validated": "partial; prior two-question 10k BF16 numerical receipt retained; v10 short FP32 HF 96-adapter and four 100-gradient seam gates plus fresh 8191/8192 and 10520-token state numerical gates passed; not complete issue 3", "quality_measured": "unknown", "Jev_compared": "unknown", "Layev_source_paths": [ @@ -471,7 +471,8 @@ "tests/test_parallel_questions.py" ], "evidence_paths": [ - "docs/evidence/native/long-bf16-gradient-10k-20260929.json" + "docs/evidence/native/long-bf16-gradient-10k-20260929.json", + "docs/evidence/native/short-fp32-canonical-v10-20260929.json" ], "issues": [ "https://github.com/CompleteDotTech/layev/issues/9", @@ -512,8 +513,8 @@ "type_safe_primary_source": "https://docs.typesafe.ai/models", "classification": "comparison_feature", "implemented": "source_present", - "tested": "CPU suite 857 passed on 2026-09-29; actual-weight two-question 10k BF16 cached/full numerical gate passed", - "native_validated": "partial; one actual-weight two-question 10k BF16 shared-prefix gradient gate passed", + "tested": "CPU suite 858 passed on 2026-09-29; 98 affected CUDA tests and five resume-version cases passed; actual-weight v10 FP32 short HF adapter and four cached/batched/full seam gates passed", + "native_validated": "partial; prior two-question 10k BF16 numerical receipt retained; v10 short FP32 HF 96-adapter and four 100-gradient seam gates plus fresh 8191/8192 and 10520-token state numerical gates passed; not complete issue 3", "quality_measured": "unknown", "Jev_compared": "unknown", "Layev_source_paths": [ @@ -523,7 +524,8 @@ "tests/test_parallel_questions.py" ], "evidence_paths": [ - "docs/evidence/native/long-bf16-gradient-10k-20260929.json" + "docs/evidence/native/long-bf16-gradient-10k-20260929.json", + "docs/evidence/native/short-fp32-canonical-v10-20260929.json" ], "issues": [ "https://github.com/CompleteDotTech/layev/issues/9", diff --git a/scripts/verify_feature_matrix.py b/scripts/verify_feature_matrix.py index 96c3e18..d4f7858 100644 --- a/scripts/verify_feature_matrix.py +++ b/scripts/verify_feature_matrix.py @@ -3,6 +3,7 @@ import hashlib import json +import re from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -31,6 +32,84 @@ def verify() -> None: assert comparison["passed"] is True assert comparison["gradient_count"] == 100 assert comparison["failed_gradient_count"] == 0 + fp32_receipt_path = "docs/evidence/native/short-fp32-canonical-v10-20260929.json" + fp32 = json.loads((ROOT / fp32_receipt_path).read_text(encoding="utf-8")) + assert fp32["precision"] == "fp32" + assert fp32["derivative_version"] == "cuda-fp32-canonical-v10" + assert fp32["attention_operator_version"] == "cuda-fp32-absolute-query-bands-fixed256-cpurope-v3" + assert fp32["cache_plan_version"] == "cuda-double-kv-gqa-storage-v2" + assert fp32["transformers_version"] == "4.57.1" + assert fp32["cuda_allocator_fraction"] == 0.7 and fp32["tf32"] is False + assert fp32["tolerances"] == {"hidden_atol_and_rtol": 1e-4, + "logits_and_probabilities_atol_and_rtol": 1e-5, + "gradients_atol_and_rtol": 2e-5} + assert fp32["checkpoint_sha256"] == "99f797ffd0521adbee964d130d73eb381c9b6bd3b7896ce61b5007886d19ccf9" + hashes = [fp32["checkpoint_sha256"], *fp32["source_file_sha256"].values()] + assert set(fp32["source_file_sha256"]) == {"src/kev_laya/model.py", + "src/kev_laya/native_gradients.py", "src/kev_laya/training.py", "src/kev_laya/execution.py"} + assert re.fullmatch(r"[0-9a-f]{40}", fp32["source_parent"]) + hf = fp32["hf_oracle"] + hashes.extend([hf["private_receipt_sha256"], hf["protocol_script_sha256"]]) + assert hf["gradient_count"] == 96 and hf["all_gradients_passed"] is True + assert hf["tokens"] == 62 and hf["hidden"]["passed"] is True + assert 0 <= hf["hidden"]["max_tolerance_ratio"] <= 1 + assert hf["hidden"]["max_absolute"] == 0 + assert 0 <= hf["worst_gradient_tolerance_ratio"] <= 1 + cases = fp32["seam_cases"] + assert len(cases) == 4 + assert {case["case"] for case in cases} == {"short_full", "short_medium_255", + "short_medium_256", "medium_257"} + for case in cases: + hashes.extend([case["private_receipt_sha256"], case["protocol_script_sha256"]]) + assert len(case["branch_tokens"]) == 2 + assert 1 <= case["state_tokens"] <= 257 + assert all(case["state_tokens"] < length < 8192 for length in case["branch_tokens"]) + if case["case"] == "short_full": + assert max(case["branch_tokens"]) <= 256 + else: + assert case["state_tokens"] == {"short_medium_255": 255, + "short_medium_256": 256, "medium_257": 257}[case["case"]] + if case["case"].startswith("short_medium"): + assert min(case["branch_tokens"]) > 256 + assert set(case["comparisons"]) == {"batched_serial", "serial_full", "batched_full"} + for comparison in case["comparisons"].values(): + assert comparison["passed"] is True + assert comparison["gradient_count"] == 100 + assert comparison["failed_gradient_count"] == 0 and comparison["failed_gradients"] == {} + for metric in ("logits", "probabilities", "worst_gradient"): + assert comparison[metric]["passed"] is True + assert 0 <= comparison[metric]["max_tolerance_ratio"] <= 1 + extended = fp32["boundary_and_long_cases"] + assert len(extended) == 3 + assert {case["case"] for case in extended} == {"boundary_8191", "boundary_8192", "long_10520"} + for case in extended: + assert case["checkpoint_sha256"] == fp32["checkpoint_sha256"] + hashes.extend([case["private_receipt_sha256"], case["protocol_script_sha256"], case["source"]["diff_sha256"]]) + assert case["source"]["commit"] == fp32["source_parent"] + assert case["state_tokens"] == {"boundary_8191": 8191, "boundary_8192": 8192, "long_10520": 10520}[case["case"]] + assert len(case["branch_tokens"]) == 2 + assert all(case["state_tokens"] < length <= 32768 for length in case["branch_tokens"]) + assert set(case["comparisons"]) == {"batched_serial", "serial_full", "batched_full"} + for comparison in case["comparisons"].values(): + assert comparison["passed"] is True and comparison["gradient_count"] == 100 + assert comparison["failed_gradient_count"] == 0 and comparison["failed_gradients"] == {} + for metric in ("logits", "probabilities", "worst_gradient"): + assert comparison[metric]["passed"] is True + assert 0 <= comparison[metric]["max_tolerance_ratio"] <= 1 + assert comparison["logits"]["max_absolute"] == comparison["probabilities"]["max_absolute"] == 0 + assert set(case["runs"]) == {"batched", "serial", "full"} + for run in case["runs"].values(): + assert run["elapsed_seconds"] > 0 and run["peak_allocated_bytes"] > 0 + failure = fp32["exact_limit_resource_failure"] + assert failure["passed_resource_probe"] is False + assert failure["stage"] == "forward" and failure["error_type"] == "OutOfMemoryError" + assert failure["max_branch_tokens"] == 32768 and failure["aggregate_tokens"] == 65536 + assert failure["precision"] == "fp32" and failure["cuda_allocator_fraction"] == 0.7 + assert failure["checkpoint_sha256"] == fp32["checkpoint_sha256"] + hashes.extend([failure["private_receipt_sha256"], failure["protocol_script_sha256"]]) + assert failure["peak_reserved_bytes"] >= failure["peak_allocated_bytes"] > 0 + assert failure["worker_elapsed_seconds"] > 0 + assert all(re.fullmatch(r"[0-9a-f]{64}", digest) for digest in hashes) for source, row in zip(original["features"], mapping["features"], strict=True): for key in ("id", "title", "definition", "group"): assert source[key] == row[key], (source["id"], key) @@ -38,9 +117,9 @@ def verify() -> None: assert row["classification"] in {"comparison_feature", "Jev_absent_non_gap", "service_policy"} assert row["implemented"] in {"source_present", "unknown"} if row["id"] in {"parallel", "shared"}: - assert row["tested"].startswith("CPU suite 857 passed on 2026-09-29;") + assert row["tested"].startswith("CPU suite 858 passed on 2026-09-29;") assert row["native_validated"].startswith("partial;") - assert row["evidence_paths"] == [native_receipt_path] + assert row["evidence_paths"] == [native_receipt_path, fp32_receipt_path] assert ("not complete issue 3" in row["limitations"] or "full native" in row["limitations"]) else: diff --git a/scripts/verify_native_fp32_gradients.py b/scripts/verify_native_fp32_gradients.py new file mode 100644 index 0000000..cc513fd --- /dev/null +++ b/scripts/verify_native_fp32_gradients.py @@ -0,0 +1,248 @@ +"""Independent, opt-in short FP32 actual-weight HF gradient comparison. + +Preparation only: this script runs CUDA when explicitly invoked. It does not +change the checkpoint, source tree, or the frozen acceptance tolerances. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from dataclasses import asdict +from pathlib import Path + +import torch +from torch import nn +from torch.nn import functional as F + +SOURCE = Path(__file__).resolve().parents[1] +ALLOCATOR_FRACTION = 0.70 +CHECKPOINT = OUTPUT = None +EXPECTED_CHECKPOINT_SHA256 = None +sys.path.insert(0, str(SOURCE / "src")) + +from kev_laya.checkpoint import load_checkpoint # noqa: E402 +from kev_laya.encoding import Limits, encode_request # noqa: E402 +from kev_laya.model import LoRALinear # noqa: E402 +from kev_laya.native_validation import (PROTOCOL, merged_backbone, + require_reference_version) # noqa: E402 +from kev_laya.schema import SystemOneRequest # noqa: E402 + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class TrainableReferenceLoRA(nn.Module): + """HF Qwen projection with independent parameters and native operation order.""" + + def __init__(self, base: nn.Linear, native: LoRALinear): + super().__init__() + self.base = base + self.a = nn.Parameter(native.a.detach().clone()) + self.b = nn.Parameter(native.b.detach().clone()) + self.scale = native.scale + + def forward(self, x): + return self.base(x) + F.linear(F.linear(x, self.a), self.b) * self.scale + + +def compare(a: torch.Tensor, b: torch.Tensor, atol: float, rtol: float) -> dict: + if a.shape != b.shape or a.dtype != b.dtype or a.device != b.device: + raise ValueError("oracle comparison shape, dtype, or device mismatch") + if not bool(torch.isfinite(a).all() and torch.isfinite(b).all()): + raise ValueError("oracle comparison contains nonfinite values") + absolute = (a - b).abs() + denominator = atol + rtol * b.abs() + ratio = absolute / denominator + return {"passed": bool(torch.all(absolute <= denominator)), + "max_absolute": float(absolute.detach().max()), + "max_tolerance_ratio": float(ratio.detach().max())} + + +def source_fingerprint(): + import subprocess + return {"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=SOURCE, text=True).strip(), + "worktree_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=SOURCE, text=True).strip()), + "files": {name: hashlib.sha256((SOURCE / name).read_bytes()).hexdigest() + for name in ("src/kev_laya/model.py", "src/kev_laya/native_gradients.py", + "src/kev_laya/training.py", "src/kev_laya/execution.py")}} + + +def main() -> None: + global CHECKPOINT, OUTPUT, EXPECTED_CHECKPOINT_SHA256 + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + CHECKPOINT, OUTPUT = args.checkpoint, args.out + EXPECTED_CHECKPOINT_SHA256 = args.expected_checkpoint_sha256 + if OUTPUT.exists() or OUTPUT.with_suffix(OUTPUT.suffix + ".failure.json").exists(): + raise FileExistsError("output or failure receipt already exists") + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + if OUTPUT.exists(): + raise FileExistsError(OUTPUT) + if sha256(CHECKPOINT) != EXPECTED_CHECKPOINT_SHA256: + raise ValueError("checkpoint hash differs from frozen pin") + source_identity = source_fingerprint() + source_commit = source_identity["commit"] + version = require_reference_version() + import transformers + if transformers.__version__ != version: + raise ValueError("imported Transformers differs from pinned distribution") + if not torch.cuda.is_available(): + raise RuntimeError("CUDA is required for this native probe") + torch.cuda.set_per_process_memory_fraction(ALLOCATOR_FRACTION, device=0) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.set_float32_matmul_precision("highest") + if (torch.backends.cuda.matmul.allow_tf32 or torch.backends.cudnn.allow_tf32 or + torch.get_float32_matmul_precision() != "highest"): + raise RuntimeError("FP32 computation settings were not applied") + free_before, total_memory = torch.cuda.mem_get_info(0) + torch.cuda.reset_peak_memory_stats(0) + from transformers import Qwen2Config, Qwen2Model + + model, tokenizer, point = load_checkpoint(CHECKPOINT, "cuda:0") + if not model.native_weights_loaded or model.training_steps < 1 or model.cfg.lora_rank < 1: + raise ValueError("checkpoint is not a trained native LoRA checkpoint") + if point["training_state"]["step"] < 1: + raise ValueError("checkpoint has no training exposure") + request = SystemOneRequest( + state="A short real-weight reference state.", + questions={"q": {"type": "noul", "instructions": "Is the state short?"}}, + model="kev-laya-preview") + encoded = encode_request(request, tokenizer, Limits()) + ids = encoded.state + encoded.branches[0].ids + if not 1 <= len(ids) <= 256: + raise ValueError("probe must stay on the short attention path") + + fields = asdict(model.cfg) + keep = {key: value for key, value in fields.items() if key in { + "vocab_size", "hidden_size", "intermediate_size", "num_hidden_layers", + "num_attention_heads", "num_key_value_heads", "max_position_embeddings", + "rope_theta", "rms_norm_eps"}} + config = Qwen2Config(**keep, hidden_act="silu", use_sliding_window=False, + attention_dropout=0.0) + config._attn_implementation = "eager" + oracle = Qwen2Model(config).to("cuda:0") + oracle.load_state_dict(merged_backbone(model, merge_lora=False), strict=True) + oracle.requires_grad_(False) + native_adapters = {} + reference_adapters = {} + for index, (native_layer, reference_layer) in enumerate( + zip(model.backbone.layers, oracle.layers, strict=True)): + for projection in ("q_proj", "v_proj"): + native = getattr(native_layer.self_attn, projection) + if not isinstance(native, LoRALinear): + raise ValueError("expected native LoRA at every Q/V projection") + name = f"layers.{index}.self_attn.{projection}" + reference = TrainableReferenceLoRA( + getattr(reference_layer.self_attn, projection), native) + setattr(reference_layer.self_attn, projection, reference) + native_adapters[name] = native + reference_adapters[name] = reference + if not native_adapters or not any(bool(torch.count_nonzero(a.b)) for a in native_adapters.values()): + raise ValueError("no trained adapter B weights found") + + model.train() + oracle.eval() + model.zero_grad(set_to_none=True) + oracle.zero_grad(set_to_none=True) + native_hidden, _ = model.backbone(ids) + reference_hidden = oracle( + input_ids=torch.tensor(ids, device="cuda:0")[None], + use_cache=False).last_hidden_state[0] + expected_shape = (len(ids), model.cfg.hidden_size) + if native_hidden.shape != expected_shape or reference_hidden.shape != expected_shape: + raise ValueError("incomplete hidden-state tensors") + hidden_result = compare(native_hidden, reference_hidden, + PROTOCOL["hf_hidden_atol"], PROTOCOL["hf_hidden_rtol"]) + # A nonconstant deterministic VJP excites positions and all hidden channels. + row = torch.arange(len(ids), device="cuda:0", dtype=torch.float32)[:, None] + column = torch.arange(model.cfg.hidden_size, device="cuda:0", dtype=torch.float32)[None, :] + upstream = (torch.sin(row * 0.017 + column * 0.013) + + torch.cos(row * 0.031 - column * 0.007)) / math.sqrt(len(ids) * model.cfg.hidden_size) + if not bool(torch.isfinite(upstream).all()): + raise ValueError("nonfinite upstream") + torch.autograd.backward(native_hidden, upstream) + torch.autograd.backward(reference_hidden, upstream) + gradient_results = {} + for name in sorted(native_adapters): + native, reference = native_adapters[name], reference_adapters[name] + for suffix in ("a", "b"): + native_gradient = getattr(native, suffix).grad + reference_gradient = getattr(reference, suffix).grad + if native_gradient is None or reference_gradient is None: + raise ValueError(f"missing adapter gradient: {name}.{suffix}") + gradient_results[f"{name}.{suffix}"] = compare( + native_gradient, reference_gradient, + PROTOCOL["gradient_atol"], PROTOCOL["gradient_rtol"]) + expected_count = 4 * model.cfg.num_hidden_layers + if len(gradient_results) != expected_count: + raise ValueError("incomplete adapter gradient set") + free_after, _ = torch.cuda.mem_get_info(0) + receipt = { + "evidence_class": "short-actual-weight-fp32-hf-backbone-adapter-gradient-oracle", + "source_commit": source_commit, + "source_worktree_dirty": bool(__import__("subprocess").check_output( + ["git", "status", "--porcelain"], cwd=SOURCE, text=True).strip()), + "model_sha256": sha256(SOURCE / "src/kev_laya/model.py"), + "native_gradients_sha256": sha256(SOURCE / "src/kev_laya/native_gradients.py"), + "script_sha256": sha256(Path(__file__)), + "checkpoint_sha256": EXPECTED_CHECKPOINT_SHA256, + "transformers_version": version, + "torch_version": torch.__version__, + "device": torch.cuda.get_device_name(0), + "cuda_allocator_fraction": ALLOCATOR_FRACTION, + "cuda_matmul_allow_tf32": torch.backends.cuda.matmul.allow_tf32, + "cudnn_allow_tf32": torch.backends.cudnn.allow_tf32, + "float32_matmul_precision": torch.get_float32_matmul_precision(), + "cuda_total_memory_bytes": total_memory, + "cuda_free_before_bytes": free_before, + "cuda_free_after_bytes": free_after, + "cuda_peak_allocated_bytes": torch.cuda.max_memory_allocated(0), + "cuda_peak_reserved_bytes": torch.cuda.max_memory_reserved(0), + "precision": "fp32", + "attention_implementation": "transformers-Qwen2Model-eager", + "adapter_mapping": "unmerged-Q/V-two-linear-LoRA", + "training_steps": model.training_steps, + "tokens": len(ids), + "hidden": hidden_result, + "gradient_count": len(gradient_results), + "gradients": gradient_results, + "worst_gradient_name": max(gradient_results, key=lambda key: gradient_results[key]["max_tolerance_ratio"]), + "passed": hidden_result["passed"] and all(item["passed"] for item in gradient_results.values()), + "interpretation": "one short actual-weight FP32 backbone VJP; not long-context, exact-limit, or task-quality evidence", + } + if source_fingerprint() != source_identity: + raise ValueError("source changed during oracle") + receipt["source"] = source_identity + with OUTPUT.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(receipt, indent=2, sort_keys=True, allow_nan=False) + "\n") + print(json.dumps({"receipt": str(OUTPUT), "passed": receipt["passed"], + "gradient_count": receipt["gradient_count"], + "worst_gradient_name": receipt["worst_gradient_name"]}, allow_nan=False)) + if not receipt["passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + if OUTPUT is not None and not OUTPUT.exists(): + failure = OUTPUT.with_suffix(OUTPUT.suffix + ".failure.json") + if not failure.exists(): + with failure.open("x", encoding="utf-8") as stream: + json.dump({"passed": False, "stage": "oracle", "error_type": type(exc).__name__, + "checkpoint_sha256": EXPECTED_CHECKPOINT_SHA256}, stream) + stream.write("\n") + raise diff --git a/scripts/verify_native_fp32_seams.py b/scripts/verify_native_fp32_seams.py new file mode 100644 index 0000000..facd19f --- /dev/null +++ b/scripts/verify_native_fp32_seams.py @@ -0,0 +1,332 @@ +"""Opt-in actual-weight FP32 short-attention dispatch seam diagnostic.""" +from __future__ import annotations + +import hashlib +import argparse +import json +import math +import subprocess +import sys +import time +from pathlib import Path + +import torch + +SOURCE = Path(__file__).resolve().parents[1] +ARTIFACTS = CHECKPOINT = None +CHECKPOINT_SHA = None +OWNS_OUTPUT_DIRECTORY = False +OUTPUT_STEM = "fp32-seam" +sys.path.insert(0, str(SOURCE / "src")) + +from kev_laya.checkpoint import load_checkpoint # noqa: E402 +from kev_laya.encoding import Limits, encode_request # noqa: E402 +from kev_laya.io import atomic_json, sha256_file # noqa: E402 +from kev_laya.native_validation import PROTOCOL, require_reference_version # noqa: E402 +from kev_laya.objectives import ObjectiveConfig, objective # noqa: E402 +from kev_laya.schema import SystemOneRequest # noqa: E402 + + +def compare(left, right, *, atol, rtol): + if len(left) != len(right) or not left: + raise ValueError("incomplete comparison") + maximum = ratio = 0.0 + for a, b in zip(left, right, strict=True): + if a.shape != b.shape or a.dtype != b.dtype or not bool(torch.isfinite(a).all() and torch.isfinite(b).all()): + raise ValueError("nonfinite, dtype, or shape-mismatched comparison") + delta = (a - b).abs() + threshold = atol + rtol * b.abs() + maximum = max(maximum, float(delta.max())) + ratio = max(ratio, float((delta / threshold).max())) + return {"max_absolute": maximum, "max_tolerance_ratio": ratio, "passed": ratio <= 1} + + +def request_with_filler(n, suffix_filler=0): + return SystemOneRequest( + state={"route": "alpha", "urgency": 2, "active": True, "context": "x " * n}, + questions={ + "choice": {"type": "choice", "instructions": "Choose the listed route using the state." + " x" * suffix_filler, + "criteria": {"alpha": "A route", "beta": "B route", "gamma": "C route"}}, + "noul": {"type": "noul", "instructions": "Is the state explicitly marked active?" + " x" * suffix_filler}, + }, model="kev-laya-preview") + + +def exact_encoding(tokenizer, desired, suffix_filler=0): + limits = Limits(branch=32768, aggregate=65536) + low, high = 0, 20000 + while low < high: + middle = (low + high) // 2 + size = len(encode_request(request_with_filler(middle, suffix_filler), tokenizer, limits).state) + if size < desired: + low = middle + 1 + else: + high = middle + for n in range(max(0, low - 32), min(20000, low + 32) + 1): + encoded = encode_request(request_with_filler(n, suffix_filler), tokenizer, limits) + if len(encoded.state) == desired: + if len(encoded.branches) != 2: + raise ValueError("two complete branches required") + return encoded, n + raise ValueError(f"exact {desired}-token state unavailable with frozen filler") + + +def largest_short_full_case(tokenizer): + # The serialized question suffix is not empty. A 254/255-token state + # cannot yield a <=256 full row unless both suffixes fit in 1-2 tokens. + # Choose the highest representable state whose two actual rows are short. + sample, _ = exact_encoding(tokenizer, 200) + upper = min(255, 256 - max(len(branch.ids) for branch in sample.branches)) + for desired in range(upper, 0, -1): + try: + encoded, count = exact_encoding(tokenizer, desired) + except ValueError: + continue + lengths = [desired + len(branch.ids) for branch in encoded.branches] + if max(lengths) <= 256: + return encoded, count + raise ValueError("no representable two-branch short full-row case") + + +def long_suffix_case(tokenizer, desired=255): + low, high = 0, 12000 + while low < high: + middle = (low + high) // 2 + encoded, _ = exact_encoding(tokenizer, desired, middle) + if min(desired + len(branch.ids) for branch in encoded.branches) <= 8192: + low = middle + 1 + else: + high = middle + for count in range(max(0, low - 8), min(12000, low + 8) + 1): + encoded, state_filler = exact_encoding(tokenizer, desired, count) + lengths = [desired + len(branch.ids) for branch in encoded.branches] + if min(lengths) > 8192: + return encoded, state_filler, count + raise ValueError("cannot construct bounded >8192 full rows") + + +def run_case(model, encoded, filler_count, source_identity, free_before, total_memory, + *, case_name, suffix_filler): + state_tokens = len(encoded.state) + receipt_path = ARTIFACTS / f"{OUTPUT_STEM}-{case_name}.json" + progress_path = ARTIFACTS / f"{OUTPUT_STEM}-{case_name}.progress.json" + if receipt_path.exists() or progress_path.exists(): + raise FileExistsError(f"existing seam artifacts for {case_name}") + def progress(stage, **extra): + atomic_json(progress_path, {"case_name": case_name, "state_tokens": state_tokens, "stage": stage, + "source": source_identity, **extra}) + print({"case_name": case_name, "state_tokens": state_tokens, "stage": stage, **extra}, flush=True) + kinds = [branch.question.type for branch in encoded.branches] + targets = [] + for branch in encoded.branches: + n = len(branch.option_ends) + target = [1 / n] * n + target[0] += 0.1 + target[-1] -= 0.1 + targets.append(target) + results = {} + progress("encoded", filler_count=filler_count, suffix_filler=suffix_filler, + branch_tokens=[state_tokens + len(b.ids) for b in encoded.branches]) + try: + for mode, kwargs in (("batched", {}), ("serial", {"serial_reference": True}), + ("full", {"reference": True})): + progress("mode_running", mode=mode) + model.zero_grad(set_to_none=True) + torch.cuda.reset_peak_memory_stats(0) + torch.cuda.synchronize(0) + start = time.perf_counter() + logits, usage = model(encoded, **kwargs) + loss = objective(logits, targets, kinds, ObjectiveConfig(ordinal=0.1))[0] + if not bool(torch.isfinite(loss)): + raise ValueError(f"nonfinite {mode} loss") + logit_gradients = torch.autograd.grad(loss, logits) + for index, (branch_logits, branch_gradient) in enumerate( + zip(logits, logit_gradients, strict=True)): + branch_logits.backward(branch_gradient, retain_graph=index < len(logits) - 1) + torch.cuda.synchronize(0) + expected = {name for name, p in model.named_parameters() if p.requires_grad} + gradients = {name: p.grad.detach().float().cpu().clone() + for name, p in model.named_parameters() if p.requires_grad and p.grad is not None} + if (set(gradients) != expected or len(gradients) != 100 or + not all(bool(torch.isfinite(g).all()) for g in gradients.values())): + raise ValueError(f"missing/nonfinite {mode} gradients") + results[mode] = {"logits": [x.detach().float().cpu() for x in logits], + "gradients": gradients, "loss": float(loss.detach()), + "usage": usage, "elapsed_seconds": time.perf_counter() - start, + "peak_allocated_bytes": torch.cuda.max_memory_allocated(0), + "peak_reserved_bytes": torch.cuda.max_memory_reserved(0)} + progress("mode_complete", mode=mode, + elapsed_seconds=results[mode]["elapsed_seconds"], + peak_allocated_bytes=results[mode]["peak_allocated_bytes"]) + comparisons = {} + for first, second in (("batched", "serial"), ("serial", "full"), ("batched", "full")): + a, b = results[first], results[second] + gradients = {name: compare([a["gradients"][name]], [b["gradients"][name]], + atol=PROTOCOL["gradient_atol"], rtol=PROTOCOL["gradient_rtol"]) + for name in sorted(a["gradients"])} + worst = max(gradients, key=lambda name: gradients[name]["max_tolerance_ratio"]) + logits = compare(a["logits"], b["logits"], + atol=PROTOCOL["fp32_logit_atol"], rtol=PROTOCOL["fp32_logit_rtol"]) + probabilities = compare([x.softmax(-1) for x in a["logits"]], + [x.softmax(-1) for x in b["logits"]], + atol=PROTOCOL["probability_atol"], rtol=PROTOCOL["probability_rtol"]) + comparisons[f"{first}_{second}"] = { + "logits": logits, "probabilities": probabilities, + "gradient_count": len(gradients), + "failed_gradient_count": sum(not row["passed"] for row in gradients.values()), + "failed_gradients": {name: row for name, row in gradients.items() if not row["passed"]}, + "worst_gradient_name": worst, "worst_gradient": gradients[worst], + "passed": logits["passed"] and probabilities["passed"] and + all(row["passed"] for row in gradients.values())} + current_identity = source_fingerprint() + if current_identity != source_identity: + raise ValueError("source changed during boundary case") + free_after, _ = torch.cuda.mem_get_info(0) + receipt = {"evidence_class": "actual-weight-fp32-short-attention-dispatch-seam-parity", + "source": source_identity, "script_sha256": sha256_file(Path(__file__)), + "checkpoint_sha256": CHECKPOINT_SHA, "torch_version": torch.__version__, + "device": torch.cuda.get_device_name(0), "precision": "fp32", + "cuda_allocator_fraction": 0.70, + "cuda_matmul_allow_tf32": torch.backends.cuda.matmul.allow_tf32, + "cudnn_allow_tf32": torch.backends.cudnn.allow_tf32, + "float32_matmul_precision": torch.get_float32_matmul_precision(), + "cuda_total_memory_bytes": total_memory, "cuda_free_before_bytes": free_before, + "cuda_free_after_bytes": free_after, + "case_name": case_name, "state_tokens": state_tokens, + "filler_count": filler_count, "suffix_filler": suffix_filler, + "logical_tokens": encoded.logical_tokens, + "branch_tokens": [state_tokens + len(b.ids) for b in encoded.branches], + "training_steps": model.training_steps, + "runs": {name: {key: value for key, value in row.items() + if key not in ("logits", "gradients")} + for name, row in results.items()}, + "comparisons": comparisons, + "passed": all(row["passed"] for row in comparisons.values()), + "interpretation": "one synthetic attention-dispatch seam case; not exact-limit or quality evidence"} + json.dumps(receipt, allow_nan=False) + atomic_json(receipt_path, receipt) + progress("complete", receipt=str(receipt_path), receipt_sha256=sha256_file(receipt_path), + passed=receipt["passed"]) + return receipt + except BaseException as exc: + progress("failed", error_type=type(exc).__name__, error=str(exc)) + failure_path = ARTIFACTS / f"{OUTPUT_STEM}-{case_name}.failure.json" + if failure_path.exists(): + raise FileExistsError(failure_path) from exc + atomic_json(failure_path, { + "evidence_class": "actual-weight-fp32-short-attention-seam-probe-failure", + "source": source_identity, "script_sha256": sha256_file(Path(__file__)), + "checkpoint_sha256": CHECKPOINT_SHA, "case_name": case_name, + "state_tokens": state_tokens, + "stage": "failed", "error_type": type(exc).__name__, "error": str(exc), + "completed_modes": sorted(results), "passed": False}) + return {"state_tokens": state_tokens, "passed": False, + "failure_receipt": str(failure_path)} + + +def source_fingerprint(): + import subprocess + return {"commit": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=SOURCE, text=True).strip(), + "worktree_dirty": bool(subprocess.check_output(["git", "status", "--porcelain"], cwd=SOURCE, text=True).strip()), + "files": {name: hashlib.sha256((SOURCE / name).read_bytes()).hexdigest() + for name in ("src/kev_laya/model.py", "src/kev_laya/native_gradients.py", + "src/kev_laya/training.py", "src/kev_laya/execution.py")}} + + +def main(): + global CHECKPOINT, CHECKPOINT_SHA, ARTIFACTS, OWNS_OUTPUT_DIRECTORY + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--cases", nargs="+", choices=( + "short_full", "short_medium_255", "short_medium_256", "medium_257", "short_long_255", "boundary_8191", "boundary_8192", "long_10520"), + default=["short_full", "short_medium_255", "short_medium_256", "medium_257"]) + args = parser.parse_args() + CHECKPOINT, CHECKPOINT_SHA, ARTIFACTS = args.checkpoint, args.expected_checkpoint_sha256, args.out_dir + ARTIFACTS.mkdir(parents=True, exist_ok=False) + OWNS_OUTPUT_DIRECTORY = True + require_reference_version() + if sha256_file(CHECKPOINT) != CHECKPOINT_SHA: + raise ValueError("checkpoint identity changed") + if not torch.cuda.is_available(): + raise RuntimeError("CUDA device unavailable") + source_identity = source_fingerprint() + torch.cuda.set_per_process_memory_fraction(0.70, 0) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.set_float32_matmul_precision("highest") + if (torch.backends.cuda.matmul.allow_tf32 or torch.backends.cudnn.allow_tf32 or + torch.get_float32_matmul_precision() != "highest"): + raise RuntimeError("TF32 settings not applied") + free_before, total_memory = torch.cuda.mem_get_info(0) + model, tokenizer, point = load_checkpoint(CHECKPOINT, "cuda:0") + if (not model.native_weights_loaded or model.training_steps < 1 or model.cfg.lora_rank < 1 + or point["training_state"]["step"] < 1): + raise ValueError("checkpoint is not a trained native model") + model.train() + outcomes = [] + for case_name in args.cases: + state_tokens = {"short_full": None, "short_medium_255": 255, + "short_medium_256": 256, "medium_257": 257, + "short_long_255": 255, "boundary_8191": 8191, "boundary_8192": 8192, + "long_10520": 10520}[case_name] + try: + if case_name == "short_full": + encoded, filler_count = largest_short_full_case(tokenizer) + suffix_filler = 0 + elif case_name == "short_long_255": + encoded, filler_count, suffix_filler = long_suffix_case(tokenizer) + else: + encoded, filler_count = exact_encoding(tokenizer, state_tokens) + suffix_filler = 0 + actual_state_tokens = len(encoded.state) + full_lengths = [actual_state_tokens + len(branch.ids) for branch in encoded.branches] + if case_name == "short_full" and max(full_lengths) > 256: + raise ValueError("short full-row dispatch not reached") + if case_name.startswith("short_medium") and (actual_state_tokens > 256 or + min(full_lengths) <= 256 or + max(full_lengths) >= 8192): + raise ValueError("short-to-medium dispatch not reached") + if case_name == "medium_257" and actual_state_tokens != 257: + raise ValueError("medium prefix dispatch not reached") + if case_name in ("boundary_8191", "boundary_8192", "long_10520") and min(full_lengths) < 8192: + raise ValueError("long/boundary dispatch not reached") + if case_name == "short_long_255" and min(full_lengths) <= 8192: + raise ValueError("short-to-long dispatch not reached") + except BaseException as exc: + failure_path = ARTIFACTS / f"{OUTPUT_STEM}-{case_name}.failure.json" + if failure_path.exists(): + raise FileExistsError(failure_path) from exc + atomic_json(failure_path, { + "evidence_class": "actual-weight-fp32-short-attention-seam-probe-failure", + "source": source_identity, "script_sha256": sha256_file(Path(__file__)), + "checkpoint_sha256": CHECKPOINT_SHA, "case_name": case_name, + "requested_state_tokens": state_tokens, + "stage": "encoding", "error_type": type(exc).__name__, + "error": str(exc), "passed": False}) + outcomes.append({"case_name": case_name, "passed": False, + "failure_receipt": str(failure_path)}) + continue + if source_fingerprint() != source_identity: + raise ValueError("source changed before seam case") + outcomes.append(run_case(model, encoded, filler_count, source_identity, + free_before, total_memory, case_name=case_name, + suffix_filler=suffix_filler)) + print({"seam_summary": [(row.get("case_name"), row["passed"]) for row in outcomes]}, + flush=True) + if not all(row["passed"] for row in outcomes): + raise SystemExit(1) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + if OWNS_OUTPUT_DIRECTORY: + failure = ARTIFACTS / "preflight.failure.json" + if not failure.exists(): + with failure.open("x", encoding="utf-8") as stream: + json.dump({"passed": False, "stage": "preflight", "error_type": type(exc).__name__, + "checkpoint_sha256": CHECKPOINT_SHA}, stream) + stream.write("\n") + raise diff --git a/src/kev_laya/execution.py b/src/kev_laya/execution.py index 810425c..943c586 100644 --- a/src/kev_laya/execution.py +++ b/src/kev_laya/execution.py @@ -11,6 +11,7 @@ from .encoding import Encoding EXECUTION_VERSION = "parallel-questions-v1" +CUDA_TRAINING_CACHE_PLAN_VERSION = "cuda-double-kv-gqa-storage-v2" @dataclass(frozen=True) @@ -59,8 +60,9 @@ def plan_batches(encoding: Encoding, policy: BatchPolicy, *, kv_bytes_per_token: gqa_bytes_per_token: int) -> tuple[BranchBatch, ...]: """Budget parent KV + per-row concatenated KV + materialized GQA K/V. - Byte rates include all layers and use parameter element size (conservative - under autocast). This is not a total process/activation memory bound. During + Byte rates include all layers and reflect actual cache and materialized + GQA storage dtypes (including double CUDA training caches). This is not a + total process/activation memory bound. During training autograd can retain tensors across multiple microbatches. """ if not encoding.state or not encoding.branches: diff --git a/src/kev_laya/model.py b/src/kev_laya/model.py index 557f35c..8281246 100644 --- a/src/kev_laya/model.py +++ b/src/kev_laya/model.py @@ -16,7 +16,10 @@ from .encoding import Encoding from .execution import BatchPolicy, plan_batches, EXECUTION_VERSION from .native_gradients import (CanonicalLoRAProjection, CanonicalLongSDPA, - CanonicalMediumAttention, CanonicalShortAttention) + CanonicalMediumAttention, CanonicalShortAttention, + CanonicalFP32ShortAttention) + +CUDA_FP32_ATTENTION_OPERATOR_VERSION = "cuda-fp32-absolute-query-bands-fixed256-cpurope-v3" @dataclass(frozen=True) @@ -103,7 +106,7 @@ def backbone_project(module: nn.Linear | LoRALinear, x: Tensor) -> Tensor: fp32 = x.dtype == torch.float32 and not torch.is_autocast_enabled("cuda") if not (x.is_cuda and (bf16 or fp32)): return module(x) - if bf16 and isinstance(module, LoRALinear) and torch.is_grad_enabled(): + if isinstance(module, LoRALinear) and torch.is_grad_enabled(): return CanonicalLoRAProjection.apply( x, module.base.weight, module.base.bias, module.a, module.b, module.scale) @@ -132,6 +135,13 @@ def __init__(self, cfg: BackboneConfig): super().__init__() d, self.h, self.kh = cfg.hidden_size, cfg.num_attention_heads, cfg.num_key_value_heads self.hd = d // self.h + # Pinned Qwen2 RoPE initializes frequencies on CPU from int64 indices. + # Integer bit storage survives model.to(dtype=...) without quantizing + # these FP32 constants; nonpersistent keeps checkpoint schema intact. + cpu_indices = torch.arange(0, self.hd, 2, device="cpu", dtype=torch.int64).to(torch.float32) + cpu_inv_freq = 1.0 / (cfg.rope_theta ** (cpu_indices / self.hd)) + self.register_buffer("_fp32_rope_inv_freq_bits", cpu_inv_freq.view(torch.int32), + persistent=False) self.theta = cfg.rope_theta self.q_proj = nn.Linear(d, self.h * self.hd, bias=True) self.k_proj = nn.Linear(d, self.kh * self.hd, bias=True) @@ -139,13 +149,84 @@ def __init__(self, cfg: BackboneConfig): self.o_proj = nn.Linear(self.h * self.hd, d, bias=False) def rotate(self, x: Tensor, positions: Tensor) -> Tensor: - inv = 1.0 / (self.theta ** (torch.arange(0, self.hd, 2, device=x.device, dtype=torch.float32) / self.hd)) + inv = (self._fp32_rope_inv_freq_bits.view(torch.float32) + if x.is_cuda and x.dtype == torch.float32 else + 1.0 / (self.theta ** (torch.arange( + 0, self.hd, 2, device=x.device, dtype=torch.float32) / self.hd))) angles = positions.float()[:, None] * inv[None, :] angles = torch.cat((angles, angles), -1)[None, None] half = self.hd // 2 rotated = torch.cat((-x[..., half:], x[..., :half]), -1) return x * angles.cos().to(x.dtype) + rotated * angles.sin().to(x.dtype) + def _fp32_absolute_attention(self, q: Tensor, k: Tensor, v: Tensor, + offset: int) -> Tensor: + """One causal operator per absolute query position in every FP32 mode. + + Full-row reference still projects the complete row independently. No + request state split or cached prefix is supplied to this dispatch. + """ + from torch.nn.attention import SDPBackend, sdpa_kernel + + end = offset + q.shape[-2] + if k.shape[-2] != end or v.shape != k.shape: + raise ValueError("FP32 causal cache length differs from absolute query positions") + ratio = self.h // self.kh + training_vjp = self.training and torch.is_grad_enabled() + segments = [] + absolute_start = offset + for boundary in (256, 8192, end): + absolute_end = min(boundary, end) + if absolute_end <= absolute_start: + continue + local_start, local_end = absolute_start - offset, absolute_end - offset + query = q[..., local_start:local_end, :] + key, value = k[..., :absolute_end, :], v[..., :absolute_end, :] + query_length = local_end - local_start + if absolute_start < 256: + if training_vjp: + part = CanonicalFP32ShortAttention.apply(query, key, value, ratio) + else: + part = CanonicalFP32ShortAttention._eager( + query, key.to(query.dtype).repeat_interleave(ratio, dim=1), + value.to(query.dtype).repeat_interleave(ratio, dim=1)) + elif absolute_start < 8192: + repeated_key = key.repeat_interleave(ratio, dim=1) + repeated_value = value.repeat_interleave(ratio, dim=1) + mask = causal_lower_right(query_length, absolute_end) + guard = (sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]) + if absolute_end >= 8192 else nullcontext()) + with guard: + if training_vjp: + part = CanonicalLongSDPA.apply( + query, repeated_key, repeated_value, mask, False, absolute_start) + else: + part = F.scaled_dot_product_attention( + query, repeated_key.to(query.dtype), repeated_value.to(query.dtype), + attn_mask=mask, dropout_p=0.0) + else: + heads = [] + mask = causal_lower_right(query_length, absolute_end) + with sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION]): + for head in range(self.kh): + grouped_query = query[:, head * ratio:(head + 1) * ratio] + grouped_key = key[:, head:head + 1].expand(-1, ratio, -1, -1) + grouped_value = value[:, head:head + 1].expand(-1, ratio, -1, -1) + if training_vjp: + result = CanonicalLongSDPA.apply( + grouped_query, grouped_key, grouped_value, mask, False, absolute_start) + else: + result = F.scaled_dot_product_attention( + grouped_query, grouped_key.to(query.dtype), + grouped_value.to(query.dtype), attn_mask=mask, dropout_p=0.0) + heads.append(result) + part = torch.cat(heads, dim=1) + segments.append(part) + absolute_start = absolute_end + if absolute_start != end: + raise ValueError("FP32 absolute attention left queries unprocessed") + return torch.cat(segments, dim=-2) if len(segments) > 1 else segments[0] + def forward(self, x: Tensor, offset: int, past: KV | None = None, reference_split_at: int | None = None) -> tuple[Tensor, KV]: b, length, _ = x.shape @@ -156,7 +237,9 @@ def forward(self, x: Tensor, offset: int, past: KV | None = None, q, k = self.rotate(q, positions), self.rotate(k, positions) bf16_training = (self.training and torch.is_grad_enabled() and q.is_cuda and q.dtype == torch.bfloat16) - if bf16_training: + fp32_training = (self.training and torch.is_grad_enabled() and q.is_cuda + and q.dtype == torch.float32) + if bf16_training or fp32_training: k, v = k.double(), v.double() if past is not None: # Functional concatenation: never mutate a parent prefix; autograd remains connected. @@ -164,6 +247,9 @@ def forward(self, x: Tensor, offset: int, past: KV | None = None, # into the one shared prefix. cat creates request-local child storage. k = torch.cat((past[0].to(k.dtype).expand(b, -1, -1, -1), k), -2) v = torch.cat((past[1].to(v.dtype).expand(b, -1, -1, -1), v), -2) + if q.is_cuda and q.dtype == torch.float32: + h = self._fp32_absolute_attention(q, k, v, offset) + return backbone_project(self.o_proj, h.transpose(1, 2).reshape(b, length, -1)), (k, v) if (reference_split_at is not None and past is None and q.is_cuda and q.dtype == torch.bfloat16 and 0 < reference_split_at < length and 256 < length < 8192): @@ -197,10 +283,10 @@ def forward(self, x: Tensor, offset: int, past: KV | None = None, # fused calls per KV group still sum gradients into the shared parent. if self.training and torch.is_grad_enabled() and q.is_cuda and k.shape[-2] >= 8192: from torch.nn.attention import SDPBackend, sdpa_kernel - canonical_bf16 = q.dtype == torch.bfloat16 + canonical_backward = q.dtype == torch.bfloat16 # The native forward still receives BF16. Double cache storage lets # separate branch VJPs sum into one parent before parameter rounding. - if canonical_bf16: + if canonical_backward: k, v = k.double(), v.double() ratio = self.h // self.kh parts = [] @@ -210,7 +296,7 @@ def forward(self, x: Tensor, offset: int, past: KV | None = None, key = k[:, head:head + 1].expand(-1, ratio, -1, -1) value = v[:, head:head + 1].expand(-1, ratio, -1, -1) mask = None if past is None else causal_lower_right(length, k.shape[-2]) - if canonical_bf16: + if canonical_backward: part = CanonicalLongSDPA.apply( query, key, value, mask, past is None, offset) else: @@ -370,7 +456,10 @@ def logits(self, decide: Tensor, options: Tensor, owner: Tensor | None = None) - bf16_autocast = (device_type in ("cpu", "cuda") and torch.is_autocast_enabled(device_type) and torch.get_autocast_dtype(device_type) == torch.bfloat16) - if device_type in ("cpu", "cuda") and (decide.dtype == torch.bfloat16 or bf16_autocast): + canonical_fp32 = (device_type == "cuda" and decide.dtype == torch.float32 + and not torch.is_autocast_enabled(device_type)) + if (device_type in ("cpu", "cuda") + and (decide.dtype == torch.bfloat16 or bf16_autocast or canonical_fp32)): # BF16 GEMM changes rounding with batch shape even for identical # hidden rows. Keep the small pointer projection and reduction # stable across independent and parallel question execution. @@ -512,13 +601,20 @@ def forward(self, encoding: Encoding, *, reference: bool = False, selected = policy or self.batch_policy element_size = max(p.element_size() for p in self.parameters()) hd = self.cfg.hidden_size // self.cfg.num_attention_heads - cache_element_size = (8 if self.training and (torch.is_grad_enabled() or _training_cache_plan) - and self.backbone.embed_tokens.weight.is_cuda - and torch.is_autocast_enabled("cuda") - and torch.get_autocast_dtype("cuda") == torch.bfloat16 - else element_size) + cuda_training_cache = (self.training and (torch.is_grad_enabled() or _training_cache_plan) + and self.backbone.embed_tokens.weight.is_cuda) + autocast_enabled = torch.is_autocast_enabled("cuda") + bf16_autocast = (autocast_enabled + and torch.get_autocast_dtype("cuda") == torch.bfloat16) + # Attention stores K/V in double for both BF16 and unautocast FP32 + # CUDA training. The no-grad streamed prepass must plan the later + # gradient-enabled pass with the same rate. + native_canonical_dtype = (not autocast_enabled and + self.backbone.embed_tokens.weight.dtype in (torch.float32, torch.bfloat16)) + double_kv_cache = cuda_training_cache and (bf16_autocast or native_canonical_dtype) + cache_element_size = 8 if double_kv_cache else element_size kv_rate = 2 * self.cfg.num_hidden_layers * self.cfg.num_key_value_heads * hd * cache_element_size - gqa_rate = 2 * self.cfg.num_hidden_layers * self.cfg.hidden_size * element_size + gqa_rate = 2 * self.cfg.num_hidden_layers * self.cfg.hidden_size * cache_element_size plan = plan_batches(encoding, selected, kv_bytes_per_token=kv_rate, gqa_bytes_per_token=gqa_rate) state_length = len(encoding.state) if any(state_length + len(b.ids) > self.cfg.max_position_embeddings for b in encoding.branches): diff --git a/src/kev_laya/native_gradients.py b/src/kev_laya/native_gradients.py index 415a548..c271b4e 100644 --- a/src/kev_laya/native_gradients.py +++ b/src/kev_laya/native_gradients.py @@ -14,6 +14,7 @@ DERIVATIVE_VERSION = "cuda-bf16-canonical-v3" +FP32_DERIVATIVE_VERSION = "cuda-fp32-canonical-v10" ATTENTION_QUERY_TILE = 64 @@ -70,6 +71,86 @@ def backward(ctx, upstream: Tensor): return dq.to(query.dtype), dk.to(key.dtype), dv.to(value.dtype), None, None, None +class CanonicalFP32ShortAttention(torch.autograd.Function): + """Preserve eager FP32 forward with double shared K/V accumulation. + + Double K/V cache remains the shared accumulation boundary. The local + query derivative retains the rounded reference operation order. + """ + + @staticmethod + def _eager(query: Tensor, key: Tensor, value: Tensor) -> Tensor: + length, total = query.shape[-2], key.shape[-2] + if not (1 <= length <= total <= 256) or key.shape != value.shape: + raise ValueError("short FP32 attention requires 1..256 causal positions") + # Place every real query at its absolute position in one fixed 256x256 + # GEMM. Cached q[255] and full-row q[255] then use the same row/shape. + start = total - length + padded_query = F.pad(query, (0, 0, start, 256 - total)) + padded_key = F.pad(key, (0, 0, 0, 256 - total)) + padded_value = F.pad(value, (0, 0, 0, 256 - total)) + scale = query.shape[-1] ** -0.5 + scores = torch.matmul(padded_query, padded_key.transpose(-2, -1)) * scale + allowed = torch.ones((256, 256), device=query.device, dtype=torch.bool).tril() + scores = scores.masked_fill(~allowed, torch.finfo(scores.dtype).min) + probabilities = F.softmax(scores, dim=-1, dtype=torch.float32).to(query.dtype) + output = torch.matmul(probabilities, padded_value) + return output[..., start:start + length, :] + + @staticmethod + def forward(ctx, query: Tensor, key: Tensor, value: Tensor, ratio: int): + # Round each grouped KV head before repeating it, as in the pinned + # eager FP32 graph. `key` and `value` remain double cache parents. + if ratio < 1 or query.shape[1] != key.shape[1] * ratio or key.shape != value.shape: + raise ValueError("invalid grouped KV short attention shape") + ctx.save_for_backward(query, key, value) + ctx.ratio = ratio + return CanonicalFP32ShortAttention._eager( + query, key.to(query.dtype).repeat_interleave(ratio, dim=1), + value.to(query.dtype).repeat_interleave(ratio, dim=1)) + + @staticmethod + def backward(ctx, upstream: Tensor): + query, key, value = ctx.saved_tensors + length, total = query.shape[-2], key.shape[-2] + start = total - length + ratio = ctx.ratio + # Recompute the exact rounded FP32 score/softmax graph. Its local + # query and score VJPs retain the pinned eager operator's cast points. + with torch.enable_grad(): + q = query.detach().requires_grad_(True) + k = key.detach().to(query.dtype).repeat_interleave(ratio, dim=1) + v = value.detach().to(query.dtype).repeat_interleave(ratio, dim=1) + padded_q = F.pad(q, (0, 0, start, 256 - total)) + padded_k = F.pad(k, (0, 0, 0, 256 - total)) + padded_v = F.pad(v, (0, 0, 0, 256 - total)) + scores = torch.matmul(padded_q, padded_k.transpose(-2, -1)) * ( + query.shape[-1] ** -0.5) + allowed = torch.ones((256, 256), device=query.device, dtype=torch.bool).tril() + probabilities = F.softmax( + scores.masked_fill(~allowed, torch.finfo(scores.dtype).min), + dim=-1, dtype=torch.float32).to(query.dtype) + output = torch.matmul(probabilities, padded_v)[..., start:start + length, :] + dq, ds = torch.autograd.grad(output, (q, scores), upstream) + + # The rounded FP32 score VJP is the derivative of the actual forward. + # Products and query-row sums into shared K/V use double so that + # prefix/suffix and full-row gradients have one accumulation precision. + padded_upstream = F.pad(upstream, (0, 0, start, 256 - total)) + per_head_k = (ds.double().transpose(-2, -1) @ padded_q.detach().double()) * ( + query.shape[-1] ** -0.5) + per_head_v = probabilities.detach().double().transpose(-2, -1) @ ( + padded_upstream.double()) + batch, heads, _, width = per_head_k.shape + groups = heads // ratio + # Group repeated heads and accumulate their shared K/V contribution + # in double before the single cache-parent cast. Full and cached + # requests therefore have the same summation precision. + dk = per_head_k.reshape(batch, groups, ratio, 256, width).sum(dim=2)[..., :total, :] + dv = per_head_v.reshape(batch, groups, ratio, 256, width).sum(dim=2)[..., :total, :] + return dq.to(query.dtype), dk.to(key.dtype), dv.to(value.dtype), None + + class CanonicalShortAttention(torch.autograd.Function): """Preserve short eager BF16 forward while accumulating its VJP in double.""" @@ -162,11 +243,12 @@ def forward(ctx, x: Tensor, weight: Tensor, bias: Tensor | None, a: Tensor, b: Tensor, scale: float): rows = x.reshape(-1, x.shape[-1]) outputs, inners = [], [] - for start in range(0, rows.shape[0], 1024): - tile = rows[start:start + 1024] + tile_rows = 1024 if torch.is_autocast_enabled("cuda") else 64 + for start in range(0, rows.shape[0], tile_rows): + tile = rows[start:start + tile_rows] count = tile.shape[0] - if count < 1024: - tile = F.pad(tile, (0, 0, 0, 1024 - count)) + if count < tile_rows: + tile = F.pad(tile, (0, 0, 0, tile_rows - count)) inner = F.linear(tile, a) projected = F.linear(tile, weight, bias) + F.linear(inner, b) * scale outputs.append(projected[:count]) @@ -179,8 +261,9 @@ def forward(ctx, x: Tensor, weight: Tensor, bias: Tensor | None, def backward(ctx, upstream: Tensor): x, weight, a, b, inner = ctx.saved_tensors g = upstream.reshape(-1, upstream.shape[-1]).double() - x_eff = x.reshape(-1, x.shape[-1]).to(torch.bfloat16).double() - w_eff, a_eff, b_eff = (item.to(torch.bfloat16).double() + calculation_dtype = inner.dtype + x_eff = x.reshape(-1, x.shape[-1]).to(calculation_dtype).double() + w_eff, a_eff, b_eff = (item.to(calculation_dtype).double() for item in (weight, a, b)) grad_inner = (g @ b_eff) * ctx.scale grad_x = g @ w_eff + grad_inner @ a_eff diff --git a/src/kev_laya/training.py b/src/kev_laya/training.py index de936e1..fbc2ec6 100644 --- a/src/kev_laya/training.py +++ b/src/kev_laya/training.py @@ -10,11 +10,12 @@ import uuid import torch from .checkpoint import load_checkpoint, restore_rng, save_checkpoint -from .native_gradients import DERIVATIVE_VERSION +from .native_gradients import DERIVATIVE_VERSION, FP32_DERIVATIVE_VERSION from .data import Datum, Sampler, permute_choices from .encoding import Encoding, Limits, preprocessing_identity +from .execution import CUDA_TRAINING_CACHE_PLAN_VERSION from .io import atomic_json -from .model import DecisionEngine +from .model import CUDA_FP32_ATTENTION_OPERATOR_VERSION, DecisionEngine from .objectives import ObjectiveConfig, objective from .quality_budget import BudgetExceeded, StepTokenBudget from .schema import canonical @@ -119,8 +120,12 @@ def train(model: DecisionEngine, tokenizer, data: list[Datum], manifest: dict, o cfg = {"settings": recorded_settings, "objective": asdict(loss_config), "limits": asdict(limits), "backbone": model.config_dict(), "preprocessing": preprocessing_identity(tokenizer), "execution": model.batch_policy.to_dict()} - if device.type == "cuda" and settings.precision == "bf16": - cfg["derivative_version"] = DERIVATIVE_VERSION + if device.type == "cuda": + cfg["cache_plan_version"] = CUDA_TRAINING_CACHE_PLAN_VERSION + cfg["derivative_version"] = (DERIVATIVE_VERSION if settings.precision == "bf16" + else FP32_DERIVATIVE_VERSION) + if settings.precision == "fp32": + cfg["attention_operator_version"] = CUDA_FP32_ATTENTION_OPERATOR_VERSION data_hash = hashlib.sha256(canonical(manifest).encode()).hexdigest() config_hash = hashlib.sha256(canonical(cfg).encode()).hexdigest() if limits.branch > model.cfg.max_position_embeddings: @@ -337,6 +342,12 @@ def quality_gpu_peak(): torch.autograd.backward( logit, gradient, retain_graph=branch_number + 1 < len(logits)) + # Earlier branch graphs were retained while later branches + # differentiated through the shared prefix. Their saved + # activations remain reachable through these local owners + # even after the final backward. Release them before + # full-weight AdamW allocates or uses optimizer moments. + del logit, gradient, gradients, logits, loss elif not settings.streamed_branch_vjp: (loss / settings.accumulation).backward() if quality_budget is not None: diff --git a/tests/test_native_gradients.py b/tests/test_native_gradients.py index b3f3cbb..8dfb558 100644 --- a/tests/test_native_gradients.py +++ b/tests/test_native_gradients.py @@ -4,7 +4,7 @@ import pytest import torch -from kev_laya.native_gradients import causal_vjp +from kev_laya.native_gradients import CanonicalFP32ShortAttention, causal_vjp @pytest.mark.parametrize("prefix", [0, 8]) @@ -26,3 +26,80 @@ def test_causal_vjp_matches_independent_double_autograd(prefix): split_at=prefix + 2, tile=3) for observed, oracle in zip(actual, expected, strict=True): torch.testing.assert_close(observed, oracle, atol=1e-12, rtol=1e-12) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") +@pytest.mark.parametrize("prefix", [0, 11]) +def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix): + generator = torch.Generator(device="cuda").manual_seed(4521 + prefix) + q0 = torch.randn(1, 4, 7, 16, generator=generator, device="cuda", dtype=torch.float32) + k0 = torch.randn(1, 2, prefix + 7, 16, generator=generator, + device="cuda", dtype=torch.float32).double() + v0 = torch.randn(k0.shape, generator=generator, device="cuda", dtype=torch.float32).double() + upstream = torch.randn(q0.shape, generator=generator, device="cuda", dtype=torch.float32) + q, k, v = [tensor.detach().clone().requires_grad_(True) for tensor in (q0, k0, v0)] + actual = CanonicalFP32ShortAttention.apply(q, k, v, 2) + actual_vjp = torch.autograd.grad(actual, (q, k, v), upstream) + + rq, rk, rv = [tensor.detach().clone().requires_grad_(True) for tensor in (q0, k0, v0)] + repeated_k = rk.float().repeat_interleave(2, dim=1) + repeated_v = rv.float().repeat_interleave(2, dim=1) + total = prefix + 7 + padded_q = torch.nn.functional.pad(rq, (0, 0, prefix, 256 - total)) + padded_k = torch.nn.functional.pad(repeated_k, (0, 0, 0, 256 - total)) + padded_v = torch.nn.functional.pad(repeated_v, (0, 0, 0, 256 - total)) + scores = torch.matmul(padded_q, padded_k.transpose(-2, -1)) * (16 ** -0.5) + allowed = torch.ones((256, 256), device="cuda", dtype=torch.bool).tril() + scores = scores.masked_fill(~allowed, torch.finfo(scores.dtype).min) + probabilities = torch.nn.functional.softmax(scores, dim=-1, dtype=torch.float32).to(rq.dtype) + expected = torch.matmul(probabilities, padded_v)[..., prefix:prefix + 7, :] + eager_dq, eager_dk, eager_dv, score_vjp = torch.autograd.grad( + expected, (rq, rk, rv, scores), upstream) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + # Query VJP preserves the exact rounded FP32 eager graph. Shared K/V cache + # VJPs accumulate query rows and repeated heads in double before returning + # to the double cache parent. + torch.testing.assert_close(actual_vjp[0], eager_dq, atol=0, rtol=0) + padded_upstream = torch.nn.functional.pad(upstream, (0, 0, prefix, 256 - total)) + per_head_k = torch.einsum("bhqk,bhqd->bhkd", score_vjp.double(), + padded_q.detach().double()) * (16 ** -0.5) + per_head_v = torch.einsum("bhqk,bhqd->bhkd", probabilities.detach().double(), + padded_upstream.double()) + analytical_dk = per_head_k.reshape(1, 2, 2, 256, 16).sum(2)[..., :total, :] + analytical_dv = per_head_v.reshape(1, 2, 2, 256, 16).sum(2)[..., :total, :] + for actual_cache_vjp, analytical_vjp, eager_vjp in ( + (actual_vjp[1], analytical_dk, eager_dk), + (actual_vjp[2], analytical_dv, eager_dv)): + assert bool(torch.isfinite(actual_cache_vjp).all()) + torch.testing.assert_close(actual_cache_vjp, analytical_vjp, atol=2e-5, rtol=2e-5) + torch.testing.assert_close(actual_cache_vjp, eager_vjp, + atol=2e-5, rtol=2e-5) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") +@pytest.mark.parametrize("prefix_length", [1, 11, 62, 127, 200, 254, 255]) +def test_fp32_short_fixed256_matches_any_cached_prefix(prefix_length): + generator = torch.Generator(device="cuda").manual_seed(6256 + prefix_length) + query = torch.randn(1, 4, 256, 16, generator=generator, device="cuda") + key = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double() + value = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double() + upstream = torch.randn(query.shape, generator=generator, device="cuda") + full_inputs = [tensor.detach().clone().requires_grad_(True) + for tensor in (query, key, value)] + split_inputs = [tensor.detach().clone().requires_grad_(True) + for tensor in (query, key, value)] + whole = CanonicalFP32ShortAttention.apply(*full_inputs, 2) + fq, fk, fv = split_inputs + prefix = CanonicalFP32ShortAttention.apply( + fq[..., :prefix_length, :], fk[..., :prefix_length, :], + fv[..., :prefix_length, :], 2) + suffix = CanonicalFP32ShortAttention.apply( + fq[..., prefix_length:, :], fk, fv, 2) + split = torch.cat((prefix, suffix), dim=-2) + torch.testing.assert_close(split, whole, atol=0, rtol=0) + full_grads = torch.autograd.grad(whole, full_inputs, upstream) + split_grads = torch.autograd.grad(split, split_inputs, upstream) + for full_grad, split_grad in zip(full_grads, split_grads, strict=True): + assert bool(torch.isfinite(full_grad).all()) + assert bool(torch.isfinite(split_grad).all()) + torch.testing.assert_close(split_grad, full_grad, atol=2e-5, rtol=2e-5) diff --git a/tests/test_parallel_questions.py b/tests/test_parallel_questions.py index 797be0e..6cc7ce4 100644 --- a/tests/test_parallel_questions.py +++ b/tests/test_parallel_questions.py @@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import replace import json +import weakref from pathlib import Path import pytest @@ -25,6 +26,53 @@ from kev_laya.training import TrainSettings, train +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA required for BF16 branch graph lifetime') +def test_bf16_branch_activations_released_before_optimizer(tmp_path, suite, monkeypatch): + if not torch.cuda.is_bf16_supported(): + pytest.skip('GPU does not support BF16') + saved = [] + + class Branch(torch.autograd.Function): + @staticmethod + def forward(ctx, value, count): + activation = value.new_ones(4096) + saved.append(weakref.ref(activation)) + ctx.save_for_backward(activation) + return value.expand(count).clone() + + @staticmethod + def backward(ctx, upstream): + activation, = ctx.saved_tensors + return upstream.sum() * activation[0], None + + model = make_model(lora=2, policy=BatchPolicy(max_branches=2)).cuda() + original_forward = model.forward + + def forward(encoded): + with torch.no_grad(): + _, usage = original_forward(encoded) + shared = sum(parameter.sum() for parameter in model.parameters() if parameter.requires_grad) + logits = [Branch.apply(shared, len(branch.question.options())) for branch in encoded.branches] + return logits, usage + + monkeypatch.setattr(model, 'forward', forward) + original_step = torch.optim.AdamW.step + checked = [] + + def step(optimizer, *args, **kwargs): + assert len(saved) >= 2 + assert all(reference() is None for reference in saved) + checked.append(True) + return original_step(optimizer, *args, **kwargs) + + monkeypatch.setattr(torch.optim.AdamW, 'step', step) + data, manifest = suite + result = train(model, ByteTokenizer(), data['train'], manifest, tmp_path / 'graph-release', + TrainSettings(steps=1, precision='bf16'), ObjectiveConfig(), Limits(512, 8192)) + assert checked == [True] + assert result['state']['step'] == 1 + + def request(count=7): questions = {} for i in range(count): @@ -414,6 +462,82 @@ def test_cuda_fixture_fp32_parallel_and_gradients(): torch.backends.cuda.matmul.allow_tf32=old + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA hardware unavailable; native cache unverified') +@pytest.mark.parametrize('precision', ['fp32', 'bf16', 'native_bf16', 'native_fp16']) +def test_cuda_training_cache_plan_matches_double_prefix_and_rejects_before_forward(precision): + if precision in ('bf16', 'native_bf16') and not torch.cuda.is_bf16_supported(): + pytest.skip('GPU does not support BF16') + encoded = encode(request(2)) + model = make_model(lora=2, checkpointing=True, + policy=BatchPolicy(max_branches=2)).cuda().train() + if precision == 'native_bf16': + model = model.to(torch.bfloat16) + elif precision == 'native_fp16': + model = model.to(torch.float16) + enabled = precision == 'bf16' + cache_element_size = 8 if precision != 'native_fp16' else 2 + cfg = model.cfg + kv_rate = 2 * cfg.num_hidden_layers * cfg.num_key_value_heads * ( + cfg.hidden_size // cfg.num_attention_heads) * cache_element_size + gqa_rate = 2 * cfg.num_hidden_layers * cfg.hidden_size * cache_element_size + expected = plan_batches(encoded, model.batch_policy, + kv_bytes_per_token=kv_rate, gqa_bytes_per_token=gqa_rate) + with torch.autocast('cuda', dtype=torch.bfloat16, enabled=enabled): + logits, usage = model(encoded) + assert len(logits) == len(encoded.branches) + assert usage['prefix_cache_bytes'] == len(encoded.state) * kv_rate + assert usage['estimated_peak_cache_bytes'] == max(batch.estimated_cache_bytes for batch in expected) + with torch.no_grad(), torch.autocast('cuda', dtype=torch.bfloat16, enabled=enabled): + _, prepass_usage = model(encoded, _training_cache_plan=True) + assert prepass_usage['estimated_peak_cache_bytes'] == usage['estimated_peak_cache_bytes'] + singletons = plan_batches(encoded, BatchPolicy(max_branches=1), + kv_bytes_per_token=kv_rate, gqa_bytes_per_token=gqa_rate) + tight_limit = min(batch.estimated_cache_bytes for batch in singletons) - 1 + restricted = BatchPolicy(max_branches=2, max_cache_bytes=tight_limit) + calls, handle = spy_backbone(model) + try: + with torch.autocast('cuda', dtype=torch.bfloat16, enabled=enabled): + with pytest.raises(BatchBudgetExceeded) as failure: + model(encoded, policy=restricted) + assert failure.value.detail['limit'] == 'cache_bytes' + assert calls == [] + finally: + handle.remove() + +@pytest.mark.skipif(not torch.cuda.is_available(), reason='CUDA device unavailable') +def test_fp32_medium_cache_budget_accounts_observed_double_gqa(monkeypatch): + from kev_laya.native_gradients import CanonicalLongSDPA + payload = request(2) + payload['state'] = {'text': 'x' * 300} + encoded = encode(payload) + assert 256 < len(encoded.state) < 8192 + model = make_model(lora=2, checkpointing=False, + policy=BatchPolicy(max_branches=2)).cuda().train() + original = CanonicalLongSDPA.forward + observations = [] + + def observe(ctx, query, key, value, *args): + observations.append((key.dtype, value.dtype, key.shape[1], + key.element_size(), value.element_size())) + return original(ctx, query, key, value, *args) + + monkeypatch.setattr(CanonicalLongSDPA, 'forward', staticmethod(observe)) + _, usage = model(encoded) + assert observations + assert all(kdtype == vdtype == torch.float64 and heads == model.cfg.num_attention_heads + and kbytes == vbytes == 8 + for kdtype, vdtype, heads, kbytes, vbytes in observations) + cfg = model.cfg + kv_rate = 2 * cfg.num_hidden_layers * cfg.num_key_value_heads * ( + cfg.hidden_size // cfg.num_attention_heads) * observations[0][3] + gqa_rate = 2 * cfg.num_hidden_layers * cfg.hidden_size * observations[0][3] + expected = plan_batches(encoded, model.batch_policy, + kv_bytes_per_token=kv_rate, gqa_bytes_per_token=gqa_rate) + assert usage['prefix_cache_bytes'] == len(encoded.state) * kv_rate + assert usage['estimated_peak_cache_bytes'] == max(batch.estimated_cache_bytes for batch in expected) + + @pytest.mark.skipif(not torch.cuda.is_available(),reason='CUDA hardware unavailable; BF16 training unverified') def test_cuda_fixture_bf16_training(tmp_path,suite): if not torch.cuda.is_bf16_supported():pytest.skip('GPU does not support BF16') @@ -584,18 +708,30 @@ def test_cuda_bf16_streamed_resume_matches_uninterrupted_with_accumulation(tmp_p torch.testing.assert_close(value,resumed.state_dict()[name],atol=0,rtol=0) -@pytest.mark.skipif(not torch.cuda.is_available(),reason='CUDA hardware unavailable; BF16 resume unverified') -def test_cuda_bf16_resume_rejects_derivative_version_change(tmp_path,suite,monkeypatch): - if not torch.cuda.is_bf16_supported():pytest.skip('GPU does not support BF16') +@pytest.mark.skipif(not torch.cuda.is_available(),reason='CUDA hardware unavailable; precision resume unverified') +@pytest.mark.parametrize('precision,version_field',[ + ('fp32','FP32_DERIVATIVE_VERSION'), + ('bf16','DERIVATIVE_VERSION'), + ('fp32','CUDA_FP32_ATTENTION_OPERATOR_VERSION'), + ('fp32','CUDA_TRAINING_CACHE_PLAN_VERSION'), + ('bf16','CUDA_TRAINING_CACHE_PLAN_VERSION'), +]) +def test_cuda_resume_rejects_derivative_version_change(tmp_path,suite,monkeypatch,precision,version_field): + if precision=='bf16' and not torch.cuda.is_bf16_supported():pytest.skip('GPU does not support BF16') model=make_model(lora=2,checkpointing=True,policy=BatchPolicy(max_branches=2)).cuda() data,manifest=suite - settings=TrainSettings(steps=2,precision='bf16') + settings=TrainSettings(steps=2,precision=precision) root=tmp_path/'bf16-version' first=train(model,ByteTokenizer(),data['train'],manifest,root,settings, ObjectiveConfig(reinforce=.1),Limits(512,8192),stop_after=1) restored,tokenizer,_=load_checkpoint(Path(first['checkpoint']),'cuda:0') import kev_laya.training as training_module - monkeypatch.setattr(training_module,'DERIVATIVE_VERSION','unreviewed-derivative') + if version_field in ('DERIVATIVE_VERSION','FP32_DERIVATIVE_VERSION'): + assert first['state']['derivative_version']==getattr(training_module,version_field) + if version_field=='CUDA_TRAINING_CACHE_PLAN_VERSION': + config=json.loads((root/'config.json').read_text(encoding='utf-8')) + assert config['cache_plan_version']==getattr(training_module,version_field) + monkeypatch.setattr(training_module,version_field,'unreviewed-derivative') with pytest.raises(ValueError,match='resume configuration or frozen data differs'): train(restored,tokenizer,data['train'],manifest,root,settings, ObjectiveConfig(reinforce=.1),Limits(512,8192), diff --git a/tests/test_short_cuda_attention.py b/tests/test_short_cuda_attention.py index 15c68d9..f7b7243 100644 --- a/tests/test_short_cuda_attention.py +++ b/tests/test_short_cuda_attention.py @@ -33,30 +33,51 @@ def fail_sdpa(*args, **kwargs): k, v = torch.cat((parent[0], k), -2), torch.cat((parent[1], v), -2) k = k.repeat_interleave(attention.h // attention.kh, dim=1) v = v.repeat_interleave(attention.h // attention.kh, dim=1) - scores = torch.matmul(q, k.transpose(-2, -1)) * (attention.hd ** -0.5) - allowed = torch.ones((suffix_length, k.shape[-2]), dtype=torch.bool, device="cuda").tril(diagonal=prefix_length) + # Place each suffix query at its absolute row in the fixed 256 matrix. + total = k.shape[-2] + padded_q = F.pad(q, (0, 0, prefix_length, 256 - total)) + padded_k = F.pad(k, (0, 0, 0, 256 - total)) + padded_v = F.pad(v, (0, 0, 0, 256 - total)) + scores = torch.matmul(padded_q, padded_k.transpose(-2, -1)) * (attention.hd ** -0.5) + allowed = torch.ones((256, 256), dtype=torch.bool, device="cuda").tril() scores = scores.masked_fill(~allowed, torch.finfo(scores.dtype).min) - expected = torch.matmul(F.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype), v) - expected = backbone_project(attention.o_proj, expected.transpose(1, 2).reshape(1, suffix_length, -1)) + expected = torch.matmul(F.softmax(scores, dim=-1, dtype=torch.float32).to(q.dtype), + padded_v)[..., prefix_length:prefix_length + suffix_length, :] + expected = backbone_project(attention.o_proj, + expected.transpose(1, 2).reshape(1, suffix_length, -1)) torch.testing.assert_close(result, expected, atol=0, rtol=0) + # Retain the independently unpadded eager reference at the logits gate. + eager_scores = torch.matmul(q, k.transpose(-2, -1)) * (attention.hd ** -0.5) + eager_allowed = torch.ones((suffix_length, total), dtype=torch.bool, + device="cuda").tril(diagonal=prefix_length) + eager_scores = eager_scores.masked_fill(~eager_allowed, + torch.finfo(eager_scores.dtype).min) + eager = torch.matmul(F.softmax(eager_scores, dim=-1, dtype=torch.float32).to(q.dtype), v) + eager = backbone_project(attention.o_proj, + eager.transpose(1, 2).reshape(1, suffix_length, -1)) + torch.testing.assert_close(result, eager, atol=1e-5, rtol=1e-5) + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") def test_longer_cuda_sequence_keeps_sdpa(monkeypatch): + from torch.nn.attention.bias import CausalBias cfg = BackboneConfig(hidden_size=64, num_attention_heads=4, num_key_value_heads=2, intermediate_size=128, max_position_embeddings=512) attention = Attention(cfg).cuda().eval() - original = F.scaled_dot_product_attention + # CausalBias dispatch compares the public SDPA function by identity. + # Observe its dispatcher without replacing that function. + original = CausalBias._dispatch calls = [] def record(*args, **kwargs): calls.append((args[0].shape[-2], args[1].shape[-2])) return original(*args, **kwargs) - monkeypatch.setattr(F, "scaled_dot_product_attention", record) + monkeypatch.setattr(CausalBias, "_dispatch", staticmethod(record)) output, _ = attention(torch.randn(1, 257, cfg.hidden_size, device="cuda"), 0) assert output.shape == (1, 257, cfg.hidden_size) - assert calls == [(257, 257)] + assert calls == [(1, 257)] @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") @@ -78,3 +99,46 @@ def test_short_bf16_training_preserves_eager_forward_and_double_parent_cache(): actual, child_cache = attention(suffix, 11, train_cache) assert all(t.dtype == torch.float64 for t in (*train_cache, *child_cache)) torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") +@pytest.mark.parametrize("prefix_length", [255, 256]) +@pytest.mark.parametrize("training", [False, True]) +def test_fp32_absolute_query_dispatch_matches_independent_full_row(prefix_length, training): + torch.manual_seed(3256 + prefix_length) + cfg = BackboneConfig(hidden_size=64, num_attention_heads=4, num_key_value_heads=2, + intermediate_size=128, max_position_embeddings=512) + attention = Attention(cfg).cuda().train(training) + row = torch.randn(1, prefix_length + 56, cfg.hidden_size, device="cuda") + full, _ = attention(row, 0) + first, parent = attention(row[:, :prefix_length], 0) + suffix, _ = attention(row[:, prefix_length:], prefix_length, parent) + assert torch.isfinite(full).all() and torch.isfinite(first).all() and torch.isfinite(suffix).all() + torch.testing.assert_close(torch.cat((first, suffix), dim=1), full, atol=1e-5, rtol=1e-5) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") +def test_fp32_rope_uses_cpu_initialized_frequencies_after_dtype_moves_and_reload(): + cfg = BackboneConfig(hidden_size=64, num_attention_heads=4, num_key_value_heads=2, + intermediate_size=128, max_position_embeddings=32768) + attention = Attention(cfg).cuda().float().eval() + positions = torch.tensor([0, 1, 63, 255, 256, 8191, 8192, 32767], device="cuda") + values = torch.randn(1, attention.h, len(positions), attention.hd, device="cuda") + cpu_indices = torch.arange(0, attention.hd, 2, dtype=torch.int64).to(torch.float32) + cpu_inv = 1.0 / (cfg.rope_theta ** (cpu_indices / attention.hd)) + angles = positions.float()[:, None] * cpu_inv.cuda()[None, :] + angles = torch.cat((angles, angles), dim=-1) + rotated_half = torch.cat((-values[..., attention.hd // 2:], + values[..., :attention.hd // 2]), dim=-1) + expected = values * angles.cos()[None, None] + rotated_half * angles.sin()[None, None] + state = attention.state_dict() + assert "_fp32_rope_inv_freq_bits" not in state + torch.testing.assert_close(attention.rotate(values, positions), expected, atol=0, rtol=0) + split = torch.cat((attention.rotate(values[..., :4, :], positions[:4]), + attention.rotate(values[..., 4:, :], positions[4:])), dim=-2) + torch.testing.assert_close(split, expected, atol=0, rtol=0) + attention.to(dtype=torch.bfloat16).to(dtype=torch.float32) + torch.testing.assert_close(attention.rotate(values, positions), expected, atol=0, rtol=0) + restored = Attention(cfg).cuda().float().eval() + restored.load_state_dict(state, strict=True) + torch.testing.assert_close(restored.rotate(values, positions), expected, atol=0, rtol=0)