Add experimental persistent BF16 full fine-tuning

README.md

@@ -42,6 +42,74 @@ MobileFineTuner is an open-source C++ framework for practical, privacy-preservin
 
 Unlike simulation-based or desktop-bound approaches, MobileFineTuner is built around a lean native C++ implementation that eliminates Python runtime overhead in the training path and supports both Full Fine-Tuning (Full-FT) and Parameter-Efficient Fine-Tuning (PEFT/LoRA) under tight resource constraints.
 
+### Experimental: Persistent BF16 Full Fine-Tuning
+
+This fork adds an experimental persistent-BF16 parameter path for native C++ Full Fine-Tuning.
+
+#### What changed
+
+- Model parameters can be stored persistently as **BF16** instead of FP32.
+- Forward activations and backward computations use FP32 where required.
+- Trainable gradients are maintained in **FP32**.
+- Adam optimizer moments remain **FP32**, while parameters are written back to BF16.
+- SafeTensors loading can convert FP32 source weights to BF16 parameter storage.
+- BF16-aware backward paths were added for LayerNorm, RMSNorm, MatMul, and gradient accumulation.
+- BF16 storage, optimizer, tokenizer-parity, and end-to-end training tests are included.
+
+#### GPT-2 124M validation
+
+The BF16 path was validated with a native C++ build on an ARM64 Android device using Termux/PRoot Debian.
+
+GPT-2 configuration:
+
+- 124,439,808 parameters
+- 12 layers
+- 768 hidden size
+- 12 attention heads
+- vocabulary size 50,257
+- context length 1,024
+
+Full-model BF16 storage validation:
+
+- 148 BF16 parameter tensors
+- 0 FP32 parameter tensors
+- 248,879,616 parameter bytes
+- 237.35 MiB parameter storage
+- approximately half the parameter-storage footprint of FP32
+
+A full 124M forward pass completed successfully with finite FP32 logits.
+
+A full-model 10-step BF16 Full-FT safety run completed successfully:
+
+- 10/10 forward passes
+- 10/10 backward passes
+- 10/10 FP32 Adam updates
+- gradients cleared after every step
+- parameters remained BF16 throughout the run
+
+#### Real-data experiment
+
+A separate 20-step causal Full-FT experiment was run on a Classical Tamil dataset using the native GPT-2 tokenizer.
+
+Configuration:
+
+- batch size: 1
+- maximum sequence length: 32
+- learning rate: 5e-6
+- Adam β1: 0.9
+- Adam β2: 0.999
+- weight decay: 0
+- 20 optimization steps
+
+Recorded loss:
+
+- Step 1: 8.57498
+- Step 20: 5.37094
+
+This demonstrates that the BF16 parameter path can execute a real native training workload on a mobile-class ARM64 device. The loss change alone is not evidence of improved model quality or convergence.
+
+> **Scope:** This is an experimental engineering extension of MobileFineTuner, not a new optimization algorithm. It is intended for reproducibility, systems experimentation, and further research into memory-constrained on-device training.
+
 ### Verified Scope
 
 - Stable C++ operator/autograd/LoRA core with unit tests and installable CMake package.

operator/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.10)
-project(Operators VERSION 2.0.0 LANGUAGES CXX)
+project(Operators VERSION 2.0.0 LANGUAGES C CXX)
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 
@@ -565,3 +565,137 @@ message(STATUS "  - Build tests: ${BUILD_TESTS}")
 message(STATUS "  - Profiling: ${ENABLE_PROFILING}")
 message(STATUS "============================================")
 message(STATUS "")
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/finetune_ops/optim/test_gpt2_fullft_10step.cpp)
+    add_executable(test_gpt2_fullft_10step
+        finetune_ops/optim/test_gpt2_fullft_10step.cpp
+    )
+    target_link_libraries(test_gpt2_fullft_10step operators)
+    add_test(
+        NAME GPT2FullFT10Step
+        COMMAND test_gpt2_fullft_10step
+    )
+endif()
+
+add_executable(test_gpt2_fullft_124m
+    finetune_ops/optim/test_gpt2_fullft_124m.cpp
+)
+
+target_link_libraries(test_gpt2_fullft_124m PRIVATE operators)
+
+add_test(
+    NAME GPT2FullFT124M
+    COMMAND test_gpt2_fullft_124m
+)
+
+
+
+add_executable(test_gpt2_tokenizer_parity
+    finetune_ops/core/test_gpt2_tokenizer_parity.cpp
+)
+
+target_link_libraries(test_gpt2_tokenizer_parity PRIVATE operators)
+
+add_test(
+    NAME GPT2TokenizerParity
+    COMMAND test_gpt2_tokenizer_parity
+)
+
+
+add_executable(test_gpt2_fullft_real
+    finetune_ops/optim/test_gpt2_fullft_real.cpp
+)
+
+target_link_libraries(test_gpt2_fullft_real PRIVATE operators)
+
+add_test(
+    NAME GPT2FullFTReal
+    COMMAND test_gpt2_fullft_real
+)
+
+add_executable(test_gpt2_bf16_load_only
+    finetune_ops/optim/test_gpt2_bf16_load_only.cpp
+)
+target_link_libraries(test_gpt2_bf16_load_only PRIVATE operators)
+
+
+# ------------------------------------------------------------
+# ARM64 native numerical backend
+# ------------------------------------------------------------
+
+add_executable(test_arm64_neon_gemm
+    finetune_ops/core/test_arm64_neon_gemm.cpp
+    finetune_ops/core/arm64_neon_kernels.c
+)
+
+target_link_libraries(test_arm64_neon_gemm
+    PRIVATE
+    pthread
+)
+
+add_test(
+    NAME ARM64NEONGEMM
+    COMMAND test_arm64_neon_gemm
+)
+
+add_executable(test_adam_bf16_param
+    finetune_ops/optim/test_adam_bf16_param.cpp
+)
+target_link_libraries(test_adam_bf16_param PRIVATE operators)
+
+add_executable(test_bf16_storage_diag
+    finetune_ops/optim/test_bf16_storage_diag.cpp
+)
+target_link_libraries(test_bf16_storage_diag PRIVATE operators)
+
+add_executable(test_bf16_direct_write
+    finetune_ops/optim/test_bf16_direct_write.cpp
+)
+target_link_libraries(test_bf16_direct_write PRIVATE operators)
+
+add_executable(test_adam_bf16_param_strict
+    finetune_ops/optim/test_adam_bf16_param_strict.cpp
+)
+target_link_libraries(test_adam_bf16_param_strict PRIVATE operators)
+
+add_executable(test_bf16_weight_pipeline
+    finetune_ops/optim/test_bf16_weight_pipeline.cpp
+)
+target_link_libraries(test_bf16_weight_pipeline PRIVATE operators)
+
+
+add_executable(test_gpt2_bf16_forward_safe
+    finetune_ops/optim/test_gpt2_bf16_forward_safe.cpp
+)
+target_link_libraries(test_gpt2_bf16_forward_safe PRIVATE operators)
+
+
+add_executable(test_gpt2_bf16_final_safe
+    finetune_ops/optim/test_gpt2_bf16_final_safe.cpp
+)
+target_link_libraries(test_gpt2_bf16_final_safe PRIVATE operators)
+
+
+add_executable(test_gpt2_bf16_10step_safe
+    finetune_ops/optim/test_gpt2_bf16_10step_safe.cpp
+)
+
+target_link_libraries(test_gpt2_bf16_10step_safe
+    PRIVATE operators
+)
+
+add_executable(test_gpt2_bf16_real_causal_ft
+    finetune_ops/optim/test_gpt2_bf16_real_causal_ft.cpp
+)
+
+target_link_libraries(test_gpt2_bf16_real_causal_ft
+    PRIVATE operators
+)
+
+add_executable(test_gpt2_bf16_ultimate
+    finetune_ops/optim/test_gpt2_bf16_ultimate.cpp
+)
+
+target_link_libraries(test_gpt2_bf16_ultimate
+    PRIVATE operators
+)

operator/finetune_ops/core/backward_functions.cpp

@@ -141,6 +141,13 @@ std::vector<TensorPtr> MatmulBackward::apply(const TensorPtr& grad_output) {
         if (grad_b && grad_b->shape() != b_shape) {
             grad_b = sum_to_shape(grad_b, b_shape);
         }
+
+        // Parameter gradients are always accumulated in FP32.
+        // The parameter itself may remain BF16.
+        if (grad_b && b_ && b_->requires_grad() &&
+            grad_b->dtype() != DType::kFloat32) {
+            grad_b = cast(grad_b, DType::kFloat32);
+        }
     }
 
     return {grad_a, grad_b};
@@ -438,12 +445,18 @@ std::vector<TensorPtr> LayerNormBackward::apply(const TensorPtr& grad_output) {
     int64_t batch = input_->numel() / D;
 
     // Gradients
-    auto grad_input = zeros(shape, input_->dtype(), input_->device());
-    auto grad_weight = zeros(weight_->shape(), weight_->dtype(), weight_->device());
-    auto grad_bias = zeros(weight_->shape(), weight_->dtype(), weight_->device());
+    auto grad_input = zeros(shape, DType::kFloat32, input_->device());
+    auto grad_weight = zeros(weight_->shape(), kFloat32, weight_->device());
+    auto grad_bias = zeros(weight_->shape(), kFloat32, weight_->device());
+
+    // LayerNorm always computes in FP32. BF16 parameters are decoded
+    // to FP32 for the backward calculation.
+    auto weight_fp32 = (weight_->dtype() == DType::kFloat32)
+        ? weight_
+        : cast(weight_, DType::kFloat32);
 
     const float* x = input_->data<float>();
-    const float* w = weight_->data<float>();
+    const float* w = weight_fp32->data<float>();
     const float* gy = grad_output->data<float>();
     float* gx = grad_input->data<float>();
     float* gw = grad_weight->data<float>();
@@ -495,10 +508,13 @@ std::vector<TensorPtr> RMSNormBackward::apply(const TensorPtr& grad_output) {
     int64_t D = shape.back();
     int64_t batch = input_->numel() / D;
     auto grad_input = zeros(shape, input_->dtype(), input_->device());
-    auto grad_weight = zeros(weight_->shape(), weight_->dtype(), weight_->device());
+    auto grad_weight = zeros(weight_->shape(), kFloat32, weight_->device());
 
     const float* x = input_->data<float>();
-    const float* w = weight_->data<float>();
+    auto weight_fp32 = (weight_->dtype() == DType::kFloat32)
+        ? weight_
+        : cast(weight_, DType::kFloat32);
+    const float* w = weight_fp32->data<float>();
     const float* gy = grad_output->data<float>();
     float* gx = grad_input->data<float>();
     float* gw = grad_weight->data<float>();

operator/finetune_ops/core/memory_manager.h

@@ -8,11 +8,13 @@
 
 #pragma once
 
+#include <string>
 #include <unordered_map>
 #include <vector>
 #include <memory>
 #include <mutex>
 #include <cstddef>
+#include <cstdlib>
 #include <algorithm>
 
 namespace ops {