analysis: re-trim tempogram latency to center the grid-phase peak

analysis/analysis.go

@@ -36,7 +36,10 @@ const AnalysisRate = 44100
 // Bumped to 28 for the PWV4 7-bit band clamp: cached colour previews may
 // carry >127 in d2-d5, which real rekordbox never emits and which restarts
 // an XDJ-AZ on load, so v27 waveforms must re-encode.
-const cacheVersion = 28
+// Bumped to 29 for the TempogramLatencyMs re-trim (30 -> 34ms): every grid
+// shifts +4ms, substantially raising sub-10ms agreement with rekordbox's
+// grids, so v28 grids re-analyze to pick up the corrected phase.
+const cacheVersion = 29
 
 // PWV4Override and PWV5Override, when non-nil, replace every track's color
 // preview / detail waveform at serve time. Set via the --pwv4-override /
@@ -92,6 +95,14 @@ type Result struct {
 	GridEdited       bool      // user manually adjusted the beat grid — serve our blobs, not the on-disk ANLZ
 }
 
+// gridCarry is a user-edited beat grid rescued from a stale-version cache
+// entry, awaiting re-application after re-analysis (see loadFromDisk / Set).
+type gridCarry struct {
+	beats    []float64
+	bpm      float64
+	downbeat int
+}
+
 // Store holds analysis results keyed by track ID.
 // When cacheDir is set, results are persisted to disk using gob encoding,
 // keyed by a hash of the track's file path for stable cache identity.
@@ -115,6 +126,11 @@ type Store struct {
 	// which only covers analyzed/cache-loaded tracks). Set once at startup.
 	TotalTracksFn func() int
 
+	// gridCarry holds user-edited beat grids rescued from stale-version
+	// cache entries (keyed by file path), re-applied by Set after the
+	// track re-analyzes. Guarded by mu.
+	gridCarry map[string]gridCarry
+
 	// Importer, when set, is tried before running our own DSP on a track:
 	// it returns pre-existing analysis (e.g. parsed from a served rekordbox
 	// USB's ANLZ files) or nil to fall through to AnalyzeTrack. An imported
@@ -168,11 +184,24 @@ func (s *Store) Get(trackID uint32) *Result {
 	return nil
 }
 
-// Set stores an analysis result for a track.
+// Set stores an analysis result for a track. If a user-edited beat grid was
+// carried over from a stale-version cache entry for this file (see
+// loadFromDisk), it is re-applied here: a cacheVersion bump refreshes the
+// waveforms but must never revert a grid the user fixed by hand.
 func (s *Store) Set(trackID uint32, r *Result) {
 	s.mu.Lock()
-	s.results[trackID] = r
 	filePath := s.pathMap[trackID]
+	if c, ok := s.gridCarry[filePath]; ok && r != nil && !r.GridEdited {
+		r.Beats = c.beats
+		r.BPM = c.bpm
+		r.DownbeatIndex = c.downbeat
+		r.GridEdited = true
+		r.BeatGrid = nil // regenerate blobs from the carried beats
+		r.BeatGridPQT2 = nil
+		delete(s.gridCarry, filePath)
+		log.Printf("analysis-cache: re-applied user-edited beat grid for track %d after re-analysis", trackID)
+	}
+	s.results[trackID] = r
 	s.mu.Unlock()
 
 	// Persist to disk.
@@ -417,6 +446,20 @@ func (s *Store) loadFromDisk(filePath string) *Result {
 		return nil
 	}
 	if r.CacheVersion != cacheVersion {
+		// A user-edited beat grid must survive the bump: stash it for Set()
+		// to re-apply after re-analysis, and keep the file on disk so the
+		// edit also survives a restart that happens before the track is
+		// re-analyzed (the stash is in-memory only).
+		if r.GridEdited {
+			s.mu.Lock()
+			if s.gridCarry == nil {
+				s.gridCarry = make(map[string]gridCarry)
+			}
+			s.gridCarry[filePath] = gridCarry{beats: r.Beats, bpm: r.BPM, downbeat: r.DownbeatIndex}
+			s.mu.Unlock()
+			log.Printf("analysis-cache: stale version %d (want %d) but grid is user-edited — re-analyzing %s, keeping the edited grid", r.CacheVersion, cacheVersion, filepath.Base(path))
+			return nil
+		}
 		log.Printf("analysis-cache: stale version %d (want %d), discarding %s", r.CacheVersion, cacheVersion, filepath.Base(path))
 		os.Remove(path)
 		return nil

analysis/beatgrid_accuracy_test.go

@@ -76,7 +76,9 @@ func TestBeatGridAccuracy(t *testing.T) {
 	envF("VYNULL_BEAT_REF_CLARW", &ClarityWeight)
 	envF("VYNULL_BEAT_REF_GATE", &HalfBeatGate)
 	envF("VYNULL_BEAT_REF_LATENCY", &TempogramLatencyMs)
-	t.Logf("windowed=%v winSec=%.1f ampW=%.1f clarW=%.1f gate=%.1f", WindowedPhase, WindowSec, AmpWeight, ClarityWeight, HalfBeatGate)
+	envF("VYNULL_BEAT_REF_WARMUP", &BandNormWarmupSec)
+	envF("VYNULL_BEAT_REF_EARLYTAU", &PhaseEarlyTauSec)
+	t.Logf("windowed=%v winSec=%.1f ampW=%.1f clarW=%.1f gate=%.1f warmup=%.1f earlyTau=%.1f", WindowedPhase, WindowSec, AmpWeight, ClarityWeight, HalfBeatGate, BandNormWarmupSec, PhaseEarlyTauSec)
 
 	type result struct {
 		name    string

analysis/bpm.go

@@ -732,6 +732,15 @@ var (
 	BandNormAlpha     = 0.99
 	MultiBandMaxHz    = 4096.0
 
+	// BandNormWarmupSec seeds each band's normalization EMA with its average
+	// magnitude over this many leading seconds, instead of letting it warm up
+	// from zero. From zero, the first ~1.2s of flux frames are divided by a
+	// near-zero running level and come out inflated ~100x; window 0's phasor
+	// magnitude then dominates windowedTempogramPhase's amplitude-cubed combine
+	// on many tracks, effectively reducing "windowed" to "window 0 only".
+	// 0 keeps the legacy from-zero warmup.
+	BandNormWarmupSec = 0.0
+
 	// TempogramLatencyMs is the pipeline group delay (STFT framing + flux + EMA),
 	// which is codec-independent. LossyEncoderDelayMs is added on top for lossy
 	// codecs: rekordbox grids on the raw decoded stream INCLUDING the ~1105-sample
@@ -741,7 +750,12 @@ var (
 	// terms were separated by calibrating the pipeline delay on lossless (the clean
 	// case) and confirming lossy = pipeline + encoder delay reproduces the old
 	// single 55ms constant. See EncoderDelayMs and DetectBeatsWithEncoderDelay.
-	TempogramLatencyMs  = 30.0
+	// 34 = the earlier 30ms calibration re-trimmed +4ms against a library of
+	// imported rekordbox grids: the well-aligned population sat ~9ms early
+	// while a long late tail held the MEAN bias positive — centering the error
+	// distribution's PEAK, not its mean, took <10ms alignment 23.9%->34.3%
+	// with <20/<50ms and the half-beat tail unchanged (held-out validated).
+	TempogramLatencyMs  = 34.0
 	LossyEncoderDelayMs = 25.0
 
 	// HalfBeatGate, when > 0, flips the grid by half a beat if the half-beat
@@ -769,6 +783,16 @@ var (
 	AmpWeight     = 3.0
 	ClarityWeight = 1.0
 
+	// PhaseEarlyTauSec, when > 0, multiplies each window's combine weight by
+	// exp(-startSec/tau): an explicit start-anchor. Rekordbox's grid phase is
+	// anchored near the track start — discovered via the EMA warmup spike,
+	// which accidentally gave window 0 a ~10^5x weight and OUTSCORED the honest
+	// windowed combine by 12pts (<50ms 73.9% vs 61.3% with the warmup seeded).
+	// The decay makes that anchoring deliberate: early clean windows dominate,
+	// but a silent or fade-in intro still falls through to later windows.
+	// 0 disables (pure clarity-weighted combine).
+	PhaseEarlyTauSec = 0.0
+
 	// SnapVerify enables coherence refinement + integer snap of the detected
 	// BPM (see snapVerifyBPM). The autocorrelation lag grid is coarse
 	// (~2.9ms at 44.1k/128) and parabolic peak interpolation lands 0.1-0.5
@@ -868,6 +892,31 @@ func multiBandOnset(samples []float32, sampleRate int) ([]float64, float64) {
 	prevMag := make([]float64, nb)
 	bandMag := make([]float64, nb)
 	mean := make([]float64, nb) // per-band running level for adaptive normalization
+	if UseBandNorm && BandNormWarmupSec > 0 {
+		// Seed the EMA at each band's average level over the leading
+		// BandNormWarmupSec so the first flux frames are divided by a real
+		// level, not a near-zero warming-up one (see BandNormWarmupSec).
+		warm := int(BandNormWarmupSec * float64(sampleRate) / float64(hop))
+		if warm > n {
+			warm = n
+		}
+		for k := 0; k < warm && k < len(re); k++ {
+			lo := 1
+			for b := 0; b < nb; b++ {
+				var s float64
+				for bin := lo; bin < hiBin[b]; bin++ {
+					s += math.Sqrt(re[k][bin]*re[k][bin] + im[k][bin]*im[k][bin])
+				}
+				mean[b] += s
+				lo = hiBin[b]
+			}
+		}
+		if warm > 0 {
+			for b := range mean {
+				mean[b] /= float64(warm)
+			}
+		}
+	}
 	for k := 0; k < n && k < len(re); k++ {
 		lo := 1
 		for b := 0; b < nb; b++ {
@@ -948,6 +997,9 @@ func windowedTempogramPhase(onset []float64, msPerFrame, msPerBeat, winSec, ampW
 			continue
 		}
 		weight := math.Pow(mag, ampW) * math.Pow(mag/esum, clarW)
+		if PhaseEarlyTauSec > 0 {
+			weight *= math.Exp(-float64(start) * msPerFrame / 1000 / PhaseEarlyTauSec)
+		}
 		Zr += weight * zr / mag
 		Zi += weight * zi / mag
 	}

analysis/gridcarry_test.go

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package analysis
+
+import (
+	"encoding/gob"
+	"os"
+	"testing"
+)
+
+// TestGridEditSurvivesCacheBump pins the contract that a user-edited beat
+// grid survives a cacheVersion bump: the stale entry is re-analyzed for
+// fresh waveforms, but the hand-fixed grid is carried over instead of
+// being reverted to the detector's (wrong) phase. Before this, every bump
+// silently deleted all grid edits.
+func TestGridEditSurvivesCacheBump(t *testing.T) {
+	dir := t.TempDir()
+	s := NewStoreWithCache(dir)
+	const fp = "/music/some-track.flac"
+	s.SetPath(7, fp)
+
+	// A previous-version cache entry carrying a user-edited grid.
+	stale := &Result{
+		CacheVersion:  cacheVersion - 1,
+		GridEdited:    true,
+		BPM:           122,
+		Beats:         []float64{0, 491.8, 983.6},
+		DownbeatIndex: 0,
+	}
+	f, err := os.Create(s.cacheFile(fp))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := gob.NewEncoder(f).Encode(stale); err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+
+	// Get must treat it as a miss (forcing re-analysis), but keep the file
+	// so the edit also survives a restart before re-analysis happens.
+	if r := s.Get(7); r != nil {
+		t.Fatalf("stale entry served: %+v", r)
+	}
+	if _, err := os.Stat(s.cacheFile(fp)); err != nil {
+		t.Fatal("stale grid-edited cache file was deleted — edit would not survive a restart")
+	}
+
+	// Fresh analysis arrives with the detector's (wrong) grid: the edit
+	// must win.
+	fresh := &Result{CacheVersion: cacheVersion, BPM: 122, Beats: []float64{302.4, 794.2}, DownbeatIndex: 1}
+	s.Set(7, fresh)
+	got := s.Get(7)
+	if got == nil || !got.GridEdited || got.Beats[0] != 0 || got.DownbeatIndex != 0 {
+		t.Fatalf("edited grid not re-applied: %+v", got)
+	}
+
+	// And the merged result must be what persisted: a second store sees it.
+	s2 := NewStoreWithCache(dir)
+	s2.SetPath(7, fp)
+	got2 := s2.Get(7)
+	if got2 == nil || !got2.GridEdited || got2.Beats[0] != 0 {
+		t.Fatalf("persisted result lost the edit: %+v", got2)
+	}
+}