From 46889f2af99e50bf203d283b72a97cd932808e20 Mon Sep 17 00:00:00 2001 From: Benjamin Borbe Date: Tue, 17 Nov 2026 17:59:14 +0100 Subject: [PATCH 1/3] feat(index): add page index types and metadata-only readers --- mocks/index-directory-lister.go | 119 ++++++++++ mocks/index-metrics.go | 111 +++++++++ mocks/index-page-reader.go | 210 ++++++++++++++++++ pkg/index/doc.go | 11 + pkg/index/fingerprint_darwin.go | 30 +++ pkg/index/fingerprint_linux.go | 30 +++ pkg/index/index_suite_test.go | 24 ++ pkg/index/key.go | 22 ++ pkg/index/lister.go | 53 +++++ pkg/index/metrics.go | 75 +++++++ pkg/index/norace_enabled_test.go | 11 + pkg/index/race_enabled_test.go | 11 + pkg/index/reader.go | 82 +++++++ pkg/index/reader_internal_test.go | 31 +++ pkg/index/reader_test.go | 137 ++++++++++++ pkg/index/seams.go | 80 +++++++ .../068-pkg-index-metadata-page-index.md | 170 ++++++++++++++ 17 files changed, 1207 insertions(+) create mode 100644 mocks/index-directory-lister.go create mode 100644 mocks/index-metrics.go create mode 100644 mocks/index-page-reader.go create mode 100644 pkg/index/doc.go create mode 100644 pkg/index/fingerprint_darwin.go create mode 100644 pkg/index/fingerprint_linux.go create mode 100644 pkg/index/index_suite_test.go create mode 100644 pkg/index/key.go create mode 100644 pkg/index/lister.go create mode 100644 pkg/index/metrics.go create mode 100644 pkg/index/norace_enabled_test.go create mode 100644 pkg/index/race_enabled_test.go create mode 100644 pkg/index/reader.go create mode 100644 pkg/index/reader_internal_test.go create mode 100644 pkg/index/reader_test.go create mode 100644 pkg/index/seams.go create mode 100644 specs/in-progress/068-pkg-index-metadata-page-index.md diff --git a/mocks/index-directory-lister.go b/mocks/index-directory-lister.go new file mode 100644 index 0000000..6442329 --- /dev/null +++ b/mocks/index-directory-lister.go @@ -0,0 +1,119 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/bborbe/vault-cli/pkg/index" +) + +type IndexDirectoryLister struct { + ListFilesStub func(context.Context, string, string) ([]index.FileEntry, error) + listFilesMutex sync.RWMutex + listFilesArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + } + listFilesReturns struct { + result1 []index.FileEntry + result2 error + } + listFilesReturnsOnCall map[int]struct { + result1 []index.FileEntry + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *IndexDirectoryLister) ListFiles(arg1 context.Context, arg2 string, arg3 string) ([]index.FileEntry, error) { + fake.listFilesMutex.Lock() + ret, specificReturn := fake.listFilesReturnsOnCall[len(fake.listFilesArgsForCall)] + fake.listFilesArgsForCall = append(fake.listFilesArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.ListFilesStub + fakeReturns := fake.listFilesReturns + fake.recordInvocation("ListFiles", []interface{}{arg1, arg2, arg3}) + fake.listFilesMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexDirectoryLister) ListFilesCallCount() int { + fake.listFilesMutex.RLock() + defer fake.listFilesMutex.RUnlock() + return len(fake.listFilesArgsForCall) +} + +func (fake *IndexDirectoryLister) ListFilesCalls(stub func(context.Context, string, string) ([]index.FileEntry, error)) { + fake.listFilesMutex.Lock() + defer fake.listFilesMutex.Unlock() + fake.ListFilesStub = stub +} + +func (fake *IndexDirectoryLister) ListFilesArgsForCall(i int) (context.Context, string, string) { + fake.listFilesMutex.RLock() + defer fake.listFilesMutex.RUnlock() + argsForCall := fake.listFilesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *IndexDirectoryLister) ListFilesReturns(result1 []index.FileEntry, result2 error) { + fake.listFilesMutex.Lock() + defer fake.listFilesMutex.Unlock() + fake.ListFilesStub = nil + fake.listFilesReturns = struct { + result1 []index.FileEntry + result2 error + }{result1, result2} +} + +func (fake *IndexDirectoryLister) ListFilesReturnsOnCall(i int, result1 []index.FileEntry, result2 error) { + fake.listFilesMutex.Lock() + defer fake.listFilesMutex.Unlock() + fake.ListFilesStub = nil + if fake.listFilesReturnsOnCall == nil { + fake.listFilesReturnsOnCall = make(map[int]struct { + result1 []index.FileEntry + result2 error + }) + } + fake.listFilesReturnsOnCall[i] = struct { + result1 []index.FileEntry + result2 error + }{result1, result2} +} + +func (fake *IndexDirectoryLister) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *IndexDirectoryLister) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ index.DirectoryLister = new(IndexDirectoryLister) diff --git a/mocks/index-metrics.go b/mocks/index-metrics.go new file mode 100644 index 0000000..775ea4b --- /dev/null +++ b/mocks/index-metrics.go @@ -0,0 +1,111 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "sync" + + "github.com/bborbe/vault-cli/pkg/index" +) + +type IndexMetrics struct { + ChangedPagesUnboundedStub func(string) + changedPagesUnboundedMutex sync.RWMutex + changedPagesUnboundedArgsForCall []struct { + arg1 string + } + FileReadStub func(string) + fileReadMutex sync.RWMutex + fileReadArgsForCall []struct { + arg1 string + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *IndexMetrics) ChangedPagesUnbounded(arg1 string) { + fake.changedPagesUnboundedMutex.Lock() + fake.changedPagesUnboundedArgsForCall = append(fake.changedPagesUnboundedArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ChangedPagesUnboundedStub + fake.recordInvocation("ChangedPagesUnbounded", []interface{}{arg1}) + fake.changedPagesUnboundedMutex.Unlock() + if stub != nil { + fake.ChangedPagesUnboundedStub(arg1) + } +} + +func (fake *IndexMetrics) ChangedPagesUnboundedCallCount() int { + fake.changedPagesUnboundedMutex.RLock() + defer fake.changedPagesUnboundedMutex.RUnlock() + return len(fake.changedPagesUnboundedArgsForCall) +} + +func (fake *IndexMetrics) ChangedPagesUnboundedCalls(stub func(string)) { + fake.changedPagesUnboundedMutex.Lock() + defer fake.changedPagesUnboundedMutex.Unlock() + fake.ChangedPagesUnboundedStub = stub +} + +func (fake *IndexMetrics) ChangedPagesUnboundedArgsForCall(i int) string { + fake.changedPagesUnboundedMutex.RLock() + defer fake.changedPagesUnboundedMutex.RUnlock() + argsForCall := fake.changedPagesUnboundedArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *IndexMetrics) FileRead(arg1 string) { + fake.fileReadMutex.Lock() + fake.fileReadArgsForCall = append(fake.fileReadArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.FileReadStub + fake.recordInvocation("FileRead", []interface{}{arg1}) + fake.fileReadMutex.Unlock() + if stub != nil { + fake.FileReadStub(arg1) + } +} + +func (fake *IndexMetrics) FileReadCallCount() int { + fake.fileReadMutex.RLock() + defer fake.fileReadMutex.RUnlock() + return len(fake.fileReadArgsForCall) +} + +func (fake *IndexMetrics) FileReadCalls(stub func(string)) { + fake.fileReadMutex.Lock() + defer fake.fileReadMutex.Unlock() + fake.FileReadStub = stub +} + +func (fake *IndexMetrics) FileReadArgsForCall(i int) string { + fake.fileReadMutex.RLock() + defer fake.fileReadMutex.RUnlock() + argsForCall := fake.fileReadArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *IndexMetrics) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *IndexMetrics) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ index.Metrics = new(IndexMetrics) diff --git a/mocks/index-page-reader.go b/mocks/index-page-reader.go new file mode 100644 index 0000000..adcefa7 --- /dev/null +++ b/mocks/index-page-reader.go @@ -0,0 +1,210 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/index" +) + +type IndexPageReader struct { + ReadEntryStub func(context.Context, string, string, string) (*index.Entry, index.FileFingerprint, error) + readEntryMutex sync.RWMutex + readEntryArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + } + readEntryReturns struct { + result1 *index.Entry + result2 index.FileFingerprint + result3 error + } + readEntryReturnsOnCall map[int]struct { + result1 *index.Entry + result2 index.FileFingerprint + result3 error + } + ReadPageStub func(context.Context, string, string, string) (*domain.Page, error) + readPageMutex sync.RWMutex + readPageArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + } + readPageReturns struct { + result1 *domain.Page + result2 error + } + readPageReturnsOnCall map[int]struct { + result1 *domain.Page + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *IndexPageReader) ReadEntry(arg1 context.Context, arg2 string, arg3 string, arg4 string) (*index.Entry, index.FileFingerprint, error) { + fake.readEntryMutex.Lock() + ret, specificReturn := fake.readEntryReturnsOnCall[len(fake.readEntryArgsForCall)] + fake.readEntryArgsForCall = append(fake.readEntryArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + }{arg1, arg2, arg3, arg4}) + stub := fake.ReadEntryStub + fakeReturns := fake.readEntryReturns + fake.recordInvocation("ReadEntry", []interface{}{arg1, arg2, arg3, arg4}) + fake.readEntryMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *IndexPageReader) ReadEntryCallCount() int { + fake.readEntryMutex.RLock() + defer fake.readEntryMutex.RUnlock() + return len(fake.readEntryArgsForCall) +} + +func (fake *IndexPageReader) ReadEntryCalls(stub func(context.Context, string, string, string) (*index.Entry, index.FileFingerprint, error)) { + fake.readEntryMutex.Lock() + defer fake.readEntryMutex.Unlock() + fake.ReadEntryStub = stub +} + +func (fake *IndexPageReader) ReadEntryArgsForCall(i int) (context.Context, string, string, string) { + fake.readEntryMutex.RLock() + defer fake.readEntryMutex.RUnlock() + argsForCall := fake.readEntryArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *IndexPageReader) ReadEntryReturns(result1 *index.Entry, result2 index.FileFingerprint, result3 error) { + fake.readEntryMutex.Lock() + defer fake.readEntryMutex.Unlock() + fake.ReadEntryStub = nil + fake.readEntryReturns = struct { + result1 *index.Entry + result2 index.FileFingerprint + result3 error + }{result1, result2, result3} +} + +func (fake *IndexPageReader) ReadEntryReturnsOnCall(i int, result1 *index.Entry, result2 index.FileFingerprint, result3 error) { + fake.readEntryMutex.Lock() + defer fake.readEntryMutex.Unlock() + fake.ReadEntryStub = nil + if fake.readEntryReturnsOnCall == nil { + fake.readEntryReturnsOnCall = make(map[int]struct { + result1 *index.Entry + result2 index.FileFingerprint + result3 error + }) + } + fake.readEntryReturnsOnCall[i] = struct { + result1 *index.Entry + result2 index.FileFingerprint + result3 error + }{result1, result2, result3} +} + +func (fake *IndexPageReader) ReadPage(arg1 context.Context, arg2 string, arg3 string, arg4 string) (*domain.Page, error) { + fake.readPageMutex.Lock() + ret, specificReturn := fake.readPageReturnsOnCall[len(fake.readPageArgsForCall)] + fake.readPageArgsForCall = append(fake.readPageArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + }{arg1, arg2, arg3, arg4}) + stub := fake.ReadPageStub + fakeReturns := fake.readPageReturns + fake.recordInvocation("ReadPage", []interface{}{arg1, arg2, arg3, arg4}) + fake.readPageMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexPageReader) ReadPageCallCount() int { + fake.readPageMutex.RLock() + defer fake.readPageMutex.RUnlock() + return len(fake.readPageArgsForCall) +} + +func (fake *IndexPageReader) ReadPageCalls(stub func(context.Context, string, string, string) (*domain.Page, error)) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = stub +} + +func (fake *IndexPageReader) ReadPageArgsForCall(i int) (context.Context, string, string, string) { + fake.readPageMutex.RLock() + defer fake.readPageMutex.RUnlock() + argsForCall := fake.readPageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *IndexPageReader) ReadPageReturns(result1 *domain.Page, result2 error) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = nil + fake.readPageReturns = struct { + result1 *domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageReader) ReadPageReturnsOnCall(i int, result1 *domain.Page, result2 error) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = nil + if fake.readPageReturnsOnCall == nil { + fake.readPageReturnsOnCall = make(map[int]struct { + result1 *domain.Page + result2 error + }) + } + fake.readPageReturnsOnCall[i] = struct { + result1 *domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageReader) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *IndexPageReader) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ index.PageReader = new(IndexPageReader) diff --git a/pkg/index/doc.go b/pkg/index/doc.go new file mode 100644 index 0000000..30c7507 --- /dev/null +++ b/pkg/index/doc.go @@ -0,0 +1,11 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package index keeps the metadata of each vault folder's pages in memory so a +// list read can be answered without opening a vault file. It never holds a +// page body: listed pages carry empty Content, each entry carries its +// pre-parsed Open Questions, and a body is read from disk on request only. +// +// See docs/page-index.md for the full contract. +package index diff --git a/pkg/index/fingerprint_darwin.go b/pkg/index/fingerprint_darwin.go new file mode 100644 index 0000000..e2c4afd --- /dev/null +++ b/pkg/index/fingerprint_darwin.go @@ -0,0 +1,30 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin + +package index + +import ( + "os" + "syscall" + "time" +) + +// statFingerprint returns path's size, modification time and status-change +// time in UTC. It follows symlinks (os.Stat, not os.Lstat). +func statFingerprint(path string) (FileFingerprint, error) { + info, err := os.Stat(path) + if err != nil { + return FileFingerprint{}, err + } + fingerprint := FileFingerprint{ + Size: info.Size(), + ModTime: info.ModTime().UTC(), + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + fingerprint.StatusChangeTime = time.Unix(stat.Ctimespec.Unix()).UTC() + } + return fingerprint, nil +} diff --git a/pkg/index/fingerprint_linux.go b/pkg/index/fingerprint_linux.go new file mode 100644 index 0000000..2ddfc00 --- /dev/null +++ b/pkg/index/fingerprint_linux.go @@ -0,0 +1,30 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package index + +import ( + "os" + "syscall" + "time" +) + +// statFingerprint returns path's size, modification time and status-change +// time in UTC. It follows symlinks (os.Stat, not os.Lstat). +func statFingerprint(path string) (FileFingerprint, error) { + info, err := os.Stat(path) + if err != nil { + return FileFingerprint{}, err + } + fingerprint := FileFingerprint{ + Size: info.Size(), + ModTime: info.ModTime().UTC(), + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + fingerprint.StatusChangeTime = time.Unix(stat.Ctim.Unix()).UTC() + } + return fingerprint, nil +} diff --git a/pkg/index/index_suite_test.go b/pkg/index/index_suite_test.go new file mode 100644 index 0000000..d7cffec --- /dev/null +++ b/pkg/index/index_suite_test.go @@ -0,0 +1,24 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6@v6.12.2 -generate +func TestIndex(t *testing.T) { + time.Local = time.UTC + format.TruncatedDiff = false + RegisterFailHandler(Fail) + suiteConfig, reporterConfig := GinkgoConfiguration() + suiteConfig.Timeout = 60 * time.Second + RunSpecs(t, "Index Suite", suiteConfig, reporterConfig) +} diff --git a/pkg/index/key.go b/pkg/index/key.go new file mode 100644 index 0000000..24322f0 --- /dev/null +++ b/pkg/index/key.go @@ -0,0 +1,22 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import "path/filepath" + +// Key identifies one indexed folder: a vault root plus a vault-relative pages dir. +type Key struct { + VaultPath string + PagesDir string +} + +// NewKey builds a Key, applying filepath.Clean to both parts so the read side +// and the event side derive identical keys from equivalent paths. +func NewKey(vaultPath, pagesDir string) Key { + return Key{ + VaultPath: filepath.Clean(vaultPath), + PagesDir: filepath.Clean(pagesDir), + } +} diff --git a/pkg/index/lister.go b/pkg/index/lister.go new file mode 100644 index 0000000..9c43af2 --- /dev/null +++ b/pkg/index/lister.go @@ -0,0 +1,53 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/bborbe/errors" +) + +// directoryLister lists one folder's page files. It mirrors vault-cli's folder +// walk so the entry set, order and exclusions match its own listing. +type directoryLister struct{} + +// ListFiles returns the folder's page-file entries in os.ReadDir order. +func (l *directoryLister) ListFiles( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]FileEntry, error) { + targetDir := filepath.Join(vaultPath, pagesDir) + entries, err := os.ReadDir(targetDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, errors.Wrapf(ctx, err, "read directory %s", targetDir) + } + + result := make([]FileEntry, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if !strings.HasSuffix(entry.Name(), ".md") { + continue + } + fingerprint, statErr := statFingerprint(filepath.Join(targetDir, entry.Name())) + if statErr != nil { + // A zero fingerprint keeps the entry so the reader's exclusion path + // runs, and stops a later stat-diff from re-reading it every pass. + fingerprint = FileFingerprint{} + } + result = append(result, FileEntry{Name: entry.Name(), Fingerprint: fingerprint}) + } + return result, nil +} diff --git a/pkg/index/metrics.go b/pkg/index/metrics.go new file mode 100644 index 0000000..797b437 --- /dev/null +++ b/pkg/index/metrics.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +// The reasons a single-file read happens, reported through Metrics.FileRead. +const ( + ReadReasonBuild = "build" + ReadReasonEvent = "event" + ReadReasonWrite = "write" + ReadReasonRescan = "rescan" + ReadReasonReload = "reload" +) + +// ReadReasons is every reason Metrics.FileRead can be called with. +var ReadReasons = []string{ + ReadReasonBuild, + ReadReasonEvent, + ReadReasonWrite, + ReadReasonRescan, + ReadReasonReload, +} + +// The reasons a changed set cannot be bounded, reported through +// Metrics.ChangedPagesUnbounded. +const ( + UnboundedUnknownKey = "unknown_key" + UnboundedRevisionAhead = "revision_ahead" + UnboundedWindowNotRetained = "window_not_retained" + UnboundedForcedReload = "forced_reload" + UnboundedFirstPublication = "first_publication" + UnboundedFolderMarkExplicit = "folder_mark_explicit" + UnboundedFolderMarkWidenedName = "folder_mark_widened_name_not_in_snapshot" + UnboundedFolderMarkWidenedFile = "folder_mark_widened_file_absent" + UnboundedFolderMarkWidenedNameInvalid = "folder_mark_widened_invalid_name" + UnboundedFolderMarkWidenedKey = "folder_mark_widened_key_unpublished" +) + +// UnboundedReasons is every reason Metrics.ChangedPagesUnbounded can be called +// with. +var UnboundedReasons = []string{ + UnboundedUnknownKey, + UnboundedRevisionAhead, + UnboundedWindowNotRetained, + UnboundedForcedReload, + UnboundedFirstPublication, + UnboundedFolderMarkExplicit, + UnboundedFolderMarkWidenedName, + UnboundedFolderMarkWidenedFile, + UnboundedFolderMarkWidenedNameInvalid, + UnboundedFolderMarkWidenedKey, +} + +//counterfeiter:generate -o ../../mocks/index-metrics.go --fake-name IndexMetrics . Metrics + +// Metrics receives the index's counters. A consumer adapts it to its own +// metrics stack; the package itself imports none. +type Metrics interface { + // FileRead counts one single-file page read by reason. + FileRead(reason string) + // ChangedPagesUnbounded counts one refused changed-set bound by reason. + ChangedPagesUnbounded(reason string) +} + +// NewNoopMetrics returns a Metrics that records nothing. +func NewNoopMetrics() Metrics { + return noopMetrics{} +} + +type noopMetrics struct{} + +func (noopMetrics) FileRead(string) {} + +func (noopMetrics) ChangedPagesUnbounded(string) {} diff --git a/pkg/index/norace_enabled_test.go b/pkg/index/norace_enabled_test.go new file mode 100644 index 0000000..393956d --- /dev/null +++ b/pkg/index/norace_enabled_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !race + +package index_test + +// raceDetectorEnabled reports at compile time that the test binary was not +// built with the race detector, so allocation measurements are meaningful. +const raceDetectorEnabled = false diff --git a/pkg/index/race_enabled_test.go b/pkg/index/race_enabled_test.go new file mode 100644 index 0000000..bff81de --- /dev/null +++ b/pkg/index/race_enabled_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build race + +package index_test + +// raceDetectorEnabled reports at compile time that the test binary was built +// with the race detector, whose runtime inflates allocation measurements. +const raceDetectorEnabled = true diff --git a/pkg/index/reader.go b/pkg/index/reader.go new file mode 100644 index 0000000..cebed2b --- /dev/null +++ b/pkg/index/reader.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "path/filepath" + "strings" + + "github.com/bborbe/errors" + + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" +) + +// pageReader reads page files through vault-cli's single-page read, so a page +// parsed here is the page vault-cli's own folder listing produces. +type pageReader struct { + pages storage.PageStorage +} + +// ReadEntry parses one page file, projects its Open Questions, drops the body +// and reports the file's pre-read fingerprint. +func (r *pageReader) ReadEntry( + ctx context.Context, + vaultPath string, + pagesDir string, + filename string, +) (*Entry, FileFingerprint, error) { + filePath := filepath.Join(vaultPath, pagesDir, filename) + + fingerprint, err := statFingerprint(filePath) + if err != nil { + return nil, FileFingerprint{}, errors.Wrapf(ctx, err, "stat %s", filePath) + } + + page, err := r.pages.ReadPage(ctx, vaultPath, pagesDir, strings.TrimSuffix(filename, ".md")) + if err != nil { + return nil, fingerprint, errors.Wrapf(ctx, err, "read page %s", filePath) + } + questions, err := projectOpenQuestions(ctx, string(page.Content)) + if err != nil { + return nil, fingerprint, errors.Wrapf(ctx, err, "parse open questions %s", filePath) + } + metadataOnly := *page + metadataOnly.Content = "" + return &Entry{Page: &metadataOnly, OpenQuestions: questions}, fingerprint, nil +} + +// ReadPage reads the full page, body included, through the underlying storage. +func (r *pageReader) ReadPage( + ctx context.Context, + vaultPath string, + pagesDir string, + name string, +) (*domain.Page, error) { + page, err := r.pages.ReadPage(ctx, vaultPath, pagesDir, name) + if err != nil { + return nil, errors.Wrapf(ctx, err, "read page %s", name) + } + return page, nil +} + +// projectOpenQuestions parses the body's Open Questions section and returns +// each item's index and question text. Every text is cloned, so no retained +// string pins the body in memory. The result is never nil. +func projectOpenQuestions(ctx context.Context, content string) ([]domain.OpenQuestion, error) { + items, err := storage.ParseOpenQuestions(ctx, content) + if err != nil { + return nil, errors.Wrap(ctx, err, "parse open questions") + } + questions := make([]domain.OpenQuestion, 0, len(items)) + for _, item := range items { + questions = append(questions, domain.OpenQuestion{ + Index: item.Index, + Text: strings.Clone(item.Question), + }) + } + return questions, nil +} diff --git a/pkg/index/reader_internal_test.go b/pkg/index/reader_internal_test.go new file mode 100644 index 0000000..b62ab91 --- /dev/null +++ b/pkg/index/reader_internal_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "unsafe" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("projectOpenQuestions", func() { + ginkgo.It("does not retain open-question text that shares memory with the file body", func() { + body := "## Open Questions\n\n- first retained question\n- second retained question\n" + questions, err := projectOpenQuestions(context.Background(), body) + Expect(err).NotTo(HaveOccurred()) + Expect(questions).To(HaveLen(2)) + + bodyStart := uintptr(unsafe.Pointer(unsafe.StringData(body))) + bodyEnd := bodyStart + uintptr(len(body)) + for _, question := range questions { + Expect(question.Text).NotTo(BeEmpty()) + textStart := uintptr(unsafe.Pointer(unsafe.StringData(question.Text))) + outside := textStart+uintptr(len(question.Text)) <= bodyStart || textStart >= bodyEnd + Expect(outside).To(BeTrue(), "text %q shares memory with the body", question.Text) + } + }) +}) diff --git a/pkg/index/reader_test.go b/pkg/index/reader_test.go new file mode 100644 index 0000000..05d168b --- /dev/null +++ b/pkg/index/reader_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/index" + "github.com/bborbe/vault-cli/pkg/storage" +) + +const readerFolder = "Tasks" + +var _ = Describe("PageReader", func() { + var ( + ctx context.Context + vaultDir string + reader index.PageReader + ) + + write := func(name, content string) { + Expect(os.WriteFile(filepath.Join(vaultDir, readerFolder, name), []byte(content), 0600)). + To(Succeed()) + } + + BeforeEach(func() { + ctx = context.Background() + vaultDir = GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(vaultDir, readerFolder), 0750)).To(Succeed()) + reader = index.NewPageReader(storage.NewPageStorage(nil)) + }) + + It("projects open questions from ParseOpenQuestions", func() { + body := "---\nstatus: in_progress\n---\n## Open Questions\n\n- first question\n- second question → answer\n\n## Notes\n- not a question\n" + write("Task A.md", body) + + entry, fingerprint, err := reader.ReadEntry(ctx, vaultDir, readerFolder, "Task A.md") + Expect(err).NotTo(HaveOccurred()) + Expect(fingerprint.Size).To(Equal(int64(len(body)))) + + items, err := storage.ParseOpenQuestions(ctx, body) + Expect(err).NotTo(HaveOccurred()) + expected := make([]domain.OpenQuestion, 0, len(items)) + for _, item := range items { + expected = append(expected, domain.OpenQuestion{Index: item.Index, Text: item.Question}) + } + Expect(entry.OpenQuestions).To(Equal(expected)) + Expect(entry.OpenQuestions).To(HaveLen(2)) + }) + + It("returns an empty non-nil slice when the page has no Open Questions section", func() { + write("Task B.md", "---\nstatus: next\n---\n## Notes\n- nothing\n") + + entry, _, err := reader.ReadEntry(ctx, vaultDir, readerFolder, "Task B.md") + Expect(err).NotTo(HaveOccurred()) + Expect(entry.OpenQuestions).NotTo(BeNil()) + Expect(entry.OpenQuestions).To(BeEmpty()) + }) + + It("drops the body but keeps frontmatter, name, path and modification date", func() { + write("Task C.md", "---\nstatus: next\njira: ABC-1\n---\nbody text\n") + + entry, _, err := reader.ReadEntry(ctx, vaultDir, readerFolder, "Task C.md") + Expect(err).NotTo(HaveOccurred()) + reference, err := storage.NewPageStorage(nil).ReadPage(ctx, vaultDir, readerFolder, "Task C") + Expect(err).NotTo(HaveOccurred()) + + Expect(string(entry.Page.Content)).To(BeEmpty()) + Expect(entry.Page.RawMap()).To(Equal(reference.RawMap())) + Expect(entry.Page.FileMetadata).To(Equal(reference.FileMetadata)) + }) + + It("reads the full page including its body on ReadPage", func() { + body := "---\nstatus: next\n---\nfull body\n" + write("Task D.md", body) + + page, err := reader.ReadPage(ctx, vaultDir, readerFolder, "Task D") + Expect(err).NotTo(HaveOccurred()) + Expect(string(page.Content)).To(Equal(body)) + }) + + It("reports an error and a zero fingerprint for a missing file", func() { + _, fingerprint, err := reader.ReadEntry(ctx, vaultDir, readerFolder, "Missing.md") + Expect(err).To(HaveOccurred()) + Expect(fingerprint).To(Equal(index.FileFingerprint{})) + }) +}) + +var _ = Describe("DirectoryLister", func() { + It("lists .md files only, skipping directories, in directory order", func() { + ctx := context.Background() + vaultDir := GinkgoT().TempDir() + dir := filepath.Join(vaultDir, readerFolder) + Expect(os.MkdirAll(filepath.Join(dir, "sub.md"), 0750)).To(Succeed()) + for _, name := range []string{"b.md", "a.md", "c.txt"} { + Expect(os.WriteFile(filepath.Join(dir, name), []byte("x"), 0600)).To(Succeed()) + } + + entries, err := index.NewDirectoryLister().ListFiles(ctx, vaultDir, readerFolder) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, entry := range entries { + names = append(names, entry.Name) + Expect(entry.Fingerprint.Size).To(Equal(int64(1))) + } + Expect(names).To(Equal([]string{"a.md", "b.md"})) + }) + + It("returns nil for a missing folder", func() { + entries, err := index.NewDirectoryLister(). + ListFiles(context.Background(), GinkgoT().TempDir(), "absent") + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeNil()) + }) +}) + +var _ = Describe("NewKey", func() { + It("cleans both parts so equivalent paths share a key", func() { + Expect(index.NewKey("/vault/", "./Tasks/")).To(Equal(index.NewKey("/vault", "Tasks"))) + }) +}) + +var _ = Describe("Metrics", func() { + It("accepts calls on the no-op implementation", func() { + metrics := index.NewNoopMetrics() + metrics.FileRead(index.ReadReasonBuild) + metrics.ChangedPagesUnbounded(index.UnboundedUnknownKey) + }) +}) diff --git a/pkg/index/seams.go b/pkg/index/seams.go new file mode 100644 index 0000000..7f535a0 --- /dev/null +++ b/pkg/index/seams.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "time" + + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" +) + +// FileFingerprint is the pre-read stat of one page file. Size, ModTime and +// StatusChangeTime come from a single stat that follows symlinks. +type FileFingerprint struct { + Size int64 + ModTime time.Time + StatusChangeTime time.Time +} + +// FileEntry is one directory entry with its current fingerprint. +type FileEntry struct { + Name string + Fingerprint FileFingerprint +} + +// Entry is one indexed page: the metadata-only page (Content always empty) +// plus its pre-parsed Open Questions. OpenQuestions is never nil, and every +// Text is a copy that shares no memory with the file body. +// +// A published Entry and its Page are read-only for every consumer. +type Entry struct { + Page *domain.Page + OpenQuestions []domain.OpenQuestion +} + +//counterfeiter:generate -o ../../mocks/index-page-reader.go --fake-name IndexPageReader . PageReader + +// PageReader reads page files from a folder. +type PageReader interface { + // ReadEntry reads one page file and returns its metadata-only entry and + // pre-read fingerprint. The filename is the file's base name including the + // ".md" suffix. A non-nil error means the file is excluded (absent, + // unreadable, unparsable, or a symlink out of the vault) and the returned + // fingerprint is still the best available (zero when the stat failed). + ReadEntry( + ctx context.Context, + vaultPath string, + pagesDir string, + filename string, + ) (*Entry, FileFingerprint, error) + // ReadPage reads the full page, body included, by its bare base name + // (without ".md"). The result is never cached. + ReadPage( + ctx context.Context, + vaultPath string, + pagesDir string, + name string, + ) (*domain.Page, error) +} + +//counterfeiter:generate -o ../../mocks/index-directory-lister.go --fake-name IndexDirectoryLister . DirectoryLister + +// DirectoryLister lists one folder's page-file entries with their current +// fingerprints, in os.ReadDir order (filename ascending). +type DirectoryLister interface { + ListFiles(ctx context.Context, vaultPath string, pagesDir string) ([]FileEntry, error) +} + +// NewPageReader returns the production reader over vault-cli's page storage. +func NewPageReader(pages storage.PageStorage) PageReader { + return &pageReader{pages: pages} +} + +// NewDirectoryLister returns the production directory lister. +func NewDirectoryLister() DirectoryLister { + return &directoryLister{} +} diff --git a/specs/in-progress/068-pkg-index-metadata-page-index.md b/specs/in-progress/068-pkg-index-metadata-page-index.md new file mode 100644 index 0000000..1e16a93 --- /dev/null +++ b/specs/in-progress/068-pkg-index-metadata-page-index.md @@ -0,0 +1,170 @@ +--- +status: approved +approved: "2026-11-17T16:55:24Z" +branch: dark-factory/pkg-index-metadata-page-index +--- + +## Summary + +- vault-cli gains a shared, exported page index library: it keeps every page's frontmatter, name, path and modification time in memory per folder, but never the page body. +- Each indexed page also carries its pre-parsed Open Questions, so the one body-derived field a consumer needs on a hot path is answered without re-reading files. +- A body is read only on demand, straight from disk, and is never retained by the index. +- An optional on-disk cache lets a restarted process skip re-parsing unchanged files; it stores no bodies and recognises (and discards) the older vault-ui cache format by name. +- The library is a port of vault-ui's existing page index (same refresh/staleness contract), made body-free and free of vault-ui's logging and metrics stack, so vault-ui and vault-api can both adopt it in follow-up migrations. + +## Problem + +vault-ui holds about 1.5 GB of Go heap (6.5–7M objects for ~48k indexed markdown files, 481 MB raw markdown) because its page index keeps every page in memory with its full file body, and its on-disk cache persists the bodies too (~760 MB file). Only one vault-ui consumer reads the body — it derives each task's Open Questions — and every other consumer reads frontmatter, path and modification time only. vault-api separately re-reads every task and goal page with full bodies on each git HEAD change, only to discard the bodies after building typed items. The index logic lives in vault-ui alone, so vault-api cannot share it and the two read paths diverge. The memory cost makes vault-ui swap on the operator's machine. + +## Goal + +vault-cli ships an exported `pkg/index` package whose page index serves the same pages, in the same order, with the same frontmatter, names, paths and modification times as vault-cli's folder listing — with every body empty — plus each page's Open Questions, from immutable per-folder snapshots kept current by the ported incremental-refresh contract. A body is available on request by reading the file. An optional bolt cache persists the metadata-only records under store format version 2. A contract document in vault-cli describes the behaviour. vault-ui and vault-api can replace their own index/list paths with this package without changing what they serve. + +## Non-goals + +- Do NOT switch vault-cli's own CLI commands to the index — the CLI keeps reading files directly. +- Do NOT move typed indexing (task/goal items, filter sets, queries, readiness) out of vault-api — the shared layer is page metadata only. +- Do NOT project or intern frontmatter keys — the full frontmatter map is retained; a projection is a separate follow-up spec if heap profiling demands it. +- Do NOT shrink, compact or migrate the bolt file — a mismatching store is discarded, never converted. +- Do NOT add configuration for the store location, rescan interval, timeouts or change-log size — ported constants are invariant; a consumer needing variation is a separate spec. +- Do NOT cache bodies anywhere (memory or store) — not even an opt-in body cache. +- Do NOT port vault-ui's board-level sections (task-list snapshot, body cache, frame ordering, watcher wiring) — they stay in vault-ui. +- Live heap, pprof and pod-memory verification belong to the consumer migrations, not this spec. +- **Follow-up consumer migrations (out of this spec's prompts; tracked by the vault task "vault-ui and vault-api Share One Memory-Efficient Bolt-Backed Index Library", SC1–SC6):** + - vault-api: bump vault-cli; index build takes a `index.PageIndex` constructed without a store, calls `Refresh` for the tasks and goals keys then `ListPages` through it, replacing its two direct `ListPages` calls; the counting storage wrapper is kept by wrapping it in `index.NewPageReader`; body reads stay on the underlying storage's `ReadPage`. + - vault-ui: bump vault-cli from v0.171.0; replace its page-index engine with `pkg/index` (keep key derivation, wire a prometheus `Metrics` adapter with its existing metric names, `DefaultStorePath("vault-ui")` keeps the same file path); read Open Questions from `ListEntries`; drop its own page snapshots; adapt its equivalence test to exclude only Content; add pprof. + +## Acceptance Criteria + +Test descriptions below are frozen so the verifier can grep `-ginkgo.v` output. All criteria are container-executable unless marked otherwise. + +- [ ] AC1: `make precommit` exits 0 in the vault-cli repo — evidence: exit code. +- [ ] AC2: `go doc ./pkg/index PageIndex` stdout names the embedded `storage.PageStorage` (which supplies `ListPages` and `ReadPage`) plus each of `ListEntries`, `Build`, `Refresh`, `RefreshFile`, `MarkDirty`, `MarkFileDirty`, `ForceReload`, `Revision`, `ChangedPagesSince`, `Rescan`; `go doc ./pkg/index Entry` stdout names fields `Page` and `OpenQuestions`; `go doc ./pkg/index` stdout names `NewPageIndex` and `NewPageIndexWithStore` — evidence: stdout match (1 + 10 + 2 + 2 names). +- [ ] AC3: `go test ./pkg/index/... -ginkgo.v` exits 0 and its output contains the passing spec `returns pages and entries with empty Content and populated frontmatter, name, path and modification date` (after `Build`, every page from `ListPages` and every `Entry.Page` from `ListEntries` has `Content == ""`, non-empty frontmatter for a fixture with frontmatter, and Name/FilePath/ModifiedDate equal to the file's) — evidence: exit code + stdout match. +- [ ] AC4: Same run contains passing spec `lists the same pages as vault-cli storage with only Content excluded`: over the awkward-files fixture (no frontmatter, bad YAML, non-.md file, subdirectory, symlink out of vault, broken symlink, in-vault symlink, non-ASCII name) `ListPages` is `reflect.DeepEqual` to `storage.NewPageStorage(nil).ListPages` with Content zeroed on the reference side — same set, order, frontmatter, path, mtime — evidence: stdout match. +- [ ] AC5: Same run contains passing spec `produces the same list-operation items as vault-cli storage`: `ops.NewListOperation(idx)` and `ops.NewListOperation(storage.NewPageStorage(nil))` return equal items for the same fixture — evidence: stdout match. +- [ ] AC6: Same run contains passing spec `reads the full body on ReadPage without caching it in the snapshot`: `ReadPage` returns the file's full content byte-for-byte, and a following `ListPages` still returns `Content == ""` and the same `*domain.Page` pointers as before the call — evidence: stdout match. +- [ ] AC7: Same run contains passing specs `projects open questions from ParseOpenQuestions`, `returns an empty non-nil slice when the page has no Open Questions section`, and `does not retain open-question text that shares memory with the file body` (each retained `Text` lies outside the address range of the body string the reader parsed; technique agent decides at impl time) — evidence: stdout match (3 descriptions). +- [ ] AC8: Same run contains passing spec `republishes a body-only edit and names it in ChangedPagesSince`: editing only the Open Questions section of one file, then `RefreshFile` (and separately `Refresh`), yields a new `*domain.Page` pointer and changed `OpenQuestions` for that entry, and `ChangedPagesSince(previousRevision)` returns that name with `ok == true` — evidence: stdout match. +- [ ] AC9: Same run lists Ginkgo `Describe` containers with these exact, case-sensitive texts — `Stat-diff rescan`, `Allocation`, `Per-file write marks`, `ChangedPagesSince`, `Cold read`, `Failed rebuild`, `Store hydration` — each with ≥1 passing spec ported from vault-ui's page-index tests (unchanged stat-diff publishes nothing and keeps snapshot identity; unchanged stat-diff allocates nothing beyond its folder listing; concurrent cold reads share one build; a failed rebuild keeps the previous snapshot). vault-ui's `TestUnchangedRescanAllocation` becomes a Ginkgo spec under `Allocation`, skipped when the race detector is enabled (allocation counts are meaningless under `-race`); `TestWriteMarkSkipsUnchangedFastPath` (a behavioural check, not an allocation count) becomes an unskipped spec under `Per-file write marks`; vault-ui's cold-read sharing spec (under `ListPages`) and failed-rebuild specs (under `failures`) are re-homed under `Cold read` and `Failed rebuild`; the `Allocation` container must still report ≥1 passing spec in a non-race run of `go test ./pkg/index/... -ginkgo.v` — evidence: stdout match per container. +- [ ] AC10: Same run contains passing spec `persists no Content field in the store record`: a reflection walk over the persisted record type finds no field named `Content` at any depth and finds a field named `OpenQuestions`. Passing spec `writes no body bytes to the store file`: after a cold build with a store over a fixture whose body contains the sentinel `BODY-SENTINEL-7f3a` (outside frontmatter and outside any Open Questions item), and the store closed, the bolt file's raw bytes do not contain that sentinel. Passing spec (under `Store hydration`) `restores open questions from the store without re-reading the file`: index A over a fixture with an Open Questions section publishes to the store and closes it; index B opens the same store over the unchanged files; B's `ListEntries` `OpenQuestions` equal A's and B's injected `Metrics` records zero file reads — evidence: stdout match (3 descriptions). +- [ ] AC11: Same run contains passing spec `discards a v1 store, deletes its entry buckets and warns store format mismatch with the path`: the test writes a bolt file using **literal** bytes (not package constants) — meta bucket `vault-ui-page-index-meta`, key `identity`, JSON `{"StoreFormat":1,"ParserVersion":"v0.171.0"}` (format AND parser both differ, the real vault-ui case), one entry bucket `vault-ui-page-index\x00\x00` — opens it, then asserts the entry bucket is gone and the captured warning contains `store format mismatch` and the file path. Passing spec `reports parser version mismatch as a distinct reason` (identity `{"StoreFormat":2,"ParserVersion":"v0.0.0-stale"}`) asserts a warning containing `parser version mismatch` and not `store format mismatch` — evidence: stdout match (2 descriptions). +- [ ] AC12: Same run contains passing spec `serves without a store when the bolt file is locked by another process`: with the file held open by a second bolt handle, construction returns within 10 s, logs a discard warning, and `ListPages` serves the fixture — evidence: stdout match. +- [ ] AC13: `go list -f '{{join .Imports "\n"}}' ./pkg/index | grep -E 'golang/glog|prometheus'` returns 0 lines (direct imports only; transitive imports via `bborbe/kv`/`boltkv` are allowed) — evidence: negative grep. +- [ ] AC14: `docs/page-index.md` exists and `grep -cE '^## (Metadata-only records|Entries and open questions|Lazy body reads|Store format version 2|Staleness bounds|Incremental updates|Key derivation)$' docs/page-index.md` returns 7; `go doc ./pkg/index` stdout contains `docs/page-index.md` — evidence: file content + stdout match. +- [ ] AC15: `awk '/^## /{sec=$0} /pkg\/index/{print sec}' CHANGELOG.md` prints `## Unreleased` at least once — evidence: file content. +- [ ] AC16: **Operator-executable:** `git diff --stat origin/master -- pkg/cli pkg/config pkg/domain pkg/ops pkg/storage` is empty on the PR branch — evidence: negative diff. +- [ ] AC17: **Operator-executable (after merge + release):** `go list -m github.com/bborbe/vault-cli@` resolves, and in a scratch module requiring that tag `go doc github.com/bborbe/vault-cli/pkg/index PageIndex` exits 0 — evidence: exit code + stdout. + +No new scenario: every behaviour is reachable with in-process tests over temp directories. + +## Verification + +### Container-executable (runs inside the YOLO container at prompt time) + +``` +make precommit +go test ./pkg/index/... -ginkgo.v +go doc ./pkg/index PageIndex +go doc ./pkg/index Entry +go doc ./pkg/index +go list -f '{{join .Imports "\n"}}' ./pkg/index | grep -E 'golang/glog|prometheus' # expect 0 lines +grep -cE '^## (Metadata-only records|Entries and open questions|Lazy body reads|Store format version 2|Staleness bounds|Incremental updates|Key derivation)$' docs/page-index.md # expect 7 +awk '/^## /{sec=$0} /pkg\/index/{print sec}' CHANGELOG.md # expect ## Unreleased +``` + +### Operator-executable (runs on the host after PR merge / release) + +``` +git fetch origin +git diff --stat origin/master -- pkg/cli pkg/config pkg/domain pkg/ops pkg/storage # expect empty +go list -m github.com/bborbe/vault-cli@ +# scratch module: go mod init x && go get github.com/bborbe/vault-cli@ +go doc github.com/bborbe/vault-cli/pkg/index PageIndex +``` + +## Desired Behavior + +1. **Metadata-only snapshots.** Each folder key holds one immutable snapshot of pages whose frontmatter (full map, every key), name, file path and modification date match vault-cli's folder listing exactly, and whose Content is always empty. The file body is read through vault-cli's own page read (one parse path, one symlink-out-of-vault guard) and dropped before the page is published. +2. **Entries carry Open Questions.** `ListEntries` serves the same snapshot as `ListPages` — same order, same page pointers — where each entry adds the page's Open Questions as `[]domain.OpenQuestion{Index, Text}` (vault-cli's own type, already the one vault-ui serves) projected from `ParseOpenQuestions` (Text = the item's question). The slice is never nil (empty when the page has no section), and every retained text is a copy that shares no memory with the body. +3. **Lazy body read.** `ReadPage` always reads the file through the underlying storage and returns the full page including Content. It never caches the body, never mutates or republishes the snapshot, and never moves a key's revision. +4. **Ported refresh contract.** Build, Refresh (stat-diff), RefreshFile, MarkDirty, MarkFileDirty, ForceReload, Revision, ChangedPagesSince and Rescan keep vault-ui's documented semantics unchanged: warm reads touch no disk; stat-diff re-reads only fingerprint-differing entries and an unchanged pass allocates and publishes nothing; per-file marks require an exact name; ForceReload is the only fingerprint-ignoring path; the change log is bounded and reports `ok == false` on folder marks, reloads, first publication and out-of-window revisions; cold reads share one build; a failed rebuild keeps the previous snapshot. Constants are ported as-is: rescan interval 50 s, rebuild timeout 2 min, change-log size 64. A body-only edit changes the fingerprint, so it is re-read, republished with a new page pointer and named by `ChangedPagesSince`. +5. **Library hygiene.** The package logs through `log/slog` by default with an injectable warning sink, and reports "file read by reason" and "changed-pages unbounded by reason" through an injected `Metrics` interface; a nil `Metrics` or nil store is a no-op. Constructors are frozen: `NewPageIndex(reader PageReader, lister DirectoryLister, currentDateTimeGetter libtime.CurrentDateTimeGetter, waiter libtime.WaiterDuration, metrics Metrics) PageIndex` and `NewPageIndexWithStore(reader PageReader, lister DirectoryLister, currentDateTimeGetter libtime.CurrentDateTimeGetter, waiter libtime.WaiterDuration, metrics Metrics, store Store) PageIndex` (vault-ui's names and leading parameters, plus `metrics`; `metrics` and `store` may be nil); the warning sink is a setter or option on the returned value — its exact form agent decides at impl time (local, defaults to slog); `NewBoltStore(ctx, path, identity, warnf, vlogf)` keeps vault-ui's signature. +6. **Optional metadata-only store (format 2).** When a store is given, each key is hydrated from it and then resolves exactly one stat-diff before it is served; publications write exactly the changed and vanished entries in one transaction, outside the index mutex. A persisted record holds filename, has-page flag, frontmatter, file metadata, Open Questions and fingerprint — no body. `DefaultStorePath(app)` is a pure path helper returning `//page-index.bolt`; `NewBoltStore` creates the file's directory with mode 0700 when absent. +7. **Store reconcile with distinct reasons.** On open, a store whose identity is missing/empty, damaged, written under another store format, or written by another vault-cli parser version is discarded: its entry buckets are deleted and one warning names the path and the reason — `store format mismatch (stored N, current 2)`, `parser version mismatch`, missing/empty, or damaged — each with distinct wording. Format is checked before parser version, so a file differing in both reports `store format mismatch`. Opening waits at most 5 s for the file lock; any open or reconcile failure yields a discarded store that serves nothing and never fails construction. +8. **Contract document.** `docs/page-index.md` in vault-cli states the metadata-only rule, the entry/Open Questions rule, lazy body reads, store format version 2 and the frozen store names, plus the ported staleness, incremental-update and key-derivation sections; the package documentation points to it; `CHANGELOG.md` `## Unreleased` names `pkg/index`. + +## Constraints + +### Decisions (frozen for this spec) + +- **New exported package `pkg/index`, port of vault-ui's engine** — one implementation for vault-ui and vault-api instead of two diverging ones. +- **Full frontmatter map retained, not a projection** — consumers (list operation, vault-ui) read arbitrary keys (`flag_set_by`, `jira`, `mode`); equivalence needs DeepEqual frontmatter. +- **Open Questions precomputed into the entry** — vault-ui reads them for every task row in one listing per vault; a lazy per-row body read would put a vault-wide file walk on the hottest path. +- **`ListPages` returns empty Content** — so `ops.NewListOperation(idx)` keeps working unchanged against the index. +- **`ReadPage` always goes to disk** — the body is the memory cost being removed; caching it would reintroduce it. +- **`pkg/index` imports no glog / prometheus directly** — vault-cli logs via slog; metrics go through an injected interface so vault-ui keeps its metric names in its own adapter (transitive imports via `bborbe/kv`/`boltkv` are unavoidable and allowed). +- **Store format version 2, persisted record without Content** — v1 records carry bodies and must not be read. +- **Frozen store names, byte-identical to vault-ui's v1 store:** meta bucket `vault-ui-page-index-meta`, identity key `identity`, identity JSON fields `StoreFormat` / `ParserVersion`, entry bucket prefix `vault-ui-page-index\x00` followed by `\x00` — so vault-ui's existing file is recognised and discarded as a format mismatch (entry buckets deleted) instead of reading as "missing" and leaving 760 MB of old buckets behind. +- **ParserVersion = vault-cli module version from build info** — the dependency entry when vault-cli is a dependency, the main module when vault-cli is itself main, `(unknown)` otherwise — so a vault-cli parser upgrade invalidates the cache. +- **Exported seams kept:** `Key`/`NewKey` (`filepath.Clean` on both parts), `PageReader`/`NewPageReader(storage.PageStorage)` (returns the metadata-only entry plus fingerprint), `DirectoryLister`/`NewDirectoryLister` (`.md` files only, no directories, directory order), `FileFingerprint` (size, modification time, status-change time via a symlink-following stat; per-OS for darwin and linux), `Store`, `NewBoltStore`, `DefaultStorePath`, `CurrentWriterIdentity`. + +### Must not change + +- No change to `pkg/cli`, `pkg/config`, `pkg/domain`, `pkg/ops`, `pkg/storage` (AC 16); the CLI's behaviour and every existing test are untouched. +- The behavioural contract in vault-ui's `docs/page-index.md` sections Staleness bounds, Incremental updates, Key derivation and Page index store holds in vault-cli, except where this spec replaces it (no bodies; format 2; distinct discard reasons). +- Published snapshots and the `*domain.Page` values in them are read-only for every consumer; the doc states this. +- Builds for darwin and linux with `CGO_ENABLED=0`. +- Version strings are owned by the releaser (`make check-versions`); do not hand-bump. Ginkgo/Gomega tests, counterfeiter mocks for new interfaces. +- New module dependencies are limited to `github.com/bborbe/boltkv`, `go.etcd.io/bbolt`, `github.com/bborbe/kv` (vault-ui's existing store stack). + +### Assumptions + +- `storage.PageStorage.ReadPage` and `storage.ParseOpenQuestions` (which only errors on context cancellation today) stay the single parse path; an Open Questions parse error fails that pass like any read error (previous snapshot kept). +- The store codec rebuilds pages with `domain.NewPage` from the raw frontmatter map — a JSON codec on `domain.Page` silently drops frontmatter (vault-ui's documented codec trap). + +## Failure Modes + +| Trigger | Expected behavior | Detection | Recovery | Reversibility / Concurrency | +|---|---|---|---|---| +| Bolt file locked by another process | Open gives up after 5 s; discarded store; index serves from full parse | One warning naming path + reason | Next start after the other process exits uses the store | Concurrency: second process never hangs startup | +| v1 or other-format store (e.g. vault-ui's existing file) | Discarded; entry buckets deleted; full parse | Warning `store format mismatch (stored 1, current 2)` + path | Next publication writes v2 records | Irreversible by design (cache only) | +| Parser version change | Discarded; full parse | Warning `parser version mismatch` + path | Automatic re-population | Cache only | +| Damaged bolt file / record decode failure | Discarded; full parse; never fails construction | Warning reason `damaged` + path | Automatic re-population | Cache only | +| Crash mid-write | Bolt transaction atomic; previous store content intact; next start hydrates then stat-diffs | none needed | One stat-diff repairs drift | Partial write impossible | +| Filesystem hang during list or read | Each storage call bounded by 2 min rebuild timeout; previous snapshot served | Error logged with key | Next mark, read or rescan retries | Readers unaffected | +| File deleted/renamed between list and read | Excluded from snapshot; fingerprint recorded | — | Next stat-diff | — | +| Unreadable file / symlink out of vault / bad YAML | Excluded (same as vault-cli listing); fingerprint stored | Warned once per fingerprint change | Fix file; next stat-diff picks it up | — | +| Clock skew / restored older mtime | Fingerprint compared by inequality, not "newer"; status-change time catches same-size-same-mtime rewrites | — | Re-read on next stat-diff | — | +| Concurrent readers during publish | Atomic snapshot swap; reader sees whole old or whole new | — | — | Concurrency: no torn reads | +| Consumer mutates a shared `*domain.Page` | Contract violation; corrupts every reader's view | Documented read-only rule | Consumer copies before mutating | Concurrency hazard documented | +| Open-question text retained as body substring | Prevented: text cloned; AC 7 guards | Test failure | — | — | +| Heap still above consumer target because full frontmatter maps dominate | Out of scope here | Consumer heap profile in the migration task | Follow-up spec for key projection/interning | — | +| Disk full on store write | Write fails; previous content intact; serving unaffected | Warning logged | Next publication retries its own entries | Cache only | + +## Security / Abuse Cases + +- Inputs are local vault paths and filenames. `RefreshFile`, `MarkFileDirty` and the per-file read path reject empty names, path separators, `..` and non-`.md` names (falling back to a folder stat-diff where vault-ui does); the symlink-out-of-vault guard comes from vault-cli's page read and is preserved. +- The bolt store is untrusted input: decoding never preallocates from stored counts or lengths, and any decode failure discards the store rather than serving its content. +- The store lives under the user cache directory — never inside a vault or the repository tree — with directory mode 0700. +- Nothing retries forever: rescan runs on a fixed 50 s interval, storage calls are bounded by 2 min, store open by 5 s. + +## Suggested Decomposition + +| # | Prompt focus | Covers DBs | Covers ACs | Depends on | +|---|---|---|---|---| +| 1 | Types + seams: Key/NewKey, FileFingerprint + per-OS stat, DirectoryLister, metadata-only PageReader producing Entry (full read via storage ReadPage, Open Questions projection with clone, Content dropped), Metrics interface + no-op, warn sink; reader tests | 1, 2, 5 | 7, 13 | — | +| 2 | Snapshot engine, in-memory only: port build/merge/marks/revision/change log/rescan; Build, Refresh, RefreshFile, ListPages, ListEntries, lazy ReadPage; port engine tests, equivalence test, list-operation test; `docs/page-index.md` index sections; CHANGELOG bullet | 1, 2, 3, 4, 8 | 1–6, 8, 9 (minus `Store hydration`), 14 (non-store headings), 15 | prompt 1 | +| 3 | Bolt store v2 + hydration wiring: Store, NewBoltStore, DefaultStorePath, identity reconcile with distinct reasons, codec without Content and with OpenQuestions, frozen names, 5 s open timeout, hydration + write-through in the engine; store tests incl. literal v1-file discard and locked file; go.mod deps; doc store section | 6, 7, 8 | 9 (`Store hydration`), 10 (incl. body-sentinel + open-questions round-trip), 11, 12, 14 (`Store format version 2`) | prompt 2 | + +Rationale: the reader defines the record shape every later layer stores; the engine is fully testable without persistence; the store is the only part adding dependencies and touches the engine's publish path last. AC 16–17 are operator checks after the PR. + +## Do-Nothing Option + +vault-ui keeps ~1.5 GB of heap and a ~760 MB cache file of bodies and keeps swapping; vault-api keeps loading every body on each HEAD change; two index implementations keep diverging. Not acceptable — the body is the dominant memory cost and only one consumer field depends on it. + +## Open Questions + +- Is full-frontmatter retention small enough for vault-ui's <300 MB heap target? Depends on the in-flight heap profile; if not, a follow-up spec adds key projection/interning. +- vault-ui jumps vault-cli v0.171.0 → this release (25 minors); unrelated API drift risk lands in that migration. +- vault-api retaining a metadata index between builds (instead of transient full pages) could raise steady-state pod memory; guarded by the consumer task's no-regression check (607Mi); fallback = construct the index per build. From 15d1adbd3522eb12a4d8664f32bfd24d45d240a9 Mon Sep 17 00:00:00 2001 From: Benjamin Borbe Date: Tue, 17 Nov 2026 18:11:19 +0100 Subject: [PATCH 2/3] feat(index): add in-memory metadata-only page index engine and contract doc --- CHANGELOG.md | 4 + docs/page-index.md | 114 ++++ go.mod | 2 +- mocks/index-page-index.go | 840 ++++++++++++++++++++++++++ pkg/index/equivalence_test.go | 348 +++++++++++ pkg/index/export_test.go | 110 ++++ pkg/index/index.go | 708 ++++++++++++++++++++++ pkg/index/index_allocation_test.go | 247 ++++++++ pkg/index/index_build.go | 732 ++++++++++++++++++++++ pkg/index/index_changedpages_test.go | 142 +++++ pkg/index/index_file.go | 175 ++++++ pkg/index/index_file_internal_test.go | 81 +++ pkg/index/index_file_test.go | 371 ++++++++++++ pkg/index/index_revision_test.go | 158 +++++ pkg/index/index_statdiff_test.go | 359 +++++++++++ pkg/index/index_test.go | 788 ++++++++++++++++++++++++ pkg/index/index_writemark_test.go | 249 ++++++++ pkg/index/reader_test.go | 2 +- 18 files changed, 5428 insertions(+), 2 deletions(-) create mode 100644 docs/page-index.md create mode 100644 mocks/index-page-index.go create mode 100644 pkg/index/equivalence_test.go create mode 100644 pkg/index/export_test.go create mode 100644 pkg/index/index.go create mode 100644 pkg/index/index_allocation_test.go create mode 100644 pkg/index/index_build.go create mode 100644 pkg/index/index_changedpages_test.go create mode 100644 pkg/index/index_file.go create mode 100644 pkg/index/index_file_internal_test.go create mode 100644 pkg/index/index_file_test.go create mode 100644 pkg/index/index_revision_test.go create mode 100644 pkg/index/index_statdiff_test.go create mode 100644 pkg/index/index_test.go create mode 100644 pkg/index/index_writemark_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6468e..33e8f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Please choose versions by [Semantic Versioning](http://semver.org/). * MINOR version when you add functionality in a backwards-compatible manner, and * PATCH version when you make backwards-compatible bug fixes. +## Unreleased + +- feat: add `pkg/index`, a shared metadata-only page index (frontmatter, name, path, mtime and pre-parsed Open Questions per page, never the body) with the ported stat-diff refresh contract, lazy on-disk body reads and an optional bolt cache (store format version 2) + ## v0.196.0 - fix: `goal set` now writes the flag as an unquoted YAML boolean and records `flag_set_by` / `flag_set_at` in the same write via `--by`, matching `task set` diff --git a/docs/page-index.md b/docs/page-index.md new file mode 100644 index 0000000..61d7624 --- /dev/null +++ b/docs/page-index.md @@ -0,0 +1,114 @@ +# Page index + +`pkg/index` keeps the metadata of each vault folder's pages in memory so a list +read can be answered without opening a vault file. It implements vault-cli's +`storage.PageStorage`, so it plugs in exactly where vault-cli reads a folder: +filtering, sorting and blocked-state logic stay in vault-cli's list operation +(`ops.NewListOperation(idx)` works unchanged). Single files are read through the +`PageReader` seam, which delegates to vault-cli's own +`storage.PageStorage.ReadPage`, so one parse path and one symlink-out-of-vault +guard serve both the index and vault-cli's own folder listing. + +One immutable snapshot is held per key — a vault root plus a vault-relative +pages dir. A published snapshot is never mutated; concurrent readers share the +same `*Entry` and `*domain.Page` pointers. **Published entries and pages are +read-only for every consumer**: a consumer that needs to change one copies it +first, because mutating a shared page corrupts every other reader's view. + +The package imports no glog and no prometheus. Warnings go to `log/slog` by +default (`SetWarnf` redirects them per index), and counters go through the +injected `Metrics` interface (`FileRead(reason)`, +`ChangedPagesUnbounded(reason)`); a nil `Metrics` records nothing. + +## Metadata-only records + +- Every indexed page keeps its full frontmatter map (every key), its name, its + file path and its modification date exactly as vault-cli's folder listing + produces them — and an **empty** `Content`. +- The body is read by the `PageReader` and dropped before the page is + published. No body is held in memory and none is written to the store. +- `ListPages` over a folder equals `storage.NewPageStorage(nil).ListPages` over + the same folder with only `Content` zeroed: same set, same order, same + frontmatter, path and modification time — including files without + frontmatter, invalid YAML, non-`.md` files, subdirectories, symlinks out of + the vault, broken symlinks, in-vault symlinks and non-ASCII names. + +## Entries and open questions + +- `ListEntries` serves the same snapshot as `ListPages` — same order, same page + pointers — where each `Entry` adds the page's `OpenQuestions` as + `[]domain.OpenQuestion{Index, Text}`, projected from + `storage.ParseOpenQuestions` (`Text` is the item's question, without any + recorded answer). +- `OpenQuestions` is never nil: a page without an `Open Questions` section has + an empty slice. +- Every retained `Text` is a copy (`strings.Clone`) that shares no memory with + the body, so a retained question never pins a body in memory. +- An open-question parse error fails that read like any read error. + +## Lazy body reads + +- `ReadPage(ctx, vaultPath, pagesDir, name)` takes the bare base name (without + `.md`) and always reads the file from disk through the `PageReader`, + returning the full page including `Content`. +- It never caches the body, never mutates or republishes the snapshot and never + moves the key's revision. A name that is empty, contains a path separator or + `..` is rejected without a read. + +## Staleness bounds + +- A warm read never touches disk. `ListPages` / `ListEntries` return the + published snapshot without any filesystem access. +- `Rescan` refreshes every known key once per `RescanInterval` (50 s) plus the + 1 s poll granularity plus one stat-diff — under 60 s. Each storage call is + bounded by the rebuild timeout (2 min), so the worst case is the rescan + interval plus that timeout. The bound holds for every change to a file's + presence, size, modification time or status-change time. +- The first read after a mark resolves it before serving: a per-file mark + re-reads exactly the marked files, a folder-level mark runs one stat-diff, and + every concurrent reader waits on that same work. Marking a key the index has + never seen is a no-op: its first read builds it anyway. +- Concurrent cold reads of one key share a single build. +- An event-driven (`RefreshFile`) or rescan-driven (`Refresh`) read never + blocks `ListPages`: a reader is served from the current snapshot. +- `ForceReload` is the only path that ignores fingerprints: the next read of + every key re-reads every file. +- A failed listing keeps serving the previous snapshot and logs the error with + the key; the mark stays pending, so the next read retries. A failed first + build returns an error. + +## Incremental updates + +- `RefreshFile(ctx, key, filename)` re-reads only that file and lists nothing. + The file's current on-disk state decides the outcome: present and readable + replaces or inserts its entry at its filename-ordered position, absent or + unreadable removes it. A filename that is empty, contains a path separator or + `..`, or does not end in `.md` is rejected and nothing is read. +- Every file read records a fingerprint — size, modification time and + status-change time — from a stat that follows symlinks, including files that + end up excluded. +- `Refresh` and the rescan are a **stat-diff**: list the folder, compare each + entry's fingerprint with the one recorded at its last read (inequality, not + "newer"), and re-read only what differs or is new. A body-only edit changes + the fingerprint, so it is re-read and republished with a new page pointer and + named by `ChangedPagesSince`. +- An unchanged stat-diff reads nothing, publishes nothing, keeps the snapshot + and the recorded fingerprint set in place, and allocates nothing beyond the + one folder listing it performs. An excluded file is warned about once per + fingerprint change. +- `MarkDirty(keys...)` marks a folder stale; the next read resolves it by one + stat-diff. `MarkFileDirty(key, id)` marks one file stale only when the id is + exact (`[[`/`]]` stripped, no separator or `..`, a byte-equal page name in the + snapshot, `.md` present); anything else widens to a folder-level mark. +- `Revision(key)` advances on every publication and mark. `ChangedPagesSince` + returns the `.md` filenames changed after a revision, with `ok == false` for + an unknown key, a revision ahead of the key, a revision outside the retained + change log (64 advances), a folder-level mark, a forced reload, or the key's + first publication. + +## Key derivation + +A key is `NewKey(vault path, folder)`, which applies `filepath.Clean` to both +parts so equivalent paths produce identical keys. A consumer derives the read +side and the event side from the same vault config entry; a mismatch silently +disables event refresh and leaves the folder on its rescan cadence. diff --git a/go.mod b/go.mod index 02f5975..0f81a04 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/bborbe/kafka v1.29.14 github.com/bborbe/log v1.7.9 github.com/bborbe/notification v0.7.13 + github.com/bborbe/run v1.11.12 github.com/bborbe/time v1.27.28 github.com/bborbe/validation v1.5.10 github.com/fsnotify/fsnotify v1.10.1 @@ -29,7 +30,6 @@ require ( github.com/bborbe/kv v1.21.30 // indirect github.com/bborbe/math v1.4.20 // indirect github.com/bborbe/parse v1.12.7 // indirect - github.com/bborbe/run v1.11.12 // indirect github.com/bborbe/sentry v1.10.8 // indirect github.com/bborbe/strimzi v1.8.25 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/mocks/index-page-index.go b/mocks/index-page-index.go new file mode 100644 index 0000000..ca0779b --- /dev/null +++ b/mocks/index-page-index.go @@ -0,0 +1,840 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/index" +) + +type IndexPageIndex struct { + BuildStub func(context.Context, []index.Key) error + buildMutex sync.RWMutex + buildArgsForCall []struct { + arg1 context.Context + arg2 []index.Key + } + buildReturns struct { + result1 error + } + buildReturnsOnCall map[int]struct { + result1 error + } + ChangedPagesSinceStub func(index.Key, uint64) ([]string, bool) + changedPagesSinceMutex sync.RWMutex + changedPagesSinceArgsForCall []struct { + arg1 index.Key + arg2 uint64 + } + changedPagesSinceReturns struct { + result1 []string + result2 bool + } + changedPagesSinceReturnsOnCall map[int]struct { + result1 []string + result2 bool + } + ForceReloadStub func() + forceReloadMutex sync.RWMutex + forceReloadArgsForCall []struct { + } + ListEntriesStub func(context.Context, string, string) ([]*index.Entry, error) + listEntriesMutex sync.RWMutex + listEntriesArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + } + listEntriesReturns struct { + result1 []*index.Entry + result2 error + } + listEntriesReturnsOnCall map[int]struct { + result1 []*index.Entry + result2 error + } + ListPagesStub func(context.Context, string, string) ([]*domain.Page, error) + listPagesMutex sync.RWMutex + listPagesArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + } + listPagesReturns struct { + result1 []*domain.Page + result2 error + } + listPagesReturnsOnCall map[int]struct { + result1 []*domain.Page + result2 error + } + MarkDirtyStub func(...index.Key) + markDirtyMutex sync.RWMutex + markDirtyArgsForCall []struct { + arg1 []index.Key + } + MarkFileDirtyStub func(index.Key, string) + markFileDirtyMutex sync.RWMutex + markFileDirtyArgsForCall []struct { + arg1 index.Key + arg2 string + } + ReadPageStub func(context.Context, string, string, string) (*domain.Page, error) + readPageMutex sync.RWMutex + readPageArgsForCall []struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + } + readPageReturns struct { + result1 *domain.Page + result2 error + } + readPageReturnsOnCall map[int]struct { + result1 *domain.Page + result2 error + } + RefreshStub func(context.Context, index.Key) error + refreshMutex sync.RWMutex + refreshArgsForCall []struct { + arg1 context.Context + arg2 index.Key + } + refreshReturns struct { + result1 error + } + refreshReturnsOnCall map[int]struct { + result1 error + } + RefreshFileStub func(context.Context, index.Key, string) error + refreshFileMutex sync.RWMutex + refreshFileArgsForCall []struct { + arg1 context.Context + arg2 index.Key + arg3 string + } + refreshFileReturns struct { + result1 error + } + refreshFileReturnsOnCall map[int]struct { + result1 error + } + RescanStub func(context.Context) error + rescanMutex sync.RWMutex + rescanArgsForCall []struct { + arg1 context.Context + } + rescanReturns struct { + result1 error + } + rescanReturnsOnCall map[int]struct { + result1 error + } + RevisionStub func(index.Key) uint64 + revisionMutex sync.RWMutex + revisionArgsForCall []struct { + arg1 index.Key + } + revisionReturns struct { + result1 uint64 + } + revisionReturnsOnCall map[int]struct { + result1 uint64 + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *IndexPageIndex) Build(arg1 context.Context, arg2 []index.Key) error { + var arg2Copy []index.Key + if arg2 != nil { + arg2Copy = make([]index.Key, len(arg2)) + copy(arg2Copy, arg2) + } + fake.buildMutex.Lock() + ret, specificReturn := fake.buildReturnsOnCall[len(fake.buildArgsForCall)] + fake.buildArgsForCall = append(fake.buildArgsForCall, struct { + arg1 context.Context + arg2 []index.Key + }{arg1, arg2Copy}) + stub := fake.BuildStub + fakeReturns := fake.buildReturns + fake.recordInvocation("Build", []interface{}{arg1, arg2Copy}) + fake.buildMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexPageIndex) BuildCallCount() int { + fake.buildMutex.RLock() + defer fake.buildMutex.RUnlock() + return len(fake.buildArgsForCall) +} + +func (fake *IndexPageIndex) BuildCalls(stub func(context.Context, []index.Key) error) { + fake.buildMutex.Lock() + defer fake.buildMutex.Unlock() + fake.BuildStub = stub +} + +func (fake *IndexPageIndex) BuildArgsForCall(i int) (context.Context, []index.Key) { + fake.buildMutex.RLock() + defer fake.buildMutex.RUnlock() + argsForCall := fake.buildArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *IndexPageIndex) BuildReturns(result1 error) { + fake.buildMutex.Lock() + defer fake.buildMutex.Unlock() + fake.BuildStub = nil + fake.buildReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) BuildReturnsOnCall(i int, result1 error) { + fake.buildMutex.Lock() + defer fake.buildMutex.Unlock() + fake.BuildStub = nil + if fake.buildReturnsOnCall == nil { + fake.buildReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.buildReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) ChangedPagesSince(arg1 index.Key, arg2 uint64) ([]string, bool) { + fake.changedPagesSinceMutex.Lock() + ret, specificReturn := fake.changedPagesSinceReturnsOnCall[len(fake.changedPagesSinceArgsForCall)] + fake.changedPagesSinceArgsForCall = append(fake.changedPagesSinceArgsForCall, struct { + arg1 index.Key + arg2 uint64 + }{arg1, arg2}) + stub := fake.ChangedPagesSinceStub + fakeReturns := fake.changedPagesSinceReturns + fake.recordInvocation("ChangedPagesSince", []interface{}{arg1, arg2}) + fake.changedPagesSinceMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexPageIndex) ChangedPagesSinceCallCount() int { + fake.changedPagesSinceMutex.RLock() + defer fake.changedPagesSinceMutex.RUnlock() + return len(fake.changedPagesSinceArgsForCall) +} + +func (fake *IndexPageIndex) ChangedPagesSinceCalls(stub func(index.Key, uint64) ([]string, bool)) { + fake.changedPagesSinceMutex.Lock() + defer fake.changedPagesSinceMutex.Unlock() + fake.ChangedPagesSinceStub = stub +} + +func (fake *IndexPageIndex) ChangedPagesSinceArgsForCall(i int) (index.Key, uint64) { + fake.changedPagesSinceMutex.RLock() + defer fake.changedPagesSinceMutex.RUnlock() + argsForCall := fake.changedPagesSinceArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *IndexPageIndex) ChangedPagesSinceReturns(result1 []string, result2 bool) { + fake.changedPagesSinceMutex.Lock() + defer fake.changedPagesSinceMutex.Unlock() + fake.ChangedPagesSinceStub = nil + fake.changedPagesSinceReturns = struct { + result1 []string + result2 bool + }{result1, result2} +} + +func (fake *IndexPageIndex) ChangedPagesSinceReturnsOnCall(i int, result1 []string, result2 bool) { + fake.changedPagesSinceMutex.Lock() + defer fake.changedPagesSinceMutex.Unlock() + fake.ChangedPagesSinceStub = nil + if fake.changedPagesSinceReturnsOnCall == nil { + fake.changedPagesSinceReturnsOnCall = make(map[int]struct { + result1 []string + result2 bool + }) + } + fake.changedPagesSinceReturnsOnCall[i] = struct { + result1 []string + result2 bool + }{result1, result2} +} + +func (fake *IndexPageIndex) ForceReload() { + fake.forceReloadMutex.Lock() + fake.forceReloadArgsForCall = append(fake.forceReloadArgsForCall, struct { + }{}) + stub := fake.ForceReloadStub + fake.recordInvocation("ForceReload", []interface{}{}) + fake.forceReloadMutex.Unlock() + if stub != nil { + fake.ForceReloadStub() + } +} + +func (fake *IndexPageIndex) ForceReloadCallCount() int { + fake.forceReloadMutex.RLock() + defer fake.forceReloadMutex.RUnlock() + return len(fake.forceReloadArgsForCall) +} + +func (fake *IndexPageIndex) ForceReloadCalls(stub func()) { + fake.forceReloadMutex.Lock() + defer fake.forceReloadMutex.Unlock() + fake.ForceReloadStub = stub +} + +func (fake *IndexPageIndex) ListEntries(arg1 context.Context, arg2 string, arg3 string) ([]*index.Entry, error) { + fake.listEntriesMutex.Lock() + ret, specificReturn := fake.listEntriesReturnsOnCall[len(fake.listEntriesArgsForCall)] + fake.listEntriesArgsForCall = append(fake.listEntriesArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.ListEntriesStub + fakeReturns := fake.listEntriesReturns + fake.recordInvocation("ListEntries", []interface{}{arg1, arg2, arg3}) + fake.listEntriesMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexPageIndex) ListEntriesCallCount() int { + fake.listEntriesMutex.RLock() + defer fake.listEntriesMutex.RUnlock() + return len(fake.listEntriesArgsForCall) +} + +func (fake *IndexPageIndex) ListEntriesCalls(stub func(context.Context, string, string) ([]*index.Entry, error)) { + fake.listEntriesMutex.Lock() + defer fake.listEntriesMutex.Unlock() + fake.ListEntriesStub = stub +} + +func (fake *IndexPageIndex) ListEntriesArgsForCall(i int) (context.Context, string, string) { + fake.listEntriesMutex.RLock() + defer fake.listEntriesMutex.RUnlock() + argsForCall := fake.listEntriesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *IndexPageIndex) ListEntriesReturns(result1 []*index.Entry, result2 error) { + fake.listEntriesMutex.Lock() + defer fake.listEntriesMutex.Unlock() + fake.ListEntriesStub = nil + fake.listEntriesReturns = struct { + result1 []*index.Entry + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) ListEntriesReturnsOnCall(i int, result1 []*index.Entry, result2 error) { + fake.listEntriesMutex.Lock() + defer fake.listEntriesMutex.Unlock() + fake.ListEntriesStub = nil + if fake.listEntriesReturnsOnCall == nil { + fake.listEntriesReturnsOnCall = make(map[int]struct { + result1 []*index.Entry + result2 error + }) + } + fake.listEntriesReturnsOnCall[i] = struct { + result1 []*index.Entry + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) ListPages(arg1 context.Context, arg2 string, arg3 string) ([]*domain.Page, error) { + fake.listPagesMutex.Lock() + ret, specificReturn := fake.listPagesReturnsOnCall[len(fake.listPagesArgsForCall)] + fake.listPagesArgsForCall = append(fake.listPagesArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.ListPagesStub + fakeReturns := fake.listPagesReturns + fake.recordInvocation("ListPages", []interface{}{arg1, arg2, arg3}) + fake.listPagesMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexPageIndex) ListPagesCallCount() int { + fake.listPagesMutex.RLock() + defer fake.listPagesMutex.RUnlock() + return len(fake.listPagesArgsForCall) +} + +func (fake *IndexPageIndex) ListPagesCalls(stub func(context.Context, string, string) ([]*domain.Page, error)) { + fake.listPagesMutex.Lock() + defer fake.listPagesMutex.Unlock() + fake.ListPagesStub = stub +} + +func (fake *IndexPageIndex) ListPagesArgsForCall(i int) (context.Context, string, string) { + fake.listPagesMutex.RLock() + defer fake.listPagesMutex.RUnlock() + argsForCall := fake.listPagesArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *IndexPageIndex) ListPagesReturns(result1 []*domain.Page, result2 error) { + fake.listPagesMutex.Lock() + defer fake.listPagesMutex.Unlock() + fake.ListPagesStub = nil + fake.listPagesReturns = struct { + result1 []*domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) ListPagesReturnsOnCall(i int, result1 []*domain.Page, result2 error) { + fake.listPagesMutex.Lock() + defer fake.listPagesMutex.Unlock() + fake.ListPagesStub = nil + if fake.listPagesReturnsOnCall == nil { + fake.listPagesReturnsOnCall = make(map[int]struct { + result1 []*domain.Page + result2 error + }) + } + fake.listPagesReturnsOnCall[i] = struct { + result1 []*domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) MarkDirty(arg1 ...index.Key) { + fake.markDirtyMutex.Lock() + fake.markDirtyArgsForCall = append(fake.markDirtyArgsForCall, struct { + arg1 []index.Key + }{arg1}) + stub := fake.MarkDirtyStub + fake.recordInvocation("MarkDirty", []interface{}{arg1}) + fake.markDirtyMutex.Unlock() + if stub != nil { + fake.MarkDirtyStub(arg1...) + } +} + +func (fake *IndexPageIndex) MarkDirtyCallCount() int { + fake.markDirtyMutex.RLock() + defer fake.markDirtyMutex.RUnlock() + return len(fake.markDirtyArgsForCall) +} + +func (fake *IndexPageIndex) MarkDirtyCalls(stub func(...index.Key)) { + fake.markDirtyMutex.Lock() + defer fake.markDirtyMutex.Unlock() + fake.MarkDirtyStub = stub +} + +func (fake *IndexPageIndex) MarkDirtyArgsForCall(i int) []index.Key { + fake.markDirtyMutex.RLock() + defer fake.markDirtyMutex.RUnlock() + argsForCall := fake.markDirtyArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *IndexPageIndex) MarkFileDirty(arg1 index.Key, arg2 string) { + fake.markFileDirtyMutex.Lock() + fake.markFileDirtyArgsForCall = append(fake.markFileDirtyArgsForCall, struct { + arg1 index.Key + arg2 string + }{arg1, arg2}) + stub := fake.MarkFileDirtyStub + fake.recordInvocation("MarkFileDirty", []interface{}{arg1, arg2}) + fake.markFileDirtyMutex.Unlock() + if stub != nil { + fake.MarkFileDirtyStub(arg1, arg2) + } +} + +func (fake *IndexPageIndex) MarkFileDirtyCallCount() int { + fake.markFileDirtyMutex.RLock() + defer fake.markFileDirtyMutex.RUnlock() + return len(fake.markFileDirtyArgsForCall) +} + +func (fake *IndexPageIndex) MarkFileDirtyCalls(stub func(index.Key, string)) { + fake.markFileDirtyMutex.Lock() + defer fake.markFileDirtyMutex.Unlock() + fake.MarkFileDirtyStub = stub +} + +func (fake *IndexPageIndex) MarkFileDirtyArgsForCall(i int) (index.Key, string) { + fake.markFileDirtyMutex.RLock() + defer fake.markFileDirtyMutex.RUnlock() + argsForCall := fake.markFileDirtyArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *IndexPageIndex) ReadPage(arg1 context.Context, arg2 string, arg3 string, arg4 string) (*domain.Page, error) { + fake.readPageMutex.Lock() + ret, specificReturn := fake.readPageReturnsOnCall[len(fake.readPageArgsForCall)] + fake.readPageArgsForCall = append(fake.readPageArgsForCall, struct { + arg1 context.Context + arg2 string + arg3 string + arg4 string + }{arg1, arg2, arg3, arg4}) + stub := fake.ReadPageStub + fakeReturns := fake.readPageReturns + fake.recordInvocation("ReadPage", []interface{}{arg1, arg2, arg3, arg4}) + fake.readPageMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *IndexPageIndex) ReadPageCallCount() int { + fake.readPageMutex.RLock() + defer fake.readPageMutex.RUnlock() + return len(fake.readPageArgsForCall) +} + +func (fake *IndexPageIndex) ReadPageCalls(stub func(context.Context, string, string, string) (*domain.Page, error)) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = stub +} + +func (fake *IndexPageIndex) ReadPageArgsForCall(i int) (context.Context, string, string, string) { + fake.readPageMutex.RLock() + defer fake.readPageMutex.RUnlock() + argsForCall := fake.readPageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *IndexPageIndex) ReadPageReturns(result1 *domain.Page, result2 error) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = nil + fake.readPageReturns = struct { + result1 *domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) ReadPageReturnsOnCall(i int, result1 *domain.Page, result2 error) { + fake.readPageMutex.Lock() + defer fake.readPageMutex.Unlock() + fake.ReadPageStub = nil + if fake.readPageReturnsOnCall == nil { + fake.readPageReturnsOnCall = make(map[int]struct { + result1 *domain.Page + result2 error + }) + } + fake.readPageReturnsOnCall[i] = struct { + result1 *domain.Page + result2 error + }{result1, result2} +} + +func (fake *IndexPageIndex) Refresh(arg1 context.Context, arg2 index.Key) error { + fake.refreshMutex.Lock() + ret, specificReturn := fake.refreshReturnsOnCall[len(fake.refreshArgsForCall)] + fake.refreshArgsForCall = append(fake.refreshArgsForCall, struct { + arg1 context.Context + arg2 index.Key + }{arg1, arg2}) + stub := fake.RefreshStub + fakeReturns := fake.refreshReturns + fake.recordInvocation("Refresh", []interface{}{arg1, arg2}) + fake.refreshMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexPageIndex) RefreshCallCount() int { + fake.refreshMutex.RLock() + defer fake.refreshMutex.RUnlock() + return len(fake.refreshArgsForCall) +} + +func (fake *IndexPageIndex) RefreshCalls(stub func(context.Context, index.Key) error) { + fake.refreshMutex.Lock() + defer fake.refreshMutex.Unlock() + fake.RefreshStub = stub +} + +func (fake *IndexPageIndex) RefreshArgsForCall(i int) (context.Context, index.Key) { + fake.refreshMutex.RLock() + defer fake.refreshMutex.RUnlock() + argsForCall := fake.refreshArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *IndexPageIndex) RefreshReturns(result1 error) { + fake.refreshMutex.Lock() + defer fake.refreshMutex.Unlock() + fake.RefreshStub = nil + fake.refreshReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) RefreshReturnsOnCall(i int, result1 error) { + fake.refreshMutex.Lock() + defer fake.refreshMutex.Unlock() + fake.RefreshStub = nil + if fake.refreshReturnsOnCall == nil { + fake.refreshReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.refreshReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) RefreshFile(arg1 context.Context, arg2 index.Key, arg3 string) error { + fake.refreshFileMutex.Lock() + ret, specificReturn := fake.refreshFileReturnsOnCall[len(fake.refreshFileArgsForCall)] + fake.refreshFileArgsForCall = append(fake.refreshFileArgsForCall, struct { + arg1 context.Context + arg2 index.Key + arg3 string + }{arg1, arg2, arg3}) + stub := fake.RefreshFileStub + fakeReturns := fake.refreshFileReturns + fake.recordInvocation("RefreshFile", []interface{}{arg1, arg2, arg3}) + fake.refreshFileMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexPageIndex) RefreshFileCallCount() int { + fake.refreshFileMutex.RLock() + defer fake.refreshFileMutex.RUnlock() + return len(fake.refreshFileArgsForCall) +} + +func (fake *IndexPageIndex) RefreshFileCalls(stub func(context.Context, index.Key, string) error) { + fake.refreshFileMutex.Lock() + defer fake.refreshFileMutex.Unlock() + fake.RefreshFileStub = stub +} + +func (fake *IndexPageIndex) RefreshFileArgsForCall(i int) (context.Context, index.Key, string) { + fake.refreshFileMutex.RLock() + defer fake.refreshFileMutex.RUnlock() + argsForCall := fake.refreshFileArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *IndexPageIndex) RefreshFileReturns(result1 error) { + fake.refreshFileMutex.Lock() + defer fake.refreshFileMutex.Unlock() + fake.RefreshFileStub = nil + fake.refreshFileReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) RefreshFileReturnsOnCall(i int, result1 error) { + fake.refreshFileMutex.Lock() + defer fake.refreshFileMutex.Unlock() + fake.RefreshFileStub = nil + if fake.refreshFileReturnsOnCall == nil { + fake.refreshFileReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.refreshFileReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) Rescan(arg1 context.Context) error { + fake.rescanMutex.Lock() + ret, specificReturn := fake.rescanReturnsOnCall[len(fake.rescanArgsForCall)] + fake.rescanArgsForCall = append(fake.rescanArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.RescanStub + fakeReturns := fake.rescanReturns + fake.recordInvocation("Rescan", []interface{}{arg1}) + fake.rescanMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexPageIndex) RescanCallCount() int { + fake.rescanMutex.RLock() + defer fake.rescanMutex.RUnlock() + return len(fake.rescanArgsForCall) +} + +func (fake *IndexPageIndex) RescanCalls(stub func(context.Context) error) { + fake.rescanMutex.Lock() + defer fake.rescanMutex.Unlock() + fake.RescanStub = stub +} + +func (fake *IndexPageIndex) RescanArgsForCall(i int) context.Context { + fake.rescanMutex.RLock() + defer fake.rescanMutex.RUnlock() + argsForCall := fake.rescanArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *IndexPageIndex) RescanReturns(result1 error) { + fake.rescanMutex.Lock() + defer fake.rescanMutex.Unlock() + fake.RescanStub = nil + fake.rescanReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) RescanReturnsOnCall(i int, result1 error) { + fake.rescanMutex.Lock() + defer fake.rescanMutex.Unlock() + fake.RescanStub = nil + if fake.rescanReturnsOnCall == nil { + fake.rescanReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.rescanReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexPageIndex) Revision(arg1 index.Key) uint64 { + fake.revisionMutex.Lock() + ret, specificReturn := fake.revisionReturnsOnCall[len(fake.revisionArgsForCall)] + fake.revisionArgsForCall = append(fake.revisionArgsForCall, struct { + arg1 index.Key + }{arg1}) + stub := fake.RevisionStub + fakeReturns := fake.revisionReturns + fake.recordInvocation("Revision", []interface{}{arg1}) + fake.revisionMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexPageIndex) RevisionCallCount() int { + fake.revisionMutex.RLock() + defer fake.revisionMutex.RUnlock() + return len(fake.revisionArgsForCall) +} + +func (fake *IndexPageIndex) RevisionCalls(stub func(index.Key) uint64) { + fake.revisionMutex.Lock() + defer fake.revisionMutex.Unlock() + fake.RevisionStub = stub +} + +func (fake *IndexPageIndex) RevisionArgsForCall(i int) index.Key { + fake.revisionMutex.RLock() + defer fake.revisionMutex.RUnlock() + argsForCall := fake.revisionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *IndexPageIndex) RevisionReturns(result1 uint64) { + fake.revisionMutex.Lock() + defer fake.revisionMutex.Unlock() + fake.RevisionStub = nil + fake.revisionReturns = struct { + result1 uint64 + }{result1} +} + +func (fake *IndexPageIndex) RevisionReturnsOnCall(i int, result1 uint64) { + fake.revisionMutex.Lock() + defer fake.revisionMutex.Unlock() + fake.RevisionStub = nil + if fake.revisionReturnsOnCall == nil { + fake.revisionReturnsOnCall = make(map[int]struct { + result1 uint64 + }) + } + fake.revisionReturnsOnCall[i] = struct { + result1 uint64 + }{result1} +} + +func (fake *IndexPageIndex) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *IndexPageIndex) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ index.PageIndex = new(IndexPageIndex) diff --git a/pkg/index/equivalence_test.go b/pkg/index/equivalence_test.go new file mode 100644 index 0000000..473aeca --- /dev/null +++ b/pkg/index/equivalence_test.go @@ -0,0 +1,348 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "os" + "path/filepath" + "reflect" + "strings" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/ops" + "github.com/bborbe/vault-cli/pkg/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// equivalenceFolder is the fixture's vault-relative pages dir. +const equivalenceFolder = "24 Tasks" + +// writeFixtureFile writes one file under the vault's pages folder. +func writeFixtureFile(vaultDir, name, content string) { + Expect(os.WriteFile( + filepath.Join(vaultDir, equivalenceFolder, name), []byte(content), 0600, + )).To(Succeed()) +} + +// buildEquivalenceVault creates the awkward-files fixture: a plain page, a bare +// wikilink, a file without frontmatter, a file with invalid YAML, a non-.md +// file, a subdirectory, a symlink out of the vault, a broken symlink, a symlink +// to a page inside the vault and a non-ASCII filename. +func buildEquivalenceVault() string { + vaultDir := GinkgoT().TempDir() + folderDir := filepath.Join(vaultDir, equivalenceFolder) + Expect(os.MkdirAll(folderDir, 0750)).To(Succeed()) + Expect(os.MkdirAll(filepath.Join(folderDir, "subdir"), 0750)).To(Succeed()) + + writeFixtureFile(vaultDir, "Plain.md", "---\ntitle: Plain\nstatus: next\n---\n# Plain\n") + writeFixtureFile( + vaultDir, "Wikilink.md", "---\ntitle: Wikilink\nrelated: [[Other]]\n---\n# Wikilink\n", + ) + writeFixtureFile(vaultDir, "NoFrontmatter.md", "# No frontmatter\n") + writeFixtureFile(vaultDir, "BadYaml.md", "---\ntitle: [unclosed\n---\n# Bad\n") + writeFixtureFile(vaultDir, "notes.txt", "not a page\n") + writeFixtureFile(vaultDir, "Ünïcode.md", "---\ntitle: Ünïcode\n---\n# Ünïcode\n") + + // The outside target lives in its own temp dir, never under the vault. + outsideDir := GinkgoT().TempDir() + outsideFile := filepath.Join(outsideDir, "secret.md") + Expect(os.WriteFile(outsideFile, []byte("---\ntitle: Secret\n---\n# Secret\n"), 0600)). + To(Succeed()) + Expect(os.Symlink(outsideFile, filepath.Join(folderDir, "Outside.md"))).To(Succeed()) + + // A broken symlink counts as outside. + Expect(os.Symlink( + filepath.Join(folderDir, "does-not-exist.md"), filepath.Join(folderDir, "Broken.md"), + )).To(Succeed()) + + // A symlink to a page inside the vault stays in the listing. + Expect(os.Symlink("Plain.md", filepath.Join(folderDir, "Inside.md"))).To(Succeed()) + + return vaultDir +} + +// indexedNames returns the pages' file names in listing order. +func indexedNames(pages []*domain.Page) []string { + names := make([]string, 0, len(pages)) + for _, page := range pages { + names = append(names, page.Name) + } + return names +} + +// withoutContent returns copies of pages with Content zeroed, the reference +// shape the metadata-only index serves. A nil input stays nil. +func withoutContent(pages []*domain.Page) []*domain.Page { + if pages == nil { + return nil + } + result := make([]*domain.Page, 0, len(pages)) + for _, page := range pages { + copied := *page + copied.Content = "" + result = append(result, &copied) + } + return result +} + +// referenceListing is vault-cli's own folder listing with Content zeroed. +func referenceListing(ctx context.Context, vaultDir string) []*domain.Page { + pages, err := storage.NewPageStorage(nil).ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + return withoutContent(pages) +} + +// newEquivalenceIndex builds the index over the production reader and lister. +func newEquivalenceIndex() index.PageIndex { + return index.NewPageIndex( + index.NewPageReader(storage.NewPageStorage(nil)), + index.NewDirectoryLister(), + libtime.NewCurrentDateTime(), + libtime.NewWaiterDuration(), + index.TestMetrics, + ) +} + +var _ = Describe("Snapshot equivalence", func() { + It("lists the same pages as vault-cli storage with only Content excluded", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + + want := referenceListing(ctx, vaultDir) + // Positive control: a fixture that silently excluded everything would + // still deep-equal an equally empty snapshot. + Expect(indexedNames(want)).To(Equal([]string{"Inside", "Plain", "Wikilink", "Ünïcode"})) + + got, err := newEquivalenceIndex().ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + }) + + It("matches vault-cli on an existing folder with no page files", func() { + ctx := context.Background() + vaultDir := GinkgoT().TempDir() + Expect(os.MkdirAll( + filepath.Join(vaultDir, equivalenceFolder), 0750, + )).To(Succeed()) + + want, err := storage.NewPageStorage(nil).ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(want).NotTo(BeNil()) + Expect(want).To(BeEmpty()) + + got, err := newEquivalenceIndex().ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + }) + + It("AC3 stays equal to vault-cli through create, modify, delete and rename", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + idx := newEquivalenceIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + + // Every step is checked against vault-cli's own listing of the same dir. + assertEquivalent := func() { + want := referenceListing(ctx, vaultDir) + got, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(reflect.DeepEqual(got, want)).To( + BeTrue(), + "index %v != vault-cli %v", + indexedNames(got), + indexedNames(want), + ) + } + + // The cold build is the starting point. + assertEquivalent() + + // Create, through a per-file update. + writeFixtureFile(vaultDir, "Created.md", "---\ntitle: Created\n---\n# Created\n") + Expect(idx.RefreshFile(ctx, key, "Created.md")).To(Succeed()) + assertEquivalent() + + // Modify Plain.md, whose symlink Inside.md shares the same bytes. + writeFixtureFile(vaultDir, "Plain.md", "---\ntitle: Plain\nstatus: done\n---\n# Plain v2\n") + Expect(idx.RefreshFile(ctx, key, "Plain.md")).To(Succeed()) + Expect(idx.RefreshFile(ctx, key, "Inside.md")).To(Succeed()) + assertEquivalent() + + // Delete, through a per-file update. + Expect(os.Remove(filepath.Join(vaultDir, equivalenceFolder, "NoFrontmatter.md"))). + To(Succeed()) + Expect(idx.RefreshFile(ctx, key, "NoFrontmatter.md")).To(Succeed()) + assertEquivalent() + + // Rename is a delete plus a create. + Expect(os.Rename( + filepath.Join(vaultDir, equivalenceFolder, "Wikilink.md"), + filepath.Join(vaultDir, equivalenceFolder, "Renamed.md"), + )).To(Succeed()) + Expect(idx.RefreshFile(ctx, key, "Wikilink.md")).To(Succeed()) + Expect(idx.RefreshFile(ctx, key, "Renamed.md")).To(Succeed()) + assertEquivalent() + + // The in-vault symlink's target is rewritten with no event delivered; a + // stat-diff must pick up both the target and the symlink. + writeFixtureFile( + vaultDir, "Plain.md", "---\ntitle: Plain\nstatus: blocked\n---\n# Plain v3 longer\n", + ) + Expect(idx.Refresh(ctx, key)).To(BeNil()) + assertEquivalent() + + // And the stat-diff of an unchanged folder changes nothing. + Expect(idx.Refresh(ctx, key)).To(BeNil()) + assertEquivalent() + }) + + It("matches vault-cli on a missing folder", func() { + ctx := context.Background() + vaultDir := GinkgoT().TempDir() + + want, err := storage.NewPageStorage(nil).ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(want).To(BeNil()) + + got, err := newEquivalenceIndex().ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + }) + + It("returns pages and entries with empty Content and populated frontmatter, name, path and modification date", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + idx := newEquivalenceIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + Expect(idx.Build(ctx, []index.Key{key})).To(Succeed()) + + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + entries, err := idx.ListEntries(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(len(pages))) + Expect(pages).NotTo(BeEmpty()) + + for i, page := range pages { + Expect(entries[i].Page).To(BeIdenticalTo(page)) + Expect(string(page.Content)).To(BeEmpty()) + path := filepath.Join(vaultDir, equivalenceFolder, page.Name+".md") + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Name).To(Equal(strings.TrimSuffix(filepath.Base(path), ".md"))) + Expect(page.FileMetadata.FilePath).To(Equal(path)) + Expect(page.FileMetadata.ModifiedDate).NotTo(BeNil()) + Expect(page.FileMetadata.ModifiedDate.Equal(info.ModTime())).To(BeTrue()) + if page.Name == "Plain" { + Expect(page.RawMap()).To(HaveKeyWithValue("title", "Plain")) + } + Expect(entries[i].OpenQuestions).NotTo(BeNil()) + } + }) + + It("produces the same list-operation items as vault-cli storage", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + + want, err := ops.NewListOperation(storage.NewPageStorage(nil)). + Execute(ctx, vaultDir, "vault", equivalenceFolder, nil, true, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(want).NotTo(BeEmpty()) + got, err := ops.NewListOperation(newEquivalenceIndex()). + Execute(ctx, vaultDir, "vault", equivalenceFolder, nil, true, "", "") + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(want)) + }) + + It("reads the full body on ReadPage without caching it in the snapshot", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + idx := newEquivalenceIndex() + + before, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + raw, err := os.ReadFile(filepath.Join(vaultDir, equivalenceFolder, "Plain.md")) + Expect(err).NotTo(HaveOccurred()) + + page, err := idx.ReadPage(ctx, vaultDir, equivalenceFolder, "Plain") + Expect(err).NotTo(HaveOccurred()) + Expect(string(page.Content)).To(Equal(string(raw))) + + after, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(HaveLen(len(before))) + for i := range after { + Expect(after[i]).To(BeIdenticalTo(before[i])) + Expect(string(after[i].Content)).To(BeEmpty()) + } + }) + + It("refuses a ReadPage name that must never become a file path", func() { + ctx := context.Background() + idx := newEquivalenceIndex() + for _, name := range []string{"", "sub/Page", ".."} { + _, err := idx.ReadPage(ctx, GinkgoT().TempDir(), equivalenceFolder, name) + Expect(err).To(HaveOccurred()) + } + }) + + It("republishes a body-only edit and names it in ChangedPagesSince", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + key := index.NewKey(vaultDir, equivalenceFolder) + + for _, viaRefreshFile := range []bool{true, false} { + writeFixtureFile(vaultDir, "Questions.md", "---\ntitle: Questions\n---\n## Open Questions\n\n- first\n") + idx := newEquivalenceIndex() + Expect(idx.Build(ctx, []index.Key{key})).To(Succeed()) + before := findEntry(ctx, idx, vaultDir, "Questions") + Expect(before.OpenQuestions).To(Equal([]domain.OpenQuestion{{Index: 1, Text: "first"}})) + revision := idx.Revision(key) + + settle() + body := "---\ntitle: Questions\n---\n## Open Questions\n\n- first\n- second edited\n" + if !viaRefreshFile { + body += "\n" + } + writeFixtureFile(vaultDir, "Questions.md", body) + if viaRefreshFile { + Expect(idx.RefreshFile(ctx, key, "Questions.md")).To(Succeed()) + } else { + Expect(idx.Refresh(ctx, key)).To(Succeed()) + } + + after := findEntry(ctx, idx, vaultDir, "Questions") + Expect(after.Page).NotTo(BeIdenticalTo(before.Page)) + Expect(after.OpenQuestions).To(Equal([]domain.OpenQuestion{ + {Index: 1, Text: "first"}, + {Index: 2, Text: "second edited"}, + })) + names, ok := idx.ChangedPagesSince(key, revision) + Expect(ok).To(BeTrue()) + Expect(names).To(ContainElement("Questions.md")) + } + }) +}) + +// findEntry returns the listed entry whose page has the given name. +func findEntry(ctx context.Context, idx index.PageIndex, vaultDir, name string) *index.Entry { + entries, err := idx.ListEntries(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + for _, entry := range entries { + if entry.Page.Name == name { + return entry + } + } + Fail("no entry named " + name) + return nil +} diff --git a/pkg/index/export_test.go b/pkg/index/export_test.go new file mode 100644 index 0000000..694cea6 --- /dev/null +++ b/pkg/index/export_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "fmt" + "sync" + + libtime "github.com/bborbe/time" +) + +// CountingMetrics records every counter call, so a test can assert deltas. +type CountingMetrics struct { + mu sync.Mutex + reads map[string]int + unbounded map[string]int +} + +// NewCountingMetrics returns an empty CountingMetrics. +func NewCountingMetrics() *CountingMetrics { + return &CountingMetrics{reads: map[string]int{}, unbounded: map[string]int{}} +} + +// FileRead counts one read. +func (m *CountingMetrics) FileRead(reason string) { + m.mu.Lock() + defer m.mu.Unlock() + m.reads[reason]++ +} + +// ChangedPagesUnbounded counts one refused bound. +func (m *CountingMetrics) ChangedPagesUnbounded(reason string) { + m.mu.Lock() + defer m.mu.Unlock() + m.unbounded[reason]++ +} + +// Reads returns the read count for reason. +func (m *CountingMetrics) Reads(reason string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.reads[reason] +} + +// TotalReads returns the read count over every reason. +func (m *CountingMetrics) TotalReads() int { + m.mu.Lock() + defer m.mu.Unlock() + total := 0 + for _, count := range m.reads { + total += count + } + return total +} + +// TestMetrics is shared by every test index, mirroring vault-ui's +// package-global counter: callers assert deltas, never absolutes. +var TestMetrics = NewCountingMetrics() + +// FilesReadTotal returns the shared read counter for reason. +func FilesReadTotal(reason string) float64 { + return float64(TestMetrics.Reads(reason)) +} + +// NewPageIndexWithWarnf builds a page index whose warnings go to warnf. +func NewPageIndexWithWarnf( + reader PageReader, + lister DirectoryLister, + currentDateTimeGetter libtime.CurrentDateTimeGetter, + waiter libtime.WaiterDuration, + warnf func(format string, args ...any), +) PageIndex { + idx := NewPageIndex(reader, lister, currentDateTimeGetter, waiter, TestMetrics) + SetWarnf(idx, warnf) + return idx +} + +// FingerprintSetIdentity returns an identity token for the key's recorded +// fingerprint set. An unknown key reports "". +func FingerprintSetIdentity(idx PageIndex, key Key) string { + pi, ok := idx.(*pageIndex) + if !ok { + return "" + } + pi.mu.Lock() + defer pi.mu.Unlock() + e, ok := pi.entries[key] + if !ok { + return "" + } + return fmt.Sprintf("%p", e.fingerprints) +} + +// SnapshotIdentity returns an identity token for the key's published snapshot. +// An unknown or unpublished key reports "". +func SnapshotIdentity(idx PageIndex, key Key) string { + pi, ok := idx.(*pageIndex) + if !ok { + return "" + } + pi.mu.Lock() + defer pi.mu.Unlock() + e, ok := pi.entries[key] + if !ok || !e.hasSnapshot { + return "" + } + return fmt.Sprintf("%p", e.snapshot.entries) +} diff --git a/pkg/index/index.go b/pkg/index/index.go new file mode 100644 index 0000000..ac7df8b --- /dev/null +++ b/pkg/index/index.go @@ -0,0 +1,708 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/bborbe/errors" + "github.com/bborbe/run" + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" +) + +// RescanInterval is the time between stat-diff rescans of every known key. It +// is deliberately below the 60 s ceiling so that interval + rescanPollInterval +// + one read stays inside it. +const RescanInterval = 50 * time.Second + +// rescanPollInterval is the granularity at which the rescan loop checks the +// clock. It is not configurable. +const rescanPollInterval = time.Second + +// rebuildTimeout bounds a single storage call so a hung filesystem cannot hold +// a key's rebuild — and the callers waiting on it — forever. +const rebuildTimeout = 2 * time.Minute + +//counterfeiter:generate -o ../../mocks/index-page-index.go --fake-name IndexPageIndex . PageIndex + +// PageIndex serves vault page lists from immutable in-memory snapshots that +// hold page metadata only. ListPages returns pages with empty Content; ReadPage +// always reads the full page from disk and never caches it. +type PageIndex interface { + storage.PageStorage + // ListEntries returns the key's snapshot as entries: the same pages, in the + // same order and with the same pointers as ListPages, each with its Open + // Questions. + ListEntries(ctx context.Context, vaultPath string, pagesDir string) ([]*Entry, error) + // Build warms the given keys, discarding the pages. + Build(ctx context.Context, keys []Key) error + // Refresh re-reads the key's files whose size, modification time or + // status-change time changed since they were last read, and blocks until a + // listing that started after this call has been applied. It reads nothing + // when nothing changed and publishes a new snapshot only when something + // did. A key with no snapshot yet is built in full. + Refresh(ctx context.Context, key Key) error + // RefreshFile re-reads one file of the key's folder and publishes a new + // snapshot with the result spliced in. It blocks until a snapshot + // containing a read that started after this call has been published. The + // file's current on-disk state decides the outcome, not the caller's reason + // for calling: present and readable replaces or inserts the page at its + // filename-ordered position, absent or unreadable removes it. The filename + // is a single plain base name including ".md"; a filename that is empty, + // contains a path separator, contains "..", or does not end in ".md" is + // rejected with an error and nothing is read. + RefreshFile(ctx context.Context, key Key, filename string) error + // MarkDirty marks the named keys stale at folder level, so the next read of + // each compares the folder's fingerprints and re-reads only what changed. + MarkDirty(keys ...Key) + // MarkFileDirty marks one file of the key stale so the next read re-reads it + // before serving. The name is the item id; it is applied only when the + // exactness rule holds, otherwise the whole key is marked folder-level. + MarkFileDirty(key Key, name string) + // ForceReload marks every known key so the next read of each re-reads every + // file, whatever the fingerprints say. It is the only path that ignores + // fingerprints. + ForceReload() + // Revision reports a per-key value that advances whenever the key's page + // snapshot is published or marked stale. A derived snapshot reads it to + // detect that its page input changed. An unknown key reports 0. + Revision(key Key) uint64 + // ChangedPagesSince returns the names of the pages whose parsed form changed + // in key's publications and marks after revision. Each name is a base + // filename WITH its ".md" suffix, matching the page index's write-mark and + // snapshot naming. + // + // ok is false when the key cannot bound the set. That is any of: an unknown + // key; a revision ahead of the key's current one, which the key has never + // reached; a revision so far behind that the retained change window no longer + // covers it; a folder-level mark; a forced reload; or the key's first + // publication, which publishes the whole snapshot whatever the build's kind. + // A revision the key has advanced past while the advance is still retained is + // the normal bounded case. + ChangedPagesSince(key Key, revision uint64) (names []string, ok bool) + // Rescan refreshes every known key once per RescanInterval until ctx is done. + Rescan(ctx context.Context) error +} + +// buildKind is the work one build does for its key. +type buildKind int + +const ( + // buildCold lists the folder and reads every file. It is used when the key + // has no snapshot yet. + buildCold buildKind = iota + // buildStatDiff lists the folder, compares each entry's fingerprint with the + // one recorded at its last read and re-reads only what changed. + buildStatDiff + // buildFileReads re-reads exactly the write-marked files and lists nothing. + buildFileReads + // buildReload lists the folder and re-reads every file, ignoring + // fingerprints. + buildReload +) + +// storeDelta is one publication's change: the entries it re-read and the names +// it dropped. +// StoredEntry is one indexed file's record: the metadata-only entry, or nil when +// the file was excluded, plus the pre-read fingerprint. It never holds a body. +type StoredEntry struct { + Filename string + Entry *Entry + Fingerprint FileFingerprint +} + +type storeDelta struct { + puts []StoredEntry + deletes []string +} + +// build is one resolution of one key. It is created under the index mutex and +// its result fields are written under that mutex before done is closed. +type build struct { + kind buildKind + // reason is the metric label the creator wants for the reads this build + // makes; a cold build and a reload override it. + reason string + // startSeq is the key's requestSeq when this build last started. A mark with + // a higher sequence is not covered by this build. + startSeq uint64 + started chan struct{} + done chan struct{} + snap published + err error + // delta is the change this build's publication produced, or nil when there + // is nothing to record. It is written under the mutex before done is closed + // and read only by the goroutine that ran the build. + delta *storeDelta +} + +func newBuild(kind buildKind, reason string, startSeq uint64) *build { + return &build{ + kind: kind, + reason: reason, + startSeq: startSeq, + started: make(chan struct{}), + done: make(chan struct{}), + } +} + +// covers reports whether this build, having started at startSeq, resolves work +// recorded at needSeq of the given kind. +func (b *build) covers(kind buildKind, needSeq uint64) bool { + if b.startSeq < needSeq { + return false + } + switch b.kind { + case buildCold, buildReload: + return true + case buildFileReads: + return kind == buildFileReads + case buildStatDiff: + return kind == buildStatDiff + } + return false +} + +// entry is the per-key state. +type entry struct { + snapshot published + hasSnapshot bool + fingerprints map[string]FileFingerprint + + // fileReadSeq holds, per base filename, the read sequence of the read whose + // result is currently applied — the page present or removed. A name whose + // page was removed keeps its entry as a tombstone; it is never deleted. + fileReadSeq map[string]uint64 + + // requestSeq is bumped by every refresh, mark and forced reload. + requestSeq uint64 + // dirtySeq is requestSeq at the most recent folder-level mark; it is + // resolved once a listing that started after it succeeded. + dirtySeq uint64 + // dirtyResolvedSeq is requestSeq at the start of the most recent successful + // listing — a stat-diff from any caller, a cold build or a forced reload. + dirtyResolvedSeq uint64 + // reloadSeq is requestSeq at the most recent forced reload; it is resolved + // only by a successful full re-read. + reloadSeq uint64 + // reloadResolvedSeq is requestSeq at the start of the most recent successful + // full re-read. + reloadResolvedSeq uint64 + // writeMarked holds, per base filename, the requestSeq at which the file was + // marked for a per-file re-read. An entry is taken out when its read starts. + writeMarked map[string]uint64 + + // revision advances whenever this key's page input can have changed: a mark + // at folder or file level, a forced reload, or a published snapshot. A + // derived snapshot polls it to detect that its page input changed. + revision uint64 + // changeLog records the last maxChangeLog revision advances with the pages + // each can be attributed to, so a derived snapshot can bound which of its + // rows moved instead of rebuilding all of them. Older entries are dropped: + // a revision below the retained window can no longer be placed, which + // ChangedPagesSince reports as unknown rather than guessing. + changeLog []pageChange + + inflight *build + followUp *build +} + +// maxChangeLog bounds one key's change log. It is far above the number of +// advances a derived snapshot can fall behind by in one read. +const maxChangeLog = 64 + +// pageChange is one revision advance and the pages it changed. bounded is false +// when the advance cannot be attributed to a bounded set of single files — a +// folder-level mark or a forced reload — so no reader may treat it as a row +// patch. +type pageChange struct { + revision uint64 + names []string + bounded bool + // reason is the unbounded cause this advance carries, empty for a bounded + // advance. ChangedPagesSince reports the earliest unbounded advance's reason + // when it refuses to bound a set. + reason string +} + +// recordChangeLocked appends one revision advance to the key's change log. The +// caller must hold the mutex and must already have advanced revision, so the +// entry's revision is the value the advance produced. +func (e *entry) recordChangeLocked(names []string, bounded bool, reason string) { + e.changeLog = append(e.changeLog, pageChange{ + revision: e.revision, + names: names, + bounded: bounded, + reason: reason, + }) + if len(e.changeLog) > maxChangeLog { + trimmed := make([]pageChange, maxChangeLog) + copy(trimmed, e.changeLog[len(e.changeLog)-maxChangeLog:]) + e.changeLog = trimmed + } +} + +type pageIndex struct { + mu sync.Mutex + entries map[Key]*entry + // readSeq is bumped under mu at the start of every single-file read. It + // orders those reads against each other and nothing else. + readSeq uint64 + reader PageReader + lister DirectoryLister + currentDateTimeGetter libtime.CurrentDateTimeGetter + waiter libtime.WaiterDuration + metrics Metrics + // warnf emits the per-file exclusion warnings. It is a field so a consumer + // can redirect them per index instance. + warnf func(format string, args ...any) +} + +// NewPageIndex creates an empty page index over the given reader and lister. +// A nil metrics records nothing. +func NewPageIndex( + reader PageReader, + lister DirectoryLister, + currentDateTimeGetter libtime.CurrentDateTimeGetter, + waiter libtime.WaiterDuration, + metrics Metrics, +) PageIndex { + if metrics == nil { + metrics = NewNoopMetrics() + } + return &pageIndex{ + entries: map[Key]*entry{}, + reader: reader, + lister: lister, + currentDateTimeGetter: currentDateTimeGetter, + waiter: waiter, + metrics: metrics, + warnf: slogWarnf, + } +} + +// SetWarnf redirects the index's warnings to warnf. It must be called before +// the index reads anything; a nil warnf or an index not built by this package +// is ignored. +func SetWarnf(idx PageIndex, warnf func(format string, args ...any)) { + pi, ok := idx.(*pageIndex) + if !ok || warnf == nil { + return + } + pi.warnf = warnf +} + +// slogWarnf is the default warning sink. +func slogWarnf(format string, args ...any) { + slog.Warn(fmt.Sprintf(format, args...)) +} + +// ListPages returns the pages of the key's folder from the in-memory snapshot. +// It resolves any pending write mark before serving, so a reader sees the +// content its own write produced. A key whose only pending work is an event or +// a rescan read is served from the current snapshot without waiting. +func (p *pageIndex) ListPages( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]*domain.Page, error) { + snap, err := p.list(ctx, vaultPath, pagesDir) + if err != nil { + return nil, err + } + return snap.pages, nil +} + +// ListEntries returns the key's entries from the in-memory snapshot, under the +// same rules as ListPages. +func (p *pageIndex) ListEntries( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]*Entry, error) { + snap, err := p.list(ctx, vaultPath, pagesDir) + if err != nil { + return nil, err + } + return snap.entries, nil +} + +// list resolves the key's pending work and returns its published snapshot. +func (p *pageIndex) list(ctx context.Context, vaultPath string, pagesDir string) (published, error) { + key := NewKey(vaultPath, pagesDir) + snap, buildErr, cancelErr := p.ensure(ctx, key, false) + if cancelErr != nil { + return published{}, cancelErr + } + if buildErr != nil { + if snapshot, ok := p.snapshot(key); ok { + return snapshot, nil + } + return published{}, errors.Wrapf( + ctx, + buildErr, + "list pages %s/%s", + key.VaultPath, + key.PagesDir, + ) + } + return snap, nil +} + +// ReadPage returns one full page, body included, by its bare base name +// without the ".md" suffix — the value ListPages reports as the page's name. +// +// It always reads the file through the PageReader and fails when the file is +// missing or unparseable. It never caches the body, never touches the +// published snapshot and never moves the key's revision. +func (p *pageIndex) ReadPage( + ctx context.Context, + vaultPath string, + pagesDir string, + name string, +) (*domain.Page, error) { + if !validFileMarkName(name) { + return nil, errors.Errorf(ctx, "invalid page name %q", name) + } + page, err := p.reader.ReadPage(ctx, vaultPath, pagesDir, name) + if err != nil { + return nil, errors.Wrapf(ctx, err, "read page %s", name) + } + return page, nil +} + +// Build warms the given keys concurrently. A per-key failure is logged and does +// not fail the build; only a cancelled ctx makes Build return an error. +func (p *pageIndex) Build(ctx context.Context, keys []Key) error { + if err := ctx.Err(); err != nil { + return errors.Wrap(ctx, err, "build page index") + } + funcs := make([]run.Func, 0, len(keys)) + for _, key := range keys { + key := NewKey(key.VaultPath, key.PagesDir) + funcs = append(funcs, func(ctx context.Context) error { + _, _, cancelErr := p.ensure(ctx, key, false) + return cancelErr + }) + } + if err := run.All(ctx, funcs...); err != nil { + return errors.Wrap(ctx, err, "build page index") + } + return nil +} + +// Refresh re-reads the key's changed files and blocks until a listing that +// started after this call has been applied. It returns nil whether that +// listing succeeded or failed, and a wrapped error only when ctx was cancelled +// first. +func (p *pageIndex) Refresh(ctx context.Context, key Key) error { + key = NewKey(key.VaultPath, key.PagesDir) + _, _, cancelErr := p.ensure(ctx, key, true) + return cancelErr +} + +// MarkDirty marks the named keys stale at folder level. Keys the index has +// never seen are ignored: their first read builds them anyway. +func (p *pageIndex) MarkDirty(keys ...Key) { + p.mu.Lock() + defer p.mu.Unlock() + for _, key := range keys { + key = NewKey(key.VaultPath, key.PagesDir) + if e, ok := p.entries[key]; ok { + p.markFolderDirtyLocked(e, UnboundedFolderMarkExplicit) + } + } +} + +// markFolderDirty marks one key folder-level with the given unbounded cause. +func (p *pageIndex) markFolderDirty(key Key, reason string) { + key = NewKey(key.VaultPath, key.PagesDir) + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.entries[key]; ok { + p.markFolderDirtyLocked(e, reason) + } +} + +// markFolderDirtyLocked records one folder-level mark. The caller must hold the +// mutex. +func (p *pageIndex) markFolderDirtyLocked(e *entry, reason string) { + e.requestSeq++ + e.dirtySeq = e.requestSeq + e.revision++ + // A folder-level mark names no file, so the change cannot be bounded to a + // set of single pages. + e.recordChangeLocked(nil, false, reason) +} + +// MarkFileDirty marks one file of the key stale so the next read re-reads it +// before serving. The name is the item id; it is applied only when the +// exactness rule holds, otherwise the whole key is marked folder-level. +// +// The exactness rule mirrors vault-cli's own file lookup: the id is taken with +// its "[["/"]]" wrapper stripped, it must be a plain base name without a path +// separator or "..", the key's current snapshot must hold a page whose name +// equals it byte-for-byte, and ".md" must exist at mark time. Anything +// else may have been resolved by vault-cli's case-insensitive substring +// fallback, so the mark widens to the whole folder. +func (p *pageIndex) MarkFileDirty(key Key, name string) { + key = NewKey(key.VaultPath, key.PagesDir) + stripped := stripWikilinkName(name) + + p.mu.Lock() + e, ok := p.entries[key] + switch { + case !ok || !e.hasSnapshot: + p.mu.Unlock() + p.markFolderDirty(key, UnboundedFolderMarkWidenedKey) + return + case !validFileMarkName(stripped): + p.mu.Unlock() + p.markFolderDirty(key, UnboundedFolderMarkWidenedNameInvalid) + return + case !snapshotHasName(e.snapshot.entries, stripped): + p.mu.Unlock() + p.markFolderDirty(key, UnboundedFolderMarkWidenedName) + return + } + p.mu.Unlock() + + filename := stripped + ".md" + if _, err := os.Stat(filepath.Join(key.VaultPath, key.PagesDir, filename)); err != nil { + p.markFolderDirty(key, UnboundedFolderMarkWidenedFile) + return + } + + p.mu.Lock() + defer p.mu.Unlock() + e = p.entryLocked(key) + e.requestSeq++ + e.writeMarked[filename] = e.requestSeq + e.revision++ + e.recordChangeLocked([]string{filename}, true, "") +} + +// ForceReload marks every known key so the next read of each re-reads every +// file, whatever the fingerprints say. +func (p *pageIndex) ForceReload() { + p.mu.Lock() + defer p.mu.Unlock() + for _, e := range p.entries { + e.requestSeq++ + e.reloadSeq = e.requestSeq + e.revision++ + // A forced reload re-reads every file, so its change is unbounded. + e.recordChangeLocked(nil, false, UnboundedForcedReload) + } +} + +// Revision reports the key's revision: a value that advances whenever the +// key's page snapshot is published or marked stale. An unknown key reports 0. +func (p *pageIndex) Revision(key Key) uint64 { + key = NewKey(key.VaultPath, key.PagesDir) + p.mu.Lock() + defer p.mu.Unlock() + e, ok := p.entries[key] + if !ok { + return 0 + } + return e.revision +} + +// ChangedPagesSince returns the names of the pages whose parsed form changed in +// the key's publications and marks after revision. Each name is a base filename +// WITH its ".md" suffix, matching the page index's write-mark and snapshot +// naming. ok is false when the key cannot bound the set: an unknown key, a +// revision ahead of the key's current one, a revision the retained change window +// no longer covers, a folder-level mark, a forced reload, or the key's first +// publication — which publishes the whole snapshot whatever the build's kind, +// including a hydrate whose stored baseline carries pages the delta never names. +// +// A caller uses it to re-derive only the rows whose files moved, so a per-file +// write mark costs a bounded set of row derivations instead of a vault-wide +// rebuild. The returned slice is a fresh copy; the caller may keep or mutate it. +func (p *pageIndex) ChangedPagesSince(key Key, revision uint64) ([]string, bool) { + key = NewKey(key.VaultPath, key.PagesDir) + p.mu.Lock() + names, ok, reason := p.changedPagesSinceLocked(key, revision) + p.mu.Unlock() + if !ok { + p.metrics.ChangedPagesUnbounded(reason) + } + return names, ok +} + +// changedPagesSinceLocked is ChangedPagesSince's body under the mutex. It +// returns the refusal's reason as its third value. The caller must hold the +// mutex. +func (p *pageIndex) changedPagesSinceLocked(key Key, revision uint64) ([]string, bool, string) { + e, ok := p.entries[key] + if !ok { + return nil, false, UnboundedUnknownKey + } + if revision > e.revision { + // The caller holds a revision the key has never reached: it cannot be + // placed against this key's history. + return nil, false, UnboundedRevisionAhead + } + if revision == e.revision { + return nil, true, "" + } + if len(e.changeLog) == 0 || e.changeLog[0].revision > revision+1 { + // The advances between revision and the current one are not all + // retained, so the set cannot be bounded. + return nil, false, UnboundedWindowNotRetained + } + names := make([]string, 0, len(e.changeLog)) + seen := make(map[string]bool, len(e.changeLog)) + for _, change := range e.changeLog { + if change.revision <= revision { + continue + } + if !change.bounded { + // The earliest unbounded advance in the window decides the cause; + // entries after it are not consulted. + return nil, false, change.reason + } + for _, name := range change.names { + if seen[name] { + continue + } + seen[name] = true + names = append(names, name) + } + } + return names, true, "" +} + +// Rescan refreshes every known key once per RescanInterval until ctx is done. +func (p *pageIndex) Rescan(ctx context.Context) error { + lastRescan := p.currentDateTimeGetter.Now() + for { + if err := p.waiter.Wait(ctx, libtime.Duration(rescanPollInterval)); err != nil { + return nil + } + if p.currentDateTimeGetter.Now().Sub(lastRescan).Duration() < RescanInterval { + continue + } + lastRescan = p.currentDateTimeGetter.Now() + if err := p.refreshAll(ctx); err != nil { + return nil + } + } +} + +// refreshAll refreshes every known key concurrently and waits for all of them, +// so rescans never pile up. +func (p *pageIndex) refreshAll(ctx context.Context) error { + keys := p.knownKeys() + funcs := make([]run.Func, 0, len(keys)) + for _, key := range keys { + funcs = append(funcs, func(ctx context.Context) error { + return p.Refresh(ctx, key) + }) + } + return run.All(ctx, funcs...) +} + +// knownKeys returns every key the index has an entry for. +func (p *pageIndex) knownKeys() []Key { + p.mu.Lock() + defer p.mu.Unlock() + keys := make([]Key, 0, len(p.entries)) + for key := range p.entries { + keys = append(keys, key) + } + return keys +} + +// snapshot returns the current published snapshot, if one exists. +func (p *pageIndex) snapshot(key Key) (published, bool) { + p.mu.Lock() + defer p.mu.Unlock() + e, ok := p.entries[key] + if !ok || !e.hasSnapshot { + return published{}, false + } + return e.snapshot, true +} + +// entryLocked returns the key's entry, creating it when missing. The caller +// must hold the mutex. +func (p *pageIndex) entryLocked(key Key) *entry { + e, ok := p.entries[key] + if !ok { + e = &entry{ + fileReadSeq: map[string]uint64{}, + writeMarked: map[string]uint64{}, + } + p.entries[key] = e + } + return e +} + +// takeReadSeq reserves the next single-file read sequence. The caller must not +// hold the mutex. +func (p *pageIndex) takeReadSeq() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + p.readSeq++ + return p.readSeq +} + +// stripWikilinkName removes the "[[" and "]]" wrapper an item id may carry, +// exactly as vault-cli's own file lookup does. +func stripWikilinkName(name string) string { + name = strings.TrimPrefix(name, "[[") + return strings.TrimSuffix(name, "]]") +} + +// validFileMarkName rejects an id that must never be turned into a file path. +func validFileMarkName(name string) bool { + return name != "" && + !strings.Contains(name, "..") && + !strings.ContainsRune(name, filepath.Separator) && + !strings.ContainsRune(name, '/') +} + +// snapshotHasName reports whether the snapshot holds a page whose file name +// equals name byte-for-byte. +func snapshotHasName(entries []*Entry, name string) bool { + for _, entry := range entries { + if entry.Page.Name == name { + return true + } + } + return false +} + +// published is one immutable snapshot: the entries and the pages they hold, in +// the same order and with the same pointers. pages is nil exactly when entries +// is nil. +type published struct { + entries []*Entry + pages []*domain.Page +} + +// publish builds the published form of entries. +func publish(entries []*Entry) published { + if entries == nil { + return published{} + } + pages := make([]*domain.Page, len(entries)) + for i, entry := range entries { + pages[i] = entry.Page + } + return published{entries: entries, pages: pages} +} diff --git a/pkg/index/index_allocation_test.go b/pkg/index/index_allocation_test.go new file mode 100644 index 0000000..4ff9b62 --- /dev/null +++ b/pkg/index/index_allocation_test.go @@ -0,0 +1,247 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + . "github.com/onsi/ginkgo/v2" + + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// allocationFileCount is the folder size the allocation measurement uses. It is +// large enough that a per-page structure dwarfs the listing overhead. +const allocationFileCount = 5000 + +// writeAllocationVault builds a real temp vault holding count parseable pages +// and returns the vault root and the pages folder. +func writeAllocationVault(t FullGinkgoTInterface, count int) (vaultDir, folderDir string) { + t.Helper() + vaultDir = t.TempDir() + folderDir = filepath.Join(vaultDir, equivalenceFolder) + if err := os.MkdirAll(folderDir, 0750); err != nil { + t.Fatalf("mkdir %s: %v", folderDir, err) + } + for i := 0; i < count; i++ { + writeAllocationFile(t, folderDir, i, false) + } + return vaultDir, folderDir +} + +// writeAllocationFile writes one page file. A changed file's content differs in +// byte count, so its fingerprint moves. +func writeAllocationFile(t FullGinkgoTInterface, folderDir string, i int, changed bool) { + t.Helper() + suffix := "" + if changed { + suffix = " changed" + } + name := fmt.Sprintf("Page%05d.md", i) + content := fmt.Sprintf("---\ntitle: Page%05d\n---\n# Page%05d%s\n", i, i, suffix) + if err := os.WriteFile(filepath.Join(folderDir, name), []byte(content), 0600); err != nil { + t.Fatalf("write %s: %v", name, err) + } +} + +// measureAllocation reports the TotalAlloc delta around fn. +func measureAllocation(fn func()) uint64 { + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + fn() + var after runtime.MemStats + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +var _ = Describe("Allocation", func() { + It("allocates nothing beyond its folder listing on an unchanged stat-diff", func() { + if raceDetectorEnabled { + Skip("allocation measurement is only meaningful without -race") + } + unchangedRescanAllocation(GinkgoT()) + }) +}) + +var _ = Describe("Per-file write marks", func() { + It("re-reads a write-marked file even when no fingerprint changed", func() { + writeMarkSkipsUnchangedFastPath(GinkgoT()) + }) +}) + +// unchangedRescanAllocation measures an unchanged and a small changed stat-diff +// against the cost of one folder listing. +func unchangedRescanAllocation(t FullGinkgoTInterface) { + ctx := context.Background() + vaultDir, folderDir := writeAllocationVault(t, allocationFileCount) + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + if _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder); err != nil { + t.Fatalf("warm list: %v", err) + } + + listingBytes := measureAllocation(func() { + if _, err := ri.lister.ListFiles(ctx, vaultDir, equivalenceFolder); err != nil { + t.Fatalf("listing: %v", err) + } + }) + if listingBytes == 0 { + t.Fatal("listing allocated nothing; the measurement is invalid") + } + + ri.reader.reset() + fpBefore := index.FingerprintSetIdentity(ri.idx, key) + snapBefore := index.SnapshotIdentity(ri.idx, key) + passBytes := measureAllocation(func() { + if err := ri.idx.Refresh(ctx, key); err != nil { + t.Fatalf("unchanged refresh: %v", err) + } + }) + fpAfter := index.FingerprintSetIdentity(ri.idx, key) + snapAfter := index.SnapshotIdentity(ri.idx, key) + + ratio := float64(passBytes) / float64(listingBytes) + t.Logf("listing bytes=%d", listingBytes) + t.Logf("unchanged pass bytes=%d ratio=%.3f", passBytes, ratio) + t.Logf("fingerprint identity before=%q after=%q", fpBefore, fpAfter) + t.Logf("snapshot identity before=%q after=%q", snapBefore, snapAfter) + t.Logf("unchanged pass reads=%v", ri.reader.Names()) + + if ratio > 1.35 { + t.Errorf("unchanged pass ratio %.3f exceeds 1.35", ratio) + } + if passBytes > 5_500_000 { + t.Errorf("unchanged pass allocated %d bytes, exceeds 5.5 MB", passBytes) + } + for label, value := range map[string]string{ + "fingerprint before": fpBefore, + "fingerprint after": fpAfter, + "snapshot before": snapBefore, + "snapshot after": snapAfter, + } { + if value == "" { + t.Fatalf("%s identity is empty; the key lookup missed", label) + } + } + if fpBefore != fpAfter { + t.Errorf("unchanged pass replaced the fingerprint set: %q -> %q", fpBefore, fpAfter) + } + if snapBefore != snapAfter { + t.Errorf("unchanged pass replaced the snapshot: %q -> %q", snapBefore, snapAfter) + } + + changedRescanAllocation(ctx, t, ri, key, folderDir, listingBytes) + emptyFolderRescan(ctx, t, ri, key, folderDir) +} + +// changedRescanAllocation asserts a pass that changes 10 of the 5000 files +// re-reads exactly those 10 and allocates in proportion to them. +func changedRescanAllocation( + ctx context.Context, + t FullGinkgoTInterface, + ri *realIndex, + key index.Key, + folderDir string, + listingBytes uint64, +) { + ri.reader.reset() + settle() + const changedCount = 10 + wantNames := make([]string, 0, changedCount) + for i := 0; i < changedCount; i++ { + writeAllocationFile(t, folderDir, i, true) + wantNames = append(wantNames, fmt.Sprintf("Page%05d.md", i)) + } + changedBytes := measureAllocation(func() { + if err := ri.idx.Refresh(ctx, key); err != nil { + t.Fatalf("changed refresh: %v", err) + } + }) + changedRatio := float64(changedBytes) / float64(listingBytes) + t.Logf("changed pass bytes=%d ratio=%.3f", changedBytes, changedRatio) + t.Logf("changed pass reads=%v", ri.reader.Names()) + if got := ri.reader.Names(); !reflect.DeepEqual(got, wantNames) { + t.Errorf("changed pass read %v, want %v", got, wantNames) + } + if changedRatio > 1.5 { + t.Errorf("changed pass ratio %.3f exceeds 1.5", changedRatio) + } + +} + +// emptyFolderRescan asserts an emptied folder lists as nil and equals the +// empty recorded set, so the second pass over it is an unchanged pass. +func emptyFolderRescan( + ctx context.Context, + t FullGinkgoTInterface, + ri *realIndex, + key index.Key, + folderDir string, +) { + if err := os.RemoveAll(folderDir); err != nil { + t.Fatalf("remove folder: %v", err) + } + if err := ri.idx.Refresh(ctx, key); err != nil { + t.Fatalf("empty refresh: %v", err) + } + ri.reader.reset() + fpBefore := index.FingerprintSetIdentity(ri.idx, key) + snapBefore := index.SnapshotIdentity(ri.idx, key) + if err := ri.idx.Refresh(ctx, key); err != nil { + t.Fatalf("second empty refresh: %v", err) + } + fpAfter := index.FingerprintSetIdentity(ri.idx, key) + snapAfter := index.SnapshotIdentity(ri.idx, key) + t.Logf("empty folder fingerprint identity %q -> %q", fpBefore, fpAfter) + t.Logf("empty folder snapshot identity %q -> %q", snapBefore, snapAfter) + if fpBefore != fpAfter || snapBefore != snapAfter { + t.Errorf( + "empty-folder unchanged pass replaced identity: fingerprint %q -> %q, snapshot %q -> %q", + fpBefore, fpAfter, snapBefore, snapAfter, + ) + } + if got := ri.reader.Names(); len(got) != 0 { + t.Errorf("empty-folder unchanged pass read %v, want none", got) + } +} + +// writeMarkSkipsUnchangedFastPath proves a pending write mark defeats the +// unchanged fast path: the marked file is re-read even though no fingerprint +// changed anywhere. +func writeMarkSkipsUnchangedFastPath(t FullGinkgoTInterface) { + ctx := context.Background() + vaultDir := t.TempDir() + folderDir := filepath.Join(vaultDir, equivalenceFolder) + if err := os.MkdirAll(folderDir, 0750); err != nil { + t.Fatalf("mkdir %s: %v", folderDir, err) + } + for _, name := range []string{"Alpha.md", "Beta.md"} { + title := name[:len(name)-len(".md")] + content := fmt.Sprintf("---\ntitle: %s\n---\n# %s\n", title, title) + if err := os.WriteFile(filepath.Join(folderDir, name), []byte(content), 0600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + if _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder); err != nil { + t.Fatalf("warm list: %v", err) + } + + ri.reader.reset() + ri.idx.MarkFileDirty(key, "Alpha") + if err := ri.idx.Refresh(ctx, key); err != nil { + t.Fatalf("refresh: %v", err) + } + if got := ri.reader.Names(); !reflect.DeepEqual(got, []string{"Alpha.md"}) { + t.Fatalf("write-marked pass read %v, want [Alpha.md]", got) + } +} diff --git a/pkg/index/index_build.go b/pkg/index/index_build.go new file mode 100644 index 0000000..9800929 --- /dev/null +++ b/pkg/index/index_build.go @@ -0,0 +1,732 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "log/slog" + "sort" + + "github.com/bborbe/errors" +) + +// role describes what the caller that acquired a build has to do with it. +type role int + +const ( + // roleRun means the caller owns the build and it is already in flight. + roleRun role = iota + // roleFollowUp means the caller owns a queued build and must wait until it + // is promoted before running it. + roleFollowUp + // roleWait means the caller only waits for somebody else's build. + roleWait +) + +// ensure returns the key's published snapshot, resolving whatever work is pending for it. +// +// It returns the build error separately from a cancellation of the caller's own +// ctx: a failed build is not an error for a caller that can still serve a +// previous snapshot, while a cancelled caller must not be handed stale pages. +// +// With refresh set, the caller wants a stat-diff that started after this call +// and gets one; without it the caller blocks only while a mark is pending and +// is served from the current snapshot otherwise. +func (p *pageIndex) ensure( + ctx context.Context, + key Key, + refresh bool, +) (published, error, error) { + reason := ReadReasonWrite + if refresh { + reason = ReadReasonRescan + } + p.mu.Lock() + e := p.entryLocked(key) + if refresh { + e.requestSeq++ + } + // Only work marked before this call is resolved here: a mark recorded while + // a read is in flight stays pending for the next reader. + entryNeedSeq := e.requestSeq + p.mu.Unlock() + + var last published + for first := true; ; first = false { + p.mu.Lock() + e = p.entryLocked(key) + kind, needSeq, pending := p.pendingLocked(e, refresh) + if !pending || (!first && needSeq > entryNeedSeq) { + snapshot := e.snapshot + p.mu.Unlock() + if first { + return snapshot, nil, nil + } + return last, nil, nil + } + b, callerRole := p.acquireLocked(e, kind, reason, needSeq) + p.mu.Unlock() + + snap, buildErr, cancelErr := p.await(ctx, key, b, callerRole) + if cancelErr != nil { + return published{}, nil, cancelErr + } + last = snap + if buildErr != nil { + // A failed listing leaves the mark pending for the next reader. + return snap, buildErr, nil + } + if refresh { + if b.covers(buildStatDiff, needSeq) { + return snap, nil, nil + } + // The build that was in flight re-reads files without listing the + // folder, so it cannot satisfy a refresh: wait for one that does. + continue + } + } +} + +// pendingLocked reports the work the key needs next: a cold build when it has +// no snapshot, otherwise the highest-priority pending mark. The caller must +// hold the mutex. +func (p *pageIndex) pendingLocked(e *entry, refresh bool) (buildKind, uint64, bool) { + if !e.hasSnapshot { + return buildCold, e.requestSeq, true + } + if refresh { + return buildStatDiff, e.requestSeq, true + } + if e.reloadSeq > e.reloadResolvedSeq { + return buildReload, e.reloadSeq, true + } + if len(e.writeMarked) > 0 { + return buildFileReads, maxMarkedSeq(e.writeMarked), true + } + if e.fileReadsInflightLocked() { + // A per-file re-read is already running: join it rather than serving a + // snapshot the mark was recorded against. + return buildFileReads, 0, true + } + if e.dirtySeq > e.dirtyResolvedSeq { + return buildStatDiff, e.dirtySeq, true + } + return 0, 0, false +} + +// fileReadsInflightLocked reports whether a per-file re-read is running or +// queued for the key. The caller must hold the mutex. +func (e *entry) fileReadsInflightLocked() bool { + return (e.inflight != nil && e.inflight.kind == buildFileReads) || + (e.followUp != nil && e.followUp.kind == buildFileReads) +} + +// maxMarkedSeq returns the highest requestSeq in a write-mark map. +func maxMarkedSeq(marked map[string]uint64) uint64 { + var highest uint64 + for _, seq := range marked { + if seq > highest { + highest = seq + } + } + return highest +} + +// acquireLocked picks the build the caller waits on and starts one when none +// covers its need. The caller must hold the mutex. +func (p *pageIndex) acquireLocked( + e *entry, + kind buildKind, + reason string, + needSeq uint64, +) (*build, role) { + if e.inflight != nil && e.inflight.covers(kind, needSeq) { + return e.inflight, roleWait + } + if e.followUp != nil { + return e.followUp, roleWait + } + if e.inflight != nil { + return p.queueFollowUpLocked(e, kind, reason), roleFollowUp + } + b := newBuild(kind, reason, e.requestSeq) + e.inflight = b + return b, roleRun +} + +// queueFollowUpLocked appends the one queued build a key may have. The caller +// must hold the mutex. +func (p *pageIndex) queueFollowUpLocked(e *entry, kind buildKind, reason string) *build { + b := newBuild(kind, reason, e.requestSeq) + e.followUp = b + return b +} + +// await waits for or runs the build the caller acquired. +func (p *pageIndex) await( + ctx context.Context, + key Key, + b *build, + callerRole role, +) (published, error, error) { + if callerRole == roleWait { + select { + case <-b.done: + case <-ctx.Done(): + return published{}, nil, errors.Wrap(ctx, ctx.Err(), "wait for page index build") + } + return b.snap, b.err, nil + } + if callerRole == roleFollowUp { + // A follow-up must never be abandoned: it is already the key's queued + // build, and nobody else would run it. + <-b.started + } + p.execute(ctx, key, b) + if err := ctx.Err(); err != nil { + return published{}, b.err, errors.Wrap(ctx, err, "page index build") + } + return b.snap, b.err, nil +} + +// execute runs the build's read and publishes its result. It runs on the +// goroutine of the caller that owns the build and finishes even when that +// caller's ctx is cancelled. +func (p *pageIndex) execute(ctx context.Context, key Key, b *build) { + readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rebuildTimeout) + defer cancel() + + switch b.kind { + case buildFileReads: + p.executeFileReads(readCtx, key, b) + default: + p.executeListing(readCtx, key, b) + } + p.release(key, b) +} + +// executeFileReads re-reads exactly the write-marked files. The marks are taken +// out under the lock as the read starts, so a mark recorded during the read +// stays pending for the next one. +func (p *pageIndex) executeFileReads(ctx context.Context, key Key, b *build) { + p.mu.Lock() + e := p.entryLocked(key) + startSeq := e.requestSeq + b.startSeq = startSeq + names := make([]string, 0, len(e.writeMarked)) + for name, seq := range e.writeMarked { + if seq <= startSeq { + names = append(names, name) + delete(e.writeMarked, name) + } + } + p.mu.Unlock() + + var delta *storeDelta + for _, name := range names { + seq := p.takeReadSeq() + read, fingerprint, readErr := p.reader.ReadEntry( + ctx, + key.VaultPath, + key.PagesDir, + name, + ) + p.metrics.FileRead(ReadReasonWrite) + + p.mu.Lock() + d := p.applyFileReadLocked(e, key, name, seq, read, fingerprint, readErr) + p.mu.Unlock() + delta = mergeStoreDeltas(delta, d) + } + + p.mu.Lock() + b.delta = delta + p.mu.Unlock() +} + +// mergeStoreDeltas returns a delta holding both inputs' puts and deletes; nil +// inputs contribute nothing and the result is nil when both are nil. +func mergeStoreDeltas(a, b *storeDelta) *storeDelta { + if a == nil { + return b + } + if b == nil { + return a + } + merged := &storeDelta{ + puts: make([]StoredEntry, 0, len(a.puts)+len(b.puts)), + deletes: make([]string, 0, len(a.deletes)+len(b.deletes)), + } + merged.puts = append(merged.puts, a.puts...) + merged.puts = append(merged.puts, b.puts...) + merged.deletes = append(merged.deletes, a.deletes...) + merged.deletes = append(merged.deletes, b.deletes...) + return merged +} + +// executeListing lists the key's folder and reads the files the build's kind +// asks for, then merges the result into the published snapshot. +// +// A pass whose listing matches the recorded fingerprints, whose name set +// matches the recorded name set and that has no file to re-read is an +// unchanged pass: it publishes nothing, leaves the published snapshot and the +// recorded fingerprints exactly in place, and allocates nothing beyond the one +// listing it performs. A pass with work to do re-reads exactly the changed, +// new and forced names and updates the recorded fingerprints for those names +// in place. +func (p *pageIndex) executeListing(ctx context.Context, key Key, b *build) { + p.mu.Lock() + e := p.entryLocked(key) + startSeq := e.requestSeq + b.startSeq = startSeq + baseline := e.hasSnapshot + full := b.kind == buildCold || b.kind == buildReload || !baseline + forced := takeWriteMarksLocked(e, startSeq, full) + p.mu.Unlock() + + reason := b.reason + switch { + case !baseline, b.kind == buildCold: + reason = ReadReasonBuild + case b.kind == buildReload: + reason = ReadReasonReload + } + + listingSeq := p.takeReadSeq() + entries, listErr := p.lister.ListFiles(ctx, key.VaultPath, key.PagesDir) + if listErr != nil { + p.failListing(key, b, listErr) + return + } + + var toRead map[string]bool + if !full { + p.mu.Lock() + e = p.entryLocked(key) + var unchanged bool + toRead, unchanged = diffListingLocked(e, entries, forced) + if unchanged { + b.snap = e.snapshot + e.dirtyResolvedSeq = startSeq + p.mu.Unlock() + return + } + p.mu.Unlock() + } + + reads, collectErr := p.collect(ctx, key, listingSeq, entries, full, toRead, reason) + if collectErr != nil { + p.failListing(key, b, collectErr) + return + } + + p.mu.Lock() + defer p.mu.Unlock() + e = p.entryLocked(key) + b.delta = publishListingLocked(e, entries, reads, full) + b.snap = e.snapshot + e.dirtyResolvedSeq = startSeq + if full { + e.reloadResolvedSeq = startSeq + } +} + +// takeWriteMarksLocked takes out every write mark recorded at or before +// startSeq. A stat-diff returns them as forced names; a full listing re-reads +// every file anyway and returns nil. The caller must hold the mutex. +func takeWriteMarksLocked(e *entry, startSeq uint64, full bool) map[string]bool { + var forced map[string]bool + for name, seq := range e.writeMarked { + if seq > startSeq { + continue + } + delete(e.writeMarked, name) + if full { + continue + } + if forced == nil { + forced = map[string]bool{} + } + forced[name] = true + } + return forced +} + +// diffListingLocked compares a listing with the recorded fingerprints and +// returns the names to re-read, and whether the pass is unchanged. An +// unchanged pass requires a published snapshot to keep serving. The name set +// is equal exactly when the lengths match and every listed entry is present +// and equal, and the missing/empty state did not flip. The caller must hold +// the mutex. +func diffListingLocked( + e *entry, + entries []FileEntry, + forced map[string]bool, +) (map[string]bool, bool) { + var toRead map[string]bool + for _, fe := range entries { + recorded, ok := e.fingerprints[fe.Name] + if !forced[fe.Name] && ok && recorded == fe.Fingerprint { + continue + } + if toRead == nil { + toRead = make(map[string]bool) + } + toRead[fe.Name] = true + } + unchanged := len(forced) == 0 && + len(toRead) == 0 && + e.hasSnapshot && + len(entries) == len(e.fingerprints) && + (entries == nil) == (e.snapshot.entries == nil) + return toRead, unchanged +} + +// publishListingLocked merges a listing's reads into the key's snapshot, +// publishes it when the entries moved and returns the listing's delta. The +// caller must hold the mutex. +func publishListingLocked( + e *entry, + entries []FileEntry, + reads *snapshotReads, + full bool, +) *storeDelta { + merged, dropped := mergeSnapshot(e, entries, reads) + delta := listingDelta(reads, dropped) + if e.hasSnapshot && sameEntries(e.snapshot.entries, merged) { + return delta + } + e.snapshot = publish(merged) + e.revision++ + // A publication is attributable to a bounded set of single files only when + // it merged into an already-published snapshot. A key's first publication + // publishes the whole snapshot whatever the build's kind, so it stays + // unbounded. + bounded := !full && e.hasSnapshot + e.recordChangeLocked(listingChangedNames(full, delta), bounded, unboundedListingReason(e, bounded)) + e.hasSnapshot = true + return delta +} + +// unboundedListingReason returns the unbounded cause of a listing's +// publication, or "" when it is bounded. +func unboundedListingReason(e *entry, bounded bool) string { + switch { + case bounded: + return "" + case !e.hasSnapshot: + return UnboundedFirstPublication + default: + return UnboundedForcedReload + } +} + +// failListing records a listing or collection failure: nothing is published, +// the error is logged with the key, and the mark stays pending so the next +// read retries. +func (p *pageIndex) failListing(key Key, b *build, err error) { + p.mu.Lock() + defer p.mu.Unlock() + b.snap = published{} + b.err = err + slog.Error( + "page index rebuild failed", + "vault", key.VaultPath, + "pagesDir", key.PagesDir, + "error", err, + ) +} + +// listingDelta returns the change one listing produced: a put for every +// entry it re-read (including an excluded entry, stored as fingerprint plus a +// nil page) and a delete for every name the entry held before the merge that +// the merged set no longer holds. dropped is exactly those names, captured by +// mergeSnapshot before its in-place update removed them. +func listingDelta(reads *snapshotReads, dropped []string) *storeDelta { + delta := &storeDelta{} + for name, read := range reads.reads { + delta.puts = append(delta.puts, StoredEntry{ + Filename: name, + Entry: read.entry, + Fingerprint: read.fingerprint, + }) + } + delta.deletes = append(delta.deletes, dropped...) + return delta +} + +// listingChangedNames returns the page names a listing's publication can be +// attributed to: every name the listing re-read and every name it dropped. A +// full listing — a cold build or a reload — re-reads the whole folder, so it +// cannot be attributed to a bounded set and reports none; the caller records it +// as unbounded instead. +func listingChangedNames(full bool, delta *storeDelta) []string { + if full { + return nil + } + names := make([]string, 0, len(delta.puts)+len(delta.deletes)) + for _, put := range delta.puts { + names = append(names, put.Filename) + } + return append(names, delta.deletes...) +} + +// release clears the finished build and promotes the key's follow-up, so no +// caller can slip a parallel build into the gap. +func (p *pageIndex) release(key Key, b *build) { + p.mu.Lock() + defer p.mu.Unlock() + e := p.entryLocked(key) + if b.snap.entries == nil && b.err == nil && e.hasSnapshot { + // A per-file re-read publishes by splicing; its callers still need the + // snapshot that is current once it is done. + b.snap = e.snapshot + } + if e.inflight == b { + e.inflight = nil + } + if e.followUp != nil { + next := e.followUp + e.followUp = nil + next.startSeq = e.requestSeq + e.inflight = next + close(next.started) + } + close(b.done) +} + +// maxUnreadablePageWarnings is the number of per-file skip warnings a single +// listing emits before it stops naming files individually and prints one +// summary line instead. It bounds a folder full of unreadable pages. +const maxUnreadablePageWarnings = 10 + +// fileRead is one listed file's read result within a listing. A nil entry +// means the read failed, so the file is excluded. +type fileRead struct { + seq uint64 + entry *Entry + fingerprint FileFingerprint +} + +// snapshotReads is one listing's collected reads: the sequence the listing +// itself started at, the K-sized map of each re-read file's result and whether +// the folder was missing. It holds no per-name order slice: the listing's +// []FileEntry supplies the order. +type snapshotReads struct { + listingSeq uint64 + reads map[string]fileRead + // missing reports that the folder does not exist, which lists as nil. + missing bool +} + +// collect reads the listed entries that need it: every entry when full is set, +// otherwise only the names in toRead (the changed, new and forced names). It +// lists nothing and copies no recorded fingerprints, so its allocation is +// bounded by the entries it actually re-read. It returns the listing's +// sequence and the result and fingerprint of every entry it re-read, including +// the ones whose read failed. +func (p *pageIndex) collect( + ctx context.Context, + key Key, + listingSeq uint64, + entries []FileEntry, + full bool, + toRead map[string]bool, + reason string, +) (*snapshotReads, error) { + result := &snapshotReads{ + listingSeq: listingSeq, + reads: map[string]fileRead{}, + missing: entries == nil, + } + skipped := 0 + for _, fe := range entries { + if err := ctx.Err(); err != nil { + return nil, errors.Wrap(ctx, err, "build page snapshot") + } + if !full && !toRead[fe.Name] { + continue + } + seq := p.takeReadSeq() + read, fingerprint, readErr := p.reader.ReadEntry( + ctx, + key.VaultPath, + key.PagesDir, + fe.Name, + ) + p.metrics.FileRead(reason) + result.reads[fe.Name] = fileRead{seq: seq, entry: read, fingerprint: fingerprint} + if readErr != nil { + if skipped < maxUnreadablePageWarnings { + p.warnf( + "skipping unreadable page %q in %q/%q: %v", + fe.Name, + key.VaultPath, + key.PagesDir, + readErr, + ) + } + skipped++ + } + } + if skipped >= maxUnreadablePageWarnings { + p.warnf("skipping %d unreadable pages", skipped) + } + return result, nil +} + +// sameEntries reports whether two snapshots hold the same entries in the same +// order, distinguishing a nil snapshot from an empty one. +func sameEntries(a, b []*Entry) bool { + if (a == nil) != (b == nil) || len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// snapshotMerge accumulates one listing's merge into a new snapshot and +// updates the entry's recorded fingerprints in place. The entry's current +// snapshot is the other input, so a read a later-started one has already +// superseded is never applied. +type snapshotMerge struct { + entry *entry + reads *snapshotReads + entries []*Entry + dropped []string +} + +// listed takes a listed file: the listing's own read when no later-started read +// has already been applied to that name, else the page currently published. A +// re-read updates the recorded fingerprint in place; a name carried forward +// keeps the fingerprint it already has. +func (m *snapshotMerge) listed(name string, current *Entry) { + read, ok := m.reads.reads[name] + if ok && read.seq >= m.entry.fileReadSeq[name] { + m.entry.fileReadSeq[name] = read.seq + m.entry.fingerprints[name] = read.fingerprint + if read.entry != nil { + m.entries = append(m.entries, read.entry) + } + return + } + m.kept(current) +} + +// unlisted drops a name the listing no longer holds, unless the listing started +// before that name's last applied read — then a newer read put it there. A +// dropped name has its recorded fingerprint deleted in place so the recorded +// name set never keeps a tombstone the next listing could not match. +func (m *snapshotMerge) unlisted(name string, current *Entry) { + if m.reads.listingSeq > m.entry.fileReadSeq[name] { + m.remove(name) + return + } + m.kept(current) +} + +// kept carries a currently published page forward. A name the listing excludes +// has no page but keeps its fingerprint, so an unchanged broken file is neither +// re-read nor re-warned on the next listing. +func (m *snapshotMerge) kept(current *Entry) { + if current != nil { + m.entries = append(m.entries, current) + } +} + +// remove deletes one recorded fingerprint in place, recording it for the store +// delta when the prior value was non-zero — a zero fingerprint is the "never +// recorded" sentinel and has no stored entry to delete. +func (m *snapshotMerge) remove(name string) { + previous, ok := m.entry.fingerprints[name] + if !ok { + return + } + delete(m.entry.fingerprints, name) + if previous != (FileFingerprint{}) { + m.dropped = append(m.dropped, name) + } +} + +// pruneFingerprints deletes every recorded fingerprint that is neither listed +// nor currently published — an excluded file the listing no longer holds, whose +// fingerprint the in-place update would otherwise keep forever. A tombstone +// kept past the pass would leave the recorded name set larger than any listing, +// so the unchanged fast path could never fire again. +func (m *snapshotMerge) pruneFingerprints(entries []FileEntry, current []*Entry) { + for name := range m.entry.fingerprints { + if listedContains(entries, name) || publishedContains(current, name) { + continue + } + m.remove(name) + } +} + +// listedContains reports whether the sorted listing holds name. +func listedContains(entries []FileEntry, name string) bool { + i := sort.Search(len(entries), func(i int) bool { return entries[i].Name >= name }) + return i < len(entries) && entries[i].Name == name +} + +// publishedContains reports whether the sorted snapshot holds name, whose +// ordering key is the page's file name plus ".md". +func publishedContains(entries []*Entry, name string) bool { + i := sort.Search(len(entries), func(i int) bool { + return entries[i].Page.Name+".md" >= name + }) + return i < len(entries) && entries[i].Page.Name+".md" == name +} + +// mergeSnapshot merges a listing's reads into the entry's current snapshot in +// one O(n) pass over both ordered name sequences and updates the entry's +// recorded fingerprints in place. It returns the new entries slice and the names +// whose recorded fingerprints it dropped. The caller must hold the mutex. +func mergeSnapshot( + e *entry, + entries []FileEntry, + reads *snapshotReads, +) ([]*Entry, []string) { + current := e.snapshot.entries + if e.fingerprints == nil { + e.fingerprints = map[string]FileFingerprint{} + } + m := &snapshotMerge{ + entry: e, + reads: reads, + entries: make([]*Entry, 0, len(entries)), + } + i, j := 0, 0 + for i < len(entries) && j < len(current) { + listed := entries[i].Name + published := current[j].Page.Name + ".md" + switch { + case listed < published: + m.listed(listed, nil) + i++ + case listed > published: + m.unlisted(published, current[j]) + j++ + default: + m.listed(listed, current[j]) + i++ + j++ + } + } + for ; i < len(entries); i++ { + m.listed(entries[i].Name, nil) + } + for ; j < len(current); j++ { + m.unlisted(current[j].Page.Name+".md", current[j]) + } + m.pruneFingerprints(entries, current) + if reads.missing && len(m.entries) == 0 { + return nil, m.dropped + } + return m.entries, m.dropped +} diff --git a/pkg/index/index_changedpages_test.go b/pkg/index/index_changedpages_test.go new file mode 100644 index 0000000..7a97252 --- /dev/null +++ b/pkg/index/index_changedpages_test.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +var _ = Describe("ChangedPagesSince", func() { + var ( + ctx context.Context + fake *storageFake + vaultDir string + key index.Key + ) + + BeforeEach(func() { + ctx = context.Background() + fake = newStorageFake() + vaultDir = GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(vaultDir, "24 Tasks"), 0750)).To(Succeed()) + key = index.NewKey(vaultDir, "24 Tasks") + }) + + // build publishes a key holding Alpha and Beta and returns the index with + // the revision that publication left, which is the revision a derived + // snapshot would have recorded. + build := func() (index.PageIndex, uint64) { + fake.setPages(key, "Alpha", "Beta") + for _, name := range []string{"Alpha", "Beta"} { + Expect(os.WriteFile(pageFilePath(key, name), []byte("x"), 0600)).To(Succeed()) + } + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + return idx, idx.Revision(key) + } + + DescribeTable("bounds the set of pages that moved", + func(mutate func(idx index.PageIndex), expected []string, ok bool) { + idx, revision := build() + mutate(idx) + + names, got := idx.ChangedPagesSince(key, revision) + Expect(got).To(Equal(ok)) + if !ok { + Expect(names).To(BeEmpty()) + return + } + elements := make([]any, 0, len(expected)) + for _, name := range expected { + elements = append(elements, name) + } + Expect(names).To(ConsistOf(elements...)) + }, + Entry("a per-file write mark reports exactly that page with its .md suffix", + func(idx index.PageIndex) { idx.MarkFileDirty(key, "Alpha") }, + []string{"Alpha.md"}, true), + Entry("a per-file write mark given as a wikilink reports the same name", + func(idx index.PageIndex) { idx.MarkFileDirty(key, "[[Alpha]]") }, + []string{"Alpha.md"}, true), + Entry("two per-file marks report both pages", + func(idx index.PageIndex) { + idx.MarkFileDirty(key, "Alpha") + idx.MarkFileDirty(key, "Beta") + }, + []string{"Alpha.md", "Beta.md"}, true), + Entry("a single-file re-read that changed the page reports it", + func(idx index.PageIndex) { + fake.putPage(key, "Alpha", "# Alpha v2") + Expect(idx.RefreshFile(ctx, key, "Alpha.md")).To(Succeed()) + }, + []string{"Alpha.md"}, true), + Entry("a stat-diff that changed one page reports it", + func(idx index.PageIndex) { + fake.putPage(key, "Alpha", "# Alpha v2") + Expect(idx.Refresh(ctx, key)).To(Succeed()) + }, + []string{"Alpha.md"}, true), + Entry("a stat-diff that changed nothing reports nothing", + func(idx index.PageIndex) { + Expect(idx.Refresh(ctx, key)).To(Succeed()) + }, + nil, true), + Entry("an unchanged key reports nothing", + func(index.PageIndex) {}, + nil, true), + Entry("a folder-level mark cannot be bounded", + func(idx index.PageIndex) { idx.MarkDirty(key) }, + nil, false), + Entry("a forced reload cannot be bounded", + func(idx index.PageIndex) { idx.ForceReload() }, + nil, false), + Entry("a revision the key has advanced past its retained window cannot be placed", + func(idx index.PageIndex) { + for i := 0; i < 70; i++ { + idx.MarkFileDirty(key, "Alpha") + } + }, + nil, false), + ) + + It("reports unknown for a key it has never seen", func() { + idx, _ := build() + _, ok := idx.ChangedPagesSince(index.NewKey(vaultDir, "23 Goals"), 0) + Expect(ok).To(BeFalse()) + }) + + It("reports unknown for a revision the key has never reached", func() { + idx, revision := build() + _, ok := idx.ChangedPagesSince(key, revision+5) + Expect(ok).To(BeFalse()) + }) + + It("reports unknown once an unbounded mark falls inside the range", func() { + idx, revision := build() + idx.MarkFileDirty(key, "Alpha") + idx.MarkDirty(key) + names, ok := idx.ChangedPagesSince(key, revision) + Expect(ok).To(BeFalse()) + Expect(names).To(BeEmpty()) + }) + + It("reports unknown after a cold build publishes the whole snapshot", func() { + // A key's first publication is the whole snapshot, so a caller holding the + // revision from before it (0, the value an unknown key reports) cannot be + // told a bounded set of rows moved. + idx, _ := build() + names, ok := idx.ChangedPagesSince(key, 0) + Expect(ok).To(BeFalse()) + Expect(names).To(BeEmpty()) + }) +}) diff --git a/pkg/index/index_file.go b/pkg/index/index_file.go new file mode 100644 index 0000000..887f783 --- /dev/null +++ b/pkg/index/index_file.go @@ -0,0 +1,175 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "context" + "log/slog" + "path/filepath" + "sort" + "strings" + + "github.com/bborbe/errors" +) + +// RefreshFile re-reads one file of the key's folder and publishes a new +// snapshot with the result spliced in. It blocks until a snapshot containing a +// read that started after this call has been published. The file's current +// on-disk state decides the outcome, not the caller's reason for calling: +// present and readable replaces or inserts the page at its filename-ordered +// position, absent or unreadable removes it. The filename is a single plain +// base name including ".md"; a filename that is empty, contains a path +// separator, contains "..", or does not end in ".md" is rejected with an error +// and nothing is read. +func (p *pageIndex) RefreshFile(ctx context.Context, key Key, filename string) error { + key = NewKey(key.VaultPath, key.PagesDir) + if err := validatePageFilename(ctx, filename); err != nil { + return err + } + + p.mu.Lock() + e := p.entryLocked(key) + if !e.hasSnapshot { + p.mu.Unlock() + // The key has nothing published yet, so the folder build includes this + // file. Never join a build that predates this call. + return p.Refresh(ctx, key) + } + p.readSeq++ + seq := p.readSeq + p.mu.Unlock() + + // The read outlives the caller: a cancelled caller must not abort work + // other readers depend on, nor remove a page it never saw. + readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rebuildTimeout) + defer cancel() + read, fingerprint, readErr := p.reader.ReadEntry( + readCtx, + key.VaultPath, + key.PagesDir, + filename, + ) + p.metrics.FileRead(ReadReasonEvent) + + p.mu.Lock() + p.applyFileReadLocked(e, key, filename, seq, read, fingerprint, readErr) + p.mu.Unlock() + + if err := ctx.Err(); err != nil { + return errors.Wrap(ctx, err, "refresh file") + } + return nil +} + +// applyFileReadLocked applies one single-file read to the current snapshot and +// returns the delta the read produced, or nil when a later-started read +// has already been applied. A file that is gone is a delete; a file that is +// present, even one excluded from the snapshot, is a put carrying its +// fingerprint. The caller must hold the mutex. +func (p *pageIndex) applyFileReadLocked( + e *entry, + key Key, + filename string, + seq uint64, + read *Entry, + fingerprint FileFingerprint, + readErr error, +) *storeDelta { + if seq < e.fileReadSeq[filename] { + // A read that started later has already been applied. + return nil + } + previous, known := e.fingerprints[filename] + e.fileReadSeq[filename] = seq + e.snapshot = publish(splicePage(e.snapshot.entries, filename, read)) + e.revision++ + // One file was re-read, so the advance is attributable to exactly it. + e.recordChangeLocked([]string{filename}, true, "") + if read == nil && fingerprint == (FileFingerprint{}) { + // A vanished file leaves no recorded fingerprint behind, so the recorded + // name set stays equal to a later listing's and the unchanged fast path + // is not blocked by a zero-fingerprint tombstone. An excluded-but-present + // file keeps its non-zero fingerprint. + delete(e.fingerprints, filename) + } else { + e.fingerprints[filename] = fingerprint + } + if readErr != nil { + switch { + case fingerprint == (FileFingerprint{}): + // The stat itself failed: a plain deletion, not a broken page. + slog.Debug( + "page file gone", + "filename", filename, + "vault", key.VaultPath, + "pagesDir", key.PagesDir, + "error", readErr, + ) + case !known || previous != fingerprint: + p.warnf( + "unreadable page excluded %q in %q/%q: %v", + filename, + key.VaultPath, + key.PagesDir, + readErr, + ) + } + } + + // A vanished file is a delete, a present file a put. + if fingerprint == (FileFingerprint{}) { + return &storeDelta{deletes: []string{filename}} + } + return &storeDelta{ + puts: []StoredEntry{{Filename: filename, Entry: read, Fingerprint: fingerprint}}, + } +} + +// splicePage returns a new slice holding entries plus the entry for name, +// inserted or replaced at its filename-ordered position; a nil entry removes +// name. The +// ordering key is the page's file name plus ".md", which is the os.ReadDir +// order. pages is never mutated, so a snapshot a reader already holds stays +// intact. +func splicePage(pages []*Entry, name string, page *Entry) []*Entry { + at := sort.Search(len(pages), func(i int) bool { + return pages[i].Page.Name+".md" >= name + }) + found := at < len(pages) && pages[at].Page.Name+".md" == name + + switch { + case found && page != nil: + spliced := make([]*Entry, len(pages)) + copy(spliced, pages) + spliced[at] = page + return spliced + case found: + spliced := make([]*Entry, 0, len(pages)-1) + spliced = append(spliced, pages[:at]...) + return append(spliced, pages[at+1:]...) + case page == nil: + spliced := make([]*Entry, len(pages)) + copy(spliced, pages) + return spliced + default: + spliced := make([]*Entry, 0, len(pages)+1) + spliced = append(spliced, pages[:at]...) + spliced = append(spliced, page) + return append(spliced, pages[at:]...) + } +} + +// validatePageFilename rejects anything that is not a single plain base name +// ending in ".md", so a read can never escape the key's folder. +func validatePageFilename(ctx context.Context, filename string) error { + if filename == "" || + filepath.Base(filename) != filename || + strings.ContainsRune(filename, filepath.Separator) || + strings.Contains(filename, "..") || + !strings.HasSuffix(filename, ".md") { + return errors.Errorf(ctx, "invalid page filename %q", filename) + } + return nil +} diff --git a/pkg/index/index_file_internal_test.go b/pkg/index/index_file_internal_test.go new file mode 100644 index 0000000..bebdd5f --- /dev/null +++ b/pkg/index/index_file_internal_test.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// spliceTestPage builds a page whose ordering key is name plus ".md". +func spliceTestPage(name string) *Entry { + return &Entry{Page: domain.NewPage( + map[string]any{"title": name}, + domain.FileMetadata{Name: name}, + domain.Content("# "+name), + )} +} + +// spliceTestNames returns the ordering key of every page. +func spliceTestNames(pages []*Entry) []string { + names := make([]string, 0, len(pages)) + for _, page := range pages { + names = append(names, page.Page.Name+".md") + } + return names +} + +var _ = ginkgo.Describe("splicePage", func() { + ginkgo.It("inserts into an empty slice", func() { + Expect(spliceTestNames(splicePage(nil, "a.md", spliceTestPage("a")))). + To(Equal([]string{"a.md"})) + }) + + ginkgo.It("inserts first", func() { + pages := []*Entry{spliceTestPage("b"), spliceTestPage("c")} + Expect(spliceTestNames(splicePage(pages, "a.md", spliceTestPage("a")))). + To(Equal([]string{"a.md", "b.md", "c.md"})) + }) + + ginkgo.It("inserts last", func() { + pages := []*Entry{spliceTestPage("a"), spliceTestPage("b")} + Expect(spliceTestNames(splicePage(pages, "c.md", spliceTestPage("c")))). + To(Equal([]string{"a.md", "b.md", "c.md"})) + }) + + ginkgo.It("replaces in place without mutating the input", func() { + original := spliceTestPage("b") + pages := []*Entry{spliceTestPage("a"), original, spliceTestPage("c")} + replacement := spliceTestPage("b") + + spliced := splicePage(pages, "b.md", replacement) + + Expect(spliceTestNames(spliced)).To(Equal([]string{"a.md", "b.md", "c.md"})) + Expect(spliced[1]).To(BeIdenticalTo(replacement)) + Expect(pages[1]).To(BeIdenticalTo(original)) + }) + + ginkgo.It("removes a present page without mutating the input", func() { + pages := []*Entry{spliceTestPage("a"), spliceTestPage("b"), spliceTestPage("c")} + + spliced := splicePage(pages, "b.md", nil) + + Expect(spliceTestNames(spliced)).To(Equal([]string{"a.md", "c.md"})) + Expect(spliceTestNames(pages)).To(Equal([]string{"a.md", "b.md", "c.md"})) + }) + + ginkgo.It("returns the pages unchanged when removing an absent one", func() { + first := spliceTestPage("a") + second := spliceTestPage("b") + pages := []*Entry{first, second} + + spliced := splicePage(pages, "z.md", nil) + + Expect(spliceTestNames(spliced)).To(Equal([]string{"a.md", "b.md"})) + Expect(spliced[0]).To(BeIdenticalTo(first)) + Expect(spliced[1]).To(BeIdenticalTo(second)) + }) +}) diff --git a/pkg/index/index_file_test.go b/pkg/index/index_file_test.go new file mode 100644 index 0000000..9b0b7e7 --- /dev/null +++ b/pkg/index/index_file_test.go @@ -0,0 +1,371 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + stderrors "errors" + "time" + + "github.com/bborbe/vault-cli/pkg/domain" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +var _ = Describe("RefreshFile", func() { + var ( + ctx context.Context + fake *storageFake + key index.Key + idx index.PageIndex + ) + + // list reads the key and returns the published snapshot. + list := func() []*domain.Page { + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + return pages + } + + // readFile refreshes one file in a goroutine and reports the error. + readFile := func(readCtx context.Context, filename string) <-chan error { + done := make(chan error, 1) + go func() { + defer GinkgoRecover() + done <- idx.RefreshFile(readCtx, key, filename) + }() + return done + } + + BeforeEach(func() { + ctx = context.Background() + fake = newStorageFake() + key = index.NewKey("/vault-a", "24 Tasks") + }) + + Describe("splicing one file into a new snapshot", func() { + It("AC2(a) inserts a created file at its filename-ordered position", func() { + fake.setPages(key, "alpha", "gamma") + idx = newIndex(fake) + Expect(titles(list())).To(Equal([]string{"alpha", "gamma"})) + + fake.putPage(key, "beta", "# beta") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + Expect(titles(list())).To(Equal([]string{"alpha", "beta", "gamma"})) + // The folder is neither listed nor otherwise read. + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + Expect(fake.reader.ReadEntryCallCount()).To(Equal(3)) + }) + + It("AC2(b) replaces the changed page and shares every other pointer", func() { + fake.setPages(key, "alpha", "beta", "gamma") + idx = newIndex(fake) + before := list() + Expect(titles(before)).To(Equal([]string{"alpha", "beta", "gamma"})) + + fake.putPage(key, "beta", "# beta changed") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + after := list() + Expect(titles(after)).To(Equal([]string{"alpha", "beta", "gamma"})) + Expect(string(after[1].Content)).To(Equal("# beta changed")) + // A splice, not a rebuild: the untouched pages are the same values. + Expect(after[0]).To(BeIdenticalTo(before[0])) + Expect(after[2]).To(BeIdenticalTo(before[2])) + Expect(after[1]).NotTo(BeIdenticalTo(before[1])) + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + }) + + It("AC2(c) removes a deleted file", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + Expect(titles(list())).To(Equal([]string{"alpha", "beta"})) + + fake.dropPage(key, "beta") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + Expect(titles(list())).To(Equal([]string{"alpha"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + }) + + It("AC2(d) removes a page whose file is gone, whatever the caller meant", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + // A "modified" event for a file that no longer exists on disk: the + // file's current state decides, not the event type. + fake.dropPage(key, "beta") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + Expect(titles(list())).To(Equal([]string{"alpha"})) + }) + + It("AC2(e) keeps a page whose file is still there", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + // A "deleted" event for a file that still exists on disk. + fake.putPage(key, "beta", "# beta still here") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + pages := list() + Expect(titles(pages)).To(Equal([]string{"alpha", "beta"})) + Expect(string(pages[1].Content)).To(Equal("# beta still here")) + }) + + It("AC2(f) leaves a previously obtained slice and its pointers untouched", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + before := list() + captured := append([]*domain.Page(nil), before...) + + fake.putPage(key, "beta", "# beta changed") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + Expect(before).To(HaveLen(2)) + Expect(before[0]).To(BeIdenticalTo(captured[0])) + Expect(before[1]).To(BeIdenticalTo(captured[1])) + Expect(string(before[1].Content)).To(Equal("# beta")) + }) + + It("rejects a filename that is not a single plain .md base name", func() { + idx = newIndex(fake) + for _, filename := range []string{"", "a/b.md", "../x.md", "x.txt"} { + Expect(idx.RefreshFile(ctx, key, filename)).To(HaveOccurred(), filename) + } + Expect(fake.reader.ReadEntryCallCount()).To(Equal(0)) + Expect(fake.lister.ListFilesCallCount()).To(Equal(0)) + }) + + It("builds the folder when the key has no snapshot yet", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + Expect(titles(list())).To(Equal([]string{"alpha", "beta"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + Expect(fake.reader.ReadEntryCallCount()).To(Equal(2)) + }) + + It("keeps a dirty mark that an unrelated RefreshFile does not clear", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + fake.putPage(key, "gamma", "# gamma") + idx.MarkDirty(key) + + Expect(idx.RefreshFile(ctx, key, "alpha.md")).To(Succeed()) + + // The mark still forces the folder build, so the write is visible. + Expect(titles(list())).To(Equal([]string{"alpha", "beta", "gamma"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + }) + + It("counts reads by reason", func() { + buildBefore := index.FilesReadTotal("build") + eventBefore := index.FilesReadTotal("event") + + fake.setPages(key, "alpha", "beta", "gamma") + idx = newIndex(fake) + list() + Expect(index.FilesReadTotal("build") - buildBefore).To(Equal(3.0)) + + fake.putPage(key, "beta", "# beta v2") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + Expect(index.FilesReadTotal("event") - eventBefore).To(Equal(1.0)) + }) + }) + + Describe("ordering concurrent reads", func() { + It("AC6(a) publishes the later-started read of a file", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + entered, release := fake.blockReads("beta.md") + first := readFile(ctx, "beta.md") + Eventually(entered).Should(Receive(Equal("beta.md"))) + + // The second read starts later and captures the new content. + fake.putPage(key, "beta", "# beta v2") + second := readFile(ctx, "beta.md") + Eventually(entered).Should(Receive(Equal("beta.md"))) + + // Release them so the earlier-started read finishes last. + release("beta.md", 1) + Expect(<-second).To(BeNil()) + release("beta.md", 0) + Expect(<-first).To(BeNil()) + + pages := list() + Expect(string(pages[1].Content)).To(Equal("# beta v2")) + }) + + It("AC6(b) never drops one file's read for another's", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + entered, release := fake.blockReads("alpha.md", "beta.md") + + fake.putPage(key, "alpha", "# alpha v2") + alpha := readFile(ctx, "alpha.md") + Eventually(entered).Should(Receive(Equal("alpha.md"))) + + fake.putPage(key, "beta", "# beta v2") + beta := readFile(ctx, "beta.md") + Eventually(entered).Should(Receive(Equal("beta.md"))) + + // Either release order keeps both results. + release("beta.md", 0) + Expect(<-beta).To(BeNil()) + release("alpha.md", 0) + Expect(<-alpha).To(BeNil()) + + pages := list() + Expect(titles(pages)).To(Equal([]string{"alpha", "beta"})) + Expect(string(pages[0].Content)).To(Equal("# alpha v2")) + Expect(string(pages[1].Content)).To(Equal("# beta v2")) + }) + + It("AC6(b) keeps a file created while a folder build was listing", func() { + fake.setPages(key, "alpha") + idx = newIndex(fake) + list() + + entered, release := fake.block() + fake.putPage(key, "alpha", "# alpha v2") + buildDone := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + idx.MarkDirty(key) + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + buildDone <- pages + }() + Eventually(entered).Should(Receive()) + + // The event creates a file the build's listing never saw. + fake.putPage(key, "beta", "# beta") + Expect(idx.RefreshFile(ctx, key, "beta.md")).To(Succeed()) + + release() + Expect(titles(<-buildDone)).To(Equal([]string{"alpha", "beta"})) + Expect(titles(list())).To(Equal([]string{"alpha", "beta"})) + }) + + It("AC6(c) serves the previous snapshot while a read is blocked", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + before := list() + + entered, release := fake.blockReads("beta.md") + done := readFile(ctx, "beta.md") + Eventually(entered).Should(Receive(Equal("beta.md"))) + + start := time.Now() + pages := list() + Expect(time.Since(start)).To(BeNumerically("<", 100*time.Millisecond)) + Expect(pages[0]).To(BeIdenticalTo(before[0])) + Expect(pages[1]).To(BeIdenticalTo(before[1])) + + release("beta.md", 0) + Expect(<-done).To(BeNil()) + }) + + It("AC6/behavior 6 lets an event beat a folder build that started earlier", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + entered, release := fake.blockReads("alpha.md") + // The stat-diff re-reads alpha only because its fingerprint changed. + fake.putPage(key, "alpha", "# alpha v2") + idx.MarkDirty(key) + buildDone := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + buildDone <- pages + }() + // The stat-diff lists, then blocks reading alpha.md. + Eventually(entered).Should(Receive(Equal("alpha.md"))) + + fake.putPage(key, "alpha", "# alpha v3") + event := readFile(ctx, "alpha.md") + Eventually(entered).Should(Receive(Equal("alpha.md"))) + release("alpha.md", 1) + Expect(<-event).To(BeNil()) + + // The build's older read is discarded, so even the caller that + // joined the build sees the event's page. + release("alpha.md", 0) + Expect(string((<-buildDone)[0].Content)).To(Equal("# alpha v3")) + Expect(string(list()[0].Content)).To(Equal("# alpha v3")) + }) + + It("AC6/behavior 6 lets a folder build beat an event that started earlier", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + entered, release := fake.blockReads("alpha.md") + event := readFile(ctx, "alpha.md") + Eventually(entered).Should(Receive(Equal("alpha.md"))) + + fake.putPage(key, "alpha", "# alpha v2") + idx.MarkDirty(key) + buildDone := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + buildDone <- pages + }() + Eventually(entered).Should(Receive(Equal("alpha.md"))) + + release("alpha.md", 1) + Expect(string((<-buildDone)[0].Content)).To(Equal("# alpha v2")) + + // The event's older read is discarded. + release("alpha.md", 0) + Expect(<-event).To(BeNil()) + Expect(string(list()[0].Content)).To(Equal("# alpha v2")) + }) + }) + + Describe("a cancelled caller", func() { + It("still applies the read and reports the cancellation", func() { + fake.setPages(key, "alpha", "beta") + idx = newIndex(fake) + list() + + entered, release := fake.blockReads("alpha.md") + fake.putPage(key, "alpha", "# alpha v2") + + readCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := readFile(readCtx, "alpha.md") + Eventually(entered).Should(Receive(Equal("alpha.md"))) + + cancel() + release("alpha.md", 0) + + err := <-done + Expect(err).To(HaveOccurred()) + Expect(stderrors.Is(err, context.Canceled)).To(BeTrue()) + // The read is applied, not dropped. + Expect(string(list()[0].Content)).To(Equal("# alpha v2")) + }) + }) +}) diff --git a/pkg/index/index_revision_test.go b/pkg/index/index_revision_test.go new file mode 100644 index 0000000..e83fc4b --- /dev/null +++ b/pkg/index/index_revision_test.go @@ -0,0 +1,158 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +var _ = Describe("Revision", func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + It("reports 0 for a key the index has never seen", func() { + fake := newStorageFake() + idx := newIndex(fake) + + Expect(idx.Revision(index.NewKey("/vault-a", "24 Tasks"))).To(Equal(uint64(0))) + }) + + It("advances when a cold build publishes a snapshot", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake := newStorageFake() + fake.setPages(key, "one", "two") + idx := newIndex(fake) + + Expect(idx.Revision(key)).To(Equal(uint64(0))) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(idx.Revision(key)).To(Equal(uint64(1))) + + // A warm read publishes nothing and leaves the revision alone. + _, err = idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(idx.Revision(key)).To(Equal(uint64(1))) + }) + + It("advances on a folder-level mark", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake := newStorageFake() + fake.setPages(key, "one") + idx := newIndex(fake) + + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + before := idx.Revision(key) + + idx.MarkDirty(key) + Expect(idx.Revision(key)).To(BeNumerically(">", before)) + + // An unknown key is still ignored, revision included. + idx.MarkDirty(index.NewKey("/vault-z", "24 Tasks")) + Expect(idx.Revision(index.NewKey("/vault-z", "24 Tasks"))).To(Equal(uint64(0))) + }) + + It("advances on a file-level mark and on the folder fallback", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake := newStorageFake() + fake.setPages(key, "one", "two") + idx := newIndex(fake) + + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + before := idx.Revision(key) + + // The fake key has no on-disk file, so the exactness rule fails and the + // mark widens to the folder — the fallback path. + idx.MarkFileDirty(key, "one") + Expect(idx.Revision(key)).To(BeNumerically(">", before)) + }) + + It("advances on a forced reload", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake := newStorageFake() + fake.setPages(key, "one") + idx := newIndex(fake) + + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + before := idx.Revision(key) + + idx.ForceReload() + Expect(idx.Revision(key)).To(BeNumerically(">", before)) + }) + + It("advances on a file-level mark when the file exists on disk", func() { + vaultDir := newStatDiffVault() + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + + pages, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Alpha", "Beta"})) + before := ri.idx.Revision(key) + + ri.idx.MarkFileDirty(key, "Alpha") + Expect(ri.idx.Revision(key)).To(BeNumerically(">", before)) + }) + + It("advances when a rescan publishes a changed snapshot", func() { + vaultDir := newStatDiffVault() + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + before := ri.idx.Revision(key) + + settle() + writeFixtureFile(vaultDir, "Alpha.md", "---\ntitle: Alpha\n---\n# Alpha changed\n") + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + Expect(ri.idx.Revision(key)).To(BeNumerically(">", before)) + }) + + It("advances when a file event publishes a new page", func() { + vaultDir := newStatDiffVault() + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + before := ri.idx.Revision(key) + + writeFixtureFile(vaultDir, "Gamma.md", "---\ntitle: Gamma\n---\n# Gamma\n") + Expect(ri.idx.RefreshFile(ctx, key, "Gamma.md")).To(BeNil()) + Expect(ri.idx.Revision(key)).To(BeNumerically(">", before)) + + pages, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Alpha", "Beta", "Gamma"})) + }) + + It("does not advance when a stat-diff finds nothing changed", func() { + vaultDir := newStatDiffVault() + ri := newRealIndex() + key := index.NewKey(vaultDir, equivalenceFolder) + + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + before := ri.idx.Revision(key) + ri.reader.reset() + ri.lister.reset() + + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + Expect(ri.lister.Count()).To(Equal(1)) + Expect(ri.reader.Count()).To(Equal(0)) + Expect(ri.idx.Revision(key)).To(Equal(before)) + }) +}) diff --git a/pkg/index/index_statdiff_test.go b/pkg/index/index_statdiff_test.go new file mode 100644 index 0000000..eae6dd1 --- /dev/null +++ b/pkg/index/index_statdiff_test.go @@ -0,0 +1,359 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// recordingReader counts and names the single-file reads the index makes. +type recordingReader struct { + inner index.PageReader + + mu sync.Mutex + count int + names []string +} + +func (r *recordingReader) ReadEntry( + ctx context.Context, + vaultPath string, + pagesDir string, + filename string, +) (*index.Entry, index.FileFingerprint, error) { + r.mu.Lock() + r.count++ + r.names = append(r.names, filename) + r.mu.Unlock() + return r.inner.ReadEntry(ctx, vaultPath, pagesDir, filename) +} + +func (r *recordingReader) ReadPage( + ctx context.Context, + vaultPath string, + pagesDir string, + name string, +) (*domain.Page, error) { + return r.inner.ReadPage(ctx, vaultPath, pagesDir, name) +} + +func (r *recordingReader) Count() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.count +} + +func (r *recordingReader) Names() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.names...) +} + +func (r *recordingReader) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.count = 0 + r.names = nil +} + +// recordingLister counts the folder listings the index makes. +type recordingLister struct { + inner index.DirectoryLister + + mu sync.Mutex + count int +} + +func (l *recordingLister) ListFiles( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]index.FileEntry, error) { + l.mu.Lock() + l.count++ + l.mu.Unlock() + return l.inner.ListFiles(ctx, vaultPath, pagesDir) +} + +func (l *recordingLister) Count() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.count +} + +func (l *recordingLister) reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.count = 0 +} + +// warnSink captures the index's per-file warnings. +type warnSink struct { + mu sync.Mutex + lines []string +} + +func (w *warnSink) warnf(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() + w.lines = append(w.lines, fmt.Sprintf(format, args...)) +} + +// naming returns the captured warning lines that mention substr. +func (w *warnSink) naming(substr string) []string { + w.mu.Lock() + defer w.mu.Unlock() + matched := make([]string, 0, len(w.lines)) + for _, line := range w.lines { + if strings.Contains(line, substr) { + matched = append(matched, line) + } + } + return matched +} + +// realIndex is the index under test over the production seams, with the +// seams' calls recorded and the warnings captured. +type realIndex struct { + reader *recordingReader + lister *recordingLister + warns *warnSink + clock libtime.CurrentDateTime + idx index.PageIndex +} + +func newRealIndex() *realIndex { + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + warns := &warnSink{} + clock := libtime.NewCurrentDateTime() + return &realIndex{ + reader: reader, + lister: lister, + warns: warns, + clock: clock, + idx: index.NewPageIndexWithWarnf( + reader, lister, clock, fastWaiter(), warns.warnf, + ), + } +} + +// pageFile is the on-disk path of one fixture file. +func pageFile(vaultDir, name string) string { + return filepath.Join(vaultDir, equivalenceFolder, name) +} + +// settle waits long enough for a fresh write to move a file's status-change +// time past the granularity the recorded fingerprint can see. +func settle() { + time.Sleep(20 * time.Millisecond) +} + +// newStatDiffVault builds a real vault folder holding Alpha, Beta and an +// unparsable BadYaml. +func newStatDiffVault() string { + vaultDir := GinkgoT().TempDir() + Expect(os.MkdirAll( + filepath.Join(vaultDir, equivalenceFolder), 0750, + )).To(Succeed()) + writeFixtureFile(vaultDir, "Alpha.md", "---\ntitle: Alpha\n---\n# Alpha A\n") + writeFixtureFile(vaultDir, "Beta.md", "---\ntitle: Beta\n---\n# Beta\n") + writeFixtureFile(vaultDir, "BadYaml.md", "---\ntitle: [unclosed\n---\n# Bad\n") + return vaultDir +} + +var _ = Describe("Stat-diff rescan", func() { + var ( + ctx context.Context + vaultDir string + ri *realIndex + key index.Key + ) + + BeforeEach(func() { + ctx = context.Background() + vaultDir = newStatDiffVault() + ri = newRealIndex() + key = index.NewKey(vaultDir, equivalenceFolder) + }) + + It("AC4(a) reads nothing and republishes nothing when nothing changed", func() { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + ri.clock.SetNow(libtime.DateTime(start)) + + before, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(before)).To(Equal([]string{"Alpha", "Beta"})) + Expect(ri.lister.Count()).To(Equal(1)) + // The unparsable file was warned about once, by the cold build. + Expect(ri.warns.naming("BadYaml.md")).To(HaveLen(1)) + + ri.reader.reset() + ri.lister.reset() + + rescanCtx, cancel := context.WithCancel(ctx) + rescanDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + rescanDone <- ri.idx.Rescan(rescanCtx) + }() + // Let the loop record its start time before the clock moves. + Consistently(ri.lister.Count, "100ms").Should(Equal(0)) + + ri.clock.SetNow(libtime.DateTime(start.Add(time.Minute))) + Eventually(ri.lister.Count, "2s").Should(Equal(1)) + Consistently(ri.reader.Count, "100ms").Should(Equal(0)) + + after, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(len(after)).To(Equal(len(before))) + Expect(&after[0]).To(BeIdenticalTo(&before[0])) + Expect(ri.lister.Count()).To(Equal(1)) + Expect(ri.warns.naming("BadYaml.md")).To(HaveLen(1)) + + // A second rescan of the still-unchanged folder reads and warns nothing + // new either. + ri.clock.SetNow(libtime.DateTime(start.Add(2 * time.Minute))) + Eventually(ri.lister.Count, "2s").Should(Equal(2)) + Consistently(ri.reader.Count, "100ms").Should(Equal(0)) + Expect(ri.lister.Count()).To(Equal(2)) + Expect(ri.warns.naming("BadYaml.md")).To(HaveLen(1)) + + cancel() + Eventually(rescanDone, "2s").Should(Receive(BeNil())) + }) + + It("AC4(b) re-reads exactly the modified and the added file", func() { + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + ri.reader.reset() + ri.lister.reset() + + settle() + writeFixtureFile(vaultDir, "Alpha.md", "---\ntitle: Alpha\n---\n# Alpha changed\n") + writeFixtureFile(vaultDir, "Gamma.md", "---\ntitle: Gamma\n---\n# Gamma\n") + Expect(os.Remove(pageFile(vaultDir, "Beta.md"))).To(Succeed()) + + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + Expect(ri.lister.Count()).To(Equal(1)) + Expect(ri.reader.Names()).To(ConsistOf("Alpha.md", "Gamma.md")) + + pages, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Alpha", "Gamma"})) + }) + + It("AC4(c) detects a same-size rewrite whose modification time was restored", func() { + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + ri.reader.reset() + ri.lister.reset() + + alpha := pageFile(vaultDir, "Alpha.md") + info, err := os.Stat(alpha) + Expect(err).To(BeNil()) + + settle() + // Same byte count, different content. + writeFixtureFile(vaultDir, "Alpha.md", "---\ntitle: Alpha\n---\n# Alpha B\n") + Expect(os.Chtimes(alpha, info.ModTime(), info.ModTime())).To(Succeed()) + + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + Expect(ri.reader.Names()).To(Equal([]string{"Alpha.md"})) + + page, err := ri.idx.ReadPage(ctx, vaultDir, equivalenceFolder, "Alpha") + Expect(err).To(BeNil()) + Expect(string(page.Content)).To(ContainSubstring("# Alpha B")) + }) + + It("AC5(ii)/8 lists once and re-reads only what changed after a folder mark", func() { + _, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + ri.reader.reset() + ri.lister.reset() + + // Nothing changed: the mark costs one listing and no read. + ri.idx.MarkDirty(key) + pages, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Alpha", "Beta"})) + Expect(ri.lister.Count()).To(Equal(1)) + Expect(ri.reader.Count()).To(Equal(0)) + + ri.reader.reset() + ri.lister.reset() + settle() + writeFixtureFile(vaultDir, "Beta.md", "---\ntitle: Beta\n---\n# Beta changed\n") + ri.idx.MarkDirty(key) + pages, err = ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Alpha", "Beta"})) + Expect(ri.lister.Count()).To(Equal(1)) + Expect(ri.reader.Names()).To(Equal([]string{"Beta.md"})) + }) + + It("AC4/failure 9 keeps the snapshot and the mark pending when the listing fails", func() { + fake := newStorageFake() + fakeKey := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(fakeKey, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + idx.MarkDirty(fakeKey) + fake.setFailure(fmt.Errorf("listing boom")) + + Expect(idx.Refresh(ctx, fakeKey)).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + + // The mark is still pending, so the next read lists again. + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"old"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(3)) + + fake.setFailure(nil) + fake.setPages(fakeKey, "new") + pages, err = idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(4)) + }) + + It("empties and refills the snapshot when the folder is deleted and recreated", func() { + pages, err := ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(pages).NotTo(BeEmpty()) + + Expect(os.RemoveAll(filepath.Join(vaultDir, equivalenceFolder))).To(Succeed()) + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + pages, err = ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(pages).To(BeNil()) + + Expect(os.MkdirAll( + filepath.Join(vaultDir, equivalenceFolder), 0750, + )).To(Succeed()) + writeFixtureFile(vaultDir, "Delta.md", "---\ntitle: Delta\n---\n# Delta\n") + Expect(ri.idx.Refresh(ctx, key)).To(BeNil()) + pages, err = ri.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(BeNil()) + Expect(indexedNames(pages)).To(Equal([]string{"Delta"})) + }) +}) diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go new file mode 100644 index 0000000..8522cea --- /dev/null +++ b/pkg/index/index_test.go @@ -0,0 +1,788 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + stderrors "errors" + "sort" + "strings" + "sync" + "time" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/mocks" + "github.com/bborbe/vault-cli/pkg/index" +) + +// storageFake is a reader and lister pair whose per-key content and blocking +// behaviour the tests control. The counterfeiter fakes underneath record every +// call. +type storageFake struct { + reader *mocks.IndexPageReader + lister *mocks.IndexDirectoryLister + + mu sync.Mutex + pages map[index.Key][]*domain.Page + versions map[index.Key]map[string]int64 + gate chan struct{} + entered chan struct{} + failure error + + // readMu guards the per-read gate: while readBlocked is set, a read of a + // gated name registers its own gate and waits on it until the test opens it. + readMu sync.Mutex + readBlocked bool + readGated map[string]bool + readGates map[string][]chan struct{} + readEntered chan string +} + +func newStorageFake() *storageFake { + fake := &storageFake{ + reader: &mocks.IndexPageReader{}, + lister: &mocks.IndexDirectoryLister{}, + pages: map[index.Key][]*domain.Page{}, + versions: map[index.Key]map[string]int64{}, + } + fake.reader.ReadEntryStub = fake.readEntry + fake.reader.ReadPageStub = fake.readFullPage + fake.lister.ListFilesStub = fake.listFiles + return fake +} + +// fingerprint derives a file's fingerprint from its version counter, so a +// content change always changes the fingerprint. +func fingerprint(version int64) index.FileFingerprint { + return index.FileFingerprint{ + Size: version, + ModTime: time.Unix(version, 0).UTC(), + } +} + +func (f *storageFake) listFiles( + _ context.Context, + vaultPath string, + pagesDir string, +) ([]index.FileEntry, error) { + key := index.NewKey(vaultPath, pagesDir) + + f.mu.Lock() + gate, entered := f.gate, f.entered + entries := make([]index.FileEntry, 0, len(f.pages[key])) + for _, page := range f.pages[key] { + name := page.Name + entries = append(entries, index.FileEntry{ + Name: name + ".md", + Fingerprint: fingerprint(f.versions[key][name]), + }) + } + f.mu.Unlock() + + if entered != nil { + select { + case entered <- struct{}{}: + default: + } + } + if gate != nil { + <-gate + } + + f.mu.Lock() + defer f.mu.Unlock() + if f.failure != nil { + return nil, f.failure + } + // The production lister reads os.ReadDir, which is filename ascending. + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries, nil +} + +// readFullPage reads the file's full page by its bare name. +func (f *storageFake) readFullPage( + ctx context.Context, + vaultPath string, + pagesDir string, + name string, +) (*domain.Page, error) { + entry, _, err := f.readEntry(ctx, vaultPath, pagesDir, name+".md") + if err != nil { + return nil, err + } + return entry.Page, nil +} + +// readEntry reads the file's current page, captured before any gate wait so a +// blocked read still reports the content the caller would have seen. +func (f *storageFake) readEntry( + _ context.Context, + vaultPath string, + pagesDir string, + filename string, +) (*index.Entry, index.FileFingerprint, error) { + key := index.NewKey(vaultPath, pagesDir) + name := strings.TrimSuffix(filename, ".md") + + f.mu.Lock() + var page *domain.Page + for _, candidate := range f.pages[key] { + if candidate.Name == name { + page = candidate + break + } + } + version := f.versions[key][name] + f.mu.Unlock() + + if gate, entered := f.enterRead(filename); gate != nil { + select { + case entered <- filename: + default: + } + <-gate + } + + if page == nil { + return nil, index.FileFingerprint{}, stderrors.New("page not found: " + name) + } + return &index.Entry{Page: page, OpenQuestions: []domain.OpenQuestion{}}, fingerprint(version), nil +} + +// enterRead registers a read of filename against the active read gate, if the +// filename is gated, and returns the gate the read must wait on. +func (f *storageFake) enterRead(filename string) (chan struct{}, chan string) { + f.readMu.Lock() + defer f.readMu.Unlock() + if !f.readBlocked || !f.readGated[filename] { + return nil, nil + } + gate := make(chan struct{}) + f.readGates[filename] = append(f.readGates[filename], gate) + return gate, f.readEntered +} + +// blockReads makes every read of the named files wait on its own gate until the +// test opens it. The returned channel reports the filename of each read that +// entered, and the release func opens the nth gate registered for a filename, +// so the test decides the order the reads finish in. +func (f *storageFake) blockReads(names ...string) (<-chan string, func(name string, n int)) { + entered := make(chan string, 64) + gated := map[string]bool{} + for _, name := range names { + gated[name] = true + } + f.readMu.Lock() + f.readBlocked = true + f.readGated = gated + f.readGates = map[string][]chan struct{}{} + f.readEntered = entered + f.readMu.Unlock() + return entered, func(name string, n int) { + f.readMu.Lock() + gate := f.readGates[name][n] + f.readMu.Unlock() + close(gate) + } +} + +// block makes every listing wait until the returned release func runs. The +// returned channel reports each call that entered the stub. +func (f *storageFake) block() (<-chan struct{}, func()) { + gate := make(chan struct{}) + entered := make(chan struct{}, 64) + f.mu.Lock() + f.gate = gate + f.entered = entered + f.mu.Unlock() + return entered, func() { + f.mu.Lock() + f.gate = nil + f.entered = nil + f.mu.Unlock() + close(gate) + } +} + +func (f *storageFake) setPages(key index.Key, titles ...string) { + key = index.NewKey(key.VaultPath, key.PagesDir) + pages := make([]*domain.Page, 0, len(titles)) + for _, title := range titles { + pages = append(pages, newPage(title)) + } + f.mu.Lock() + defer f.mu.Unlock() + previous := map[string]*domain.Page{} + for _, page := range f.pages[key] { + previous[page.Name] = page + } + versions := f.versions[key] + if versions == nil { + versions = map[string]int64{} + f.versions[key] = versions + } + for _, page := range pages { + name := page.Name + if old, ok := previous[name]; !ok || old.Content != page.Content { + versions[name]++ + } + } + f.pages[key] = pages +} + +// putPage writes one file into the key's folder, replacing any file of the same +// name, and bumps its version so its fingerprint changes. Content is given +// separately from the name, so a test can modify a file without renaming it. +func (f *storageFake) putPage(key index.Key, name, content string) { + f.mu.Lock() + defer f.mu.Unlock() + key = index.NewKey(key.VaultPath, key.PagesDir) + replaced := false + pages := make([]*domain.Page, 0, len(f.pages[key])+1) + for _, page := range f.pages[key] { + if page.Name == name { + pages = append(pages, newPageContent(name, content)) + replaced = true + continue + } + pages = append(pages, page) + } + if !replaced { + pages = append(pages, newPageContent(name, content)) + } + f.pages[key] = pages + if f.versions[key] == nil { + f.versions[key] = map[string]int64{} + } + f.versions[key][name]++ +} + +// dropPage removes one file from the key's folder. +func (f *storageFake) dropPage(key index.Key, name string) { + f.mu.Lock() + defer f.mu.Unlock() + key = index.NewKey(key.VaultPath, key.PagesDir) + pages := make([]*domain.Page, 0, len(f.pages[key])) + for _, page := range f.pages[key] { + if page.Name == name { + continue + } + pages = append(pages, page) + } + f.pages[key] = pages +} + +func (f *storageFake) setFailure(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.failure = err +} + +func newPage(title string) *domain.Page { + return newPageContent(title, "# "+title) +} + +// newPageContent builds a page whose file name and content differ, so a test +// can change a file's content without changing its name. +func newPageContent(name, content string) *domain.Page { + return domain.NewPage( + map[string]any{"title": name}, + domain.FileMetadata{Name: name, FilePath: "/vault/" + name + ".md"}, + domain.Content(content), + ) +} + +func titles(pages []*domain.Page) []string { + result := make([]string, 0, len(pages)) + for _, page := range pages { + result = append(result, page.GetString("title")) + } + return result +} + +func callCountFor(fake *storageFake, key index.Key) int { + want := index.NewKey(key.VaultPath, key.PagesDir) + count := 0 + for i := 0; i < fake.lister.ListFilesCallCount(); i++ { + _, vaultPath, pagesDir := fake.lister.ListFilesArgsForCall(i) + if index.NewKey(vaultPath, pagesDir) == want { + count++ + } + } + return count +} + +func fastWaiter() libtime.WaiterDuration { + return libtime.WaiterDurationFunc(func(ctx context.Context, duration libtime.Duration) error { + timer := time.NewTimer(time.Millisecond) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } + }) +} + +func newIndex(fake *storageFake) index.PageIndex { + return index.NewPageIndex( + fake.reader, fake.lister, libtime.NewCurrentDateTime(), fastWaiter(), index.TestMetrics, + ) +} + +var _ = Describe("PageIndex", func() { + var ( + ctx context.Context + fake *storageFake + ) + + BeforeEach(func() { + ctx = context.Background() + fake = newStorageFake() + }) + + Describe("ListPages", func() { + It("serves repeat reads of a clean key without touching storage", func() { + fake.setPages(index.NewKey("/vault-a", "24 Tasks"), "one", "two") + idx := newIndex(fake) + + first, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(first)).To(Equal([]string{"one", "two"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + + for i := 0; i < 5; i++ { + again, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(again[0]).To(BeIdenticalTo(first[0])) + } + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + }) + + }) + + Describe("Cold read", func() { + It("shares a single build across concurrent cold reads", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "one") + idx := newIndex(fake) + entered, release := fake.block() + + results := make([][]*domain.Page, 8) + errs := make([]error, 8) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + defer GinkgoRecover() + results[i], errs[i] = idx.ListPages(ctx, "/vault-a", "24 Tasks") + }(i) + } + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + Expect(idx.Build(ctx, []index.Key{key})).To(BeNil()) + }() + + Eventually(entered).Should(Receive()) + Consistently(fake.lister.ListFilesCallCount, "200ms").Should(Equal(1)) + + release() + wg.Wait() + + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + for i := range results { + Expect(errs[i]).To(BeNil()) + Expect(titles(results[i])).To(Equal([]string{"one"})) + } + }) + }) + + Describe("Refresh", func() { + It("rebuilds only the requested key", func() { + aTasks := index.NewKey("/vault-a", "24 Tasks") + aGoals := index.NewKey("/vault-a", "25 Goals") + bTasks := index.NewKey("/vault-b", "24 Tasks") + bGoals := index.NewKey("/vault-b", "25 Goals") + all := []index.Key{aTasks, aGoals, bTasks, bGoals} + for _, key := range all { + fake.setPages(key, "old") + } + idx := newIndex(fake) + Expect(idx.Build(ctx, all)).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(4)) + + fake.setPages(aTasks, "new") + Expect(idx.Refresh(ctx, aTasks)).To(BeNil()) + + Expect(callCountFor(fake, aTasks)).To(Equal(2)) + for _, key := range []index.Key{aGoals, bTasks, bGoals} { + Expect(callCountFor(fake, key)).To(Equal(1)) + } + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + }) + + It("keeps serving the previous snapshot while a rebuild is blocked", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + entered, release := fake.block() + fake.setPages(key, "new") + refreshDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + refreshDone <- idx.Refresh(ctx, key) + }() + Eventually(entered).Should(Receive()) + + start := time.Now() + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"old"})) + Expect(time.Since(start)).To(BeNumerically("<", 100*time.Millisecond)) + + release() + Expect(<-refreshDone).To(BeNil()) + pages, err = idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + }) + + It("coalesces a burst of refreshes into one follow-up", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + entered, release := fake.block() + + first := make(chan error, 1) + go func() { + defer GinkgoRecover() + first <- idx.Refresh(ctx, key) + }() + Eventually(entered).Should(Receive()) + + rest := make([]chan error, 3) + for i := range rest { + rest[i] = make(chan error, 1) + done := rest[i] + go func() { + defer GinkgoRecover() + done <- idx.Refresh(ctx, key) + }() + } + Consistently(fake.lister.ListFilesCallCount, "200ms").Should(Equal(1)) + + fake.setPages(key, "new") + release() + + Expect(<-first).To(BeNil()) + for _, done := range rest { + Expect(<-done).To(BeNil()) + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + } + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + }) + }) + + Describe("dirty marks", func() { + It("rebuilds once on the next read and serves the new content", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + fake.setPages(key, "new") + idx.MarkDirty(key) + Expect(fake.lister.ListFilesCallCount()).To(Equal(1)) + + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + }) + + It("shares one rebuild between two concurrent reads after one mark", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + fake.setPages(key, "new") + idx.MarkDirty(key) + entered, release := fake.block() + + var wg sync.WaitGroup + results := make([][]*domain.Page, 2) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + defer GinkgoRecover() + results[i], _ = idx.ListPages(ctx, "/vault-a", "24 Tasks") + }(i) + } + Eventually(entered).Should(Receive()) + release() + wg.Wait() + + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + for i := range results { + Expect(titles(results[i])).To(Equal([]string{"new"})) + } + }) + + It("forces a reader onto a rebuild that started after the mark", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + entered, release := fake.block() + refreshDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + refreshDone <- idx.Refresh(ctx, key) + }() + Eventually(entered).Should(Receive()) + + idx.MarkDirty(key) + fake.setPages(key, "new") + + readDone := make(chan []*domain.Page, 1) + readErr := make(chan error, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + readDone <- pages + readErr <- err + }() + Consistently(fake.lister.ListFilesCallCount, "150ms").Should(Equal(2)) + + release() + Expect(<-refreshDone).To(BeNil()) + Expect(<-readErr).To(BeNil()) + Expect(titles(<-readDone)).To(Equal([]string{"new"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(3)) + }) + + It("re-reads every file of every known key on ForceReload", func() { + aTasks := index.NewKey("/vault-a", "24 Tasks") + bTasks := index.NewKey("/vault-b", "24 Tasks") + fake.setPages(aTasks, "a") + fake.setPages(bTasks, "b") + idx := newIndex(fake) + Expect(idx.Build(ctx, []index.Key{aTasks, bTasks})).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + before := fake.reader.ReadEntryCallCount() + + idx.ForceReload() + // The mark alone reads and lists nothing. + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + Expect(fake.reader.ReadEntryCallCount()).To(Equal(before)) + + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + _, err = idx.ListPages(ctx, "/vault-b", "24 Tasks") + Expect(err).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(4)) + // Every file is re-read although no fingerprint changed. + Expect(fake.reader.ReadEntryCallCount() - before).To(Equal(2)) + }) + + It("ignores a dirty mark for an unknown key", func() { + idx := newIndex(fake) + idx.MarkDirty(index.NewKey("/unknown", "24 Tasks")) + Expect(fake.lister.ListFilesCallCount()).To(Equal(0)) + }) + }) + + Describe("Failed rebuild", func() { + It("keeps the previous snapshot and retries while the key stays dirty", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + idx.MarkDirty(key) + fake.setFailure(stderrors.New("boom")) + + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"old"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + + Expect(idx.Refresh(ctx, key)).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(3)) + + fake.setFailure(nil) + fake.setPages(key, "new") + pages, err = idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(4)) + }) + + It("returns an error when the very first build fails", func() { + idx := newIndex(fake) + fake.setFailure(stderrors.New("boom")) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Rescan", func() { + It("refreshes every known key once the interval has passed", func() { + clock := libtime.NewCurrentDateTime() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock.SetNow(libtime.DateTime(start)) + + aTasks := index.NewKey("/vault-a", "24 Tasks") + bTasks := index.NewKey("/vault-b", "24 Tasks") + fake.setPages(aTasks, "a") + fake.setPages(bTasks, "b") + idx := index.NewPageIndex(fake.reader, fake.lister, clock, fastWaiter(), index.TestMetrics) + Expect(idx.Build(ctx, []index.Key{aTasks, bTasks})).To(BeNil()) + Expect(fake.lister.ListFilesCallCount()).To(Equal(2)) + + rescanCtx, cancel := context.WithCancel(ctx) + rescanDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + rescanDone <- idx.Rescan(rescanCtx) + }() + + Consistently(fake.lister.ListFilesCallCount, "100ms").Should(Equal(2)) + + fake.setPages(aTasks, "a2") + fake.setPages(bTasks, "b2") + clock.SetNow(libtime.DateTime(start.Add(time.Minute))) + + Eventually(fake.lister.ListFilesCallCount, "2s").Should(Equal(4)) + Eventually(func() []string { + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + return titles(pages) + }, "2s").Should(Equal([]string{"a2"})) + Consistently(fake.lister.ListFilesCallCount, "100ms").Should(Equal(4)) + + cancel() + Eventually(rescanDone, "2s").Should(Receive(BeNil())) + }) + + It("keeps the rescan interval inside the 60s ceiling", func() { + Expect(index.RescanInterval).To(BeNumerically("<=", 60*time.Second)) + }) + }) + + Describe("context cancellation", func() { + It("fails a waiting reader without aborting the build it waits on", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + idx.MarkDirty(key) + fake.setPages(key, "new") + entered, release := fake.block() + + builderDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + builderDone <- err + }() + Eventually(entered).Should(Receive()) + + readCtx, cancelRead := context.WithCancel(ctx) + readDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := idx.ListPages(readCtx, "/vault-a", "24 Tasks") + readDone <- err + }() + Consistently(fake.lister.ListFilesCallCount, "100ms").Should(Equal(2)) + cancelRead() + Eventually(readDone).Should(Receive(HaveOccurred())) + + release() + Expect(<-builderDone).To(BeNil()) + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + }) + + It("runs an owned follow-up even when the caller's ctx is cancelled", func() { + key := index.NewKey("/vault-a", "24 Tasks") + fake.setPages(key, "old") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + + entered, release := fake.block() + fake.setPages(key, "new") + refreshDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + refreshDone <- idx.Refresh(ctx, key) + }() + Eventually(entered).Should(Receive()) + + idx.MarkDirty(key) + + readCtx, cancelRead := context.WithCancel(ctx) + readDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + _, err := idx.ListPages(readCtx, "/vault-a", "24 Tasks") + readDone <- err + }() + Consistently(fake.lister.ListFilesCallCount, "150ms").Should(Equal(2)) + + cancelRead() + release() + Expect(<-refreshDone).To(BeNil()) + Eventually(readDone, "2s").Should(Receive(HaveOccurred())) + + Expect(fake.lister.ListFilesCallCount()).To(Equal(3)) + pages, err := idx.ListPages(ctx, "/vault-a", "24 Tasks") + Expect(err).To(BeNil()) + Expect(titles(pages)).To(Equal([]string{"new"})) + Expect(fake.lister.ListFilesCallCount()).To(Equal(3)) + }) + + It("returns an error from Build when ctx is already cancelled", func() { + idx := newIndex(fake) + cancelled, cancel := context.WithCancel(ctx) + cancel() + err := idx.Build(cancelled, []index.Key{index.NewKey("/vault-a", "24 Tasks")}) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("NewKey", func() { + It("cleans both parts so equivalent paths share a key", func() { + Expect(index.NewKey("/v/", "24 Tasks/")).To(Equal(index.NewKey("/v", "24 Tasks"))) + }) + }) +}) diff --git a/pkg/index/index_writemark_test.go b/pkg/index/index_writemark_test.go new file mode 100644 index 0000000..2aabbb7 --- /dev/null +++ b/pkg/index/index_writemark_test.go @@ -0,0 +1,249 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "os" + "path/filepath" + "sync" + + "github.com/bborbe/vault-cli/pkg/domain" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// pageFilePath is the on-disk path of one page file inside a key's folder. +func pageFilePath(key index.Key, name string) string { + return filepath.Join(key.VaultPath, key.PagesDir, name+".md") +} + +var _ = Describe("Per-file write marks", func() { + var ( + ctx context.Context + fake *storageFake + vaultDir string + key index.Key + ) + + BeforeEach(func() { + ctx = context.Background() + fake = newStorageFake() + vaultDir = GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(vaultDir, "24 Tasks"), 0750)).To(Succeed()) + key = index.NewKey(vaultDir, "24 Tasks") + }) + + It("AC5(i) re-reads exactly the marked file and lists nothing", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + fake.putPage(key, "Alpha", "# Alpha v2") + listings := fake.lister.ListFilesCallCount() + reads := fake.reader.ReadEntryCallCount() + idx.MarkFileDirty(key, "Alpha") + + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(fake.lister.ListFilesCallCount() - listings).To(Equal(0)) + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(1)) + Expect(string(pages[0].Content)).To(Equal("# Alpha v2")) + }) + + It("AC5(i) shares one re-read between two concurrent readers", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + fake.putPage(key, "Alpha", "# Alpha v2") + entered, release := fake.blockReads("Alpha.md") + reads := fake.reader.ReadEntryCallCount() + idx.MarkFileDirty(key, "Alpha") + + results := make([][]*domain.Page, 2) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + results[i] = pages + }(i) + } + Eventually(entered).Should(Receive(Equal("Alpha.md"))) + Consistently(fake.reader.ReadEntryCallCount, "100ms").Should(Equal(reads + 1)) + + release("Alpha.md", 0) + wg.Wait() + + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(1)) + for i := range results { + Expect(string(results[i][0].Content)).To(Equal("# Alpha v2")) + } + }) + + It("AC5(i) strips a [[wikilink]] wrapper from the id", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + fake.putPage(key, "Alpha", "# Alpha v2") + listings := fake.lister.ListFilesCallCount() + reads := fake.reader.ReadEntryCallCount() + idx.MarkFileDirty(key, "[[Alpha]]") + + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(fake.lister.ListFilesCallCount() - listings).To(Equal(0)) + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(1)) + Expect(string(pages[0].Content)).To(Equal("# Alpha v2")) + }) + + It("AC6(d) blocks a concurrent reader until the marked file is re-read", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + fake.putPage(key, "Alpha", "# Alpha v2") + entered, release := fake.blockReads("Alpha.md") + idx.MarkFileDirty(key, "Alpha") + + triggered := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + triggered <- pages + }() + Eventually(entered).Should(Receive(Equal("Alpha.md"))) + + concurrent := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + concurrent <- pages + }() + Consistently(concurrent, "150ms").ShouldNot(Receive()) + + release("Alpha.md", 0) + Expect(string((<-triggered)[0].Content)).To(Equal("# Alpha v2")) + Expect(string((<-concurrent)[0].Content)).To(Equal("# Alpha v2")) + }) + + It("re-reads a file marked again while its read was in flight", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + fake.putPage(key, "Alpha", "# Alpha v2") + entered, release := fake.blockReads("Alpha.md") + reads := fake.reader.ReadEntryCallCount() + idx.MarkFileDirty(key, "Alpha") + + triggered := make(chan []*domain.Page, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + triggered <- pages + }() + Eventually(entered).Should(Receive(Equal("Alpha.md"))) + + // A second write lands while the first re-read is blocked. + fake.putPage(key, "Alpha", "# Alpha v3") + idx.MarkFileDirty(key, "Alpha") + release("Alpha.md", 0) + Expect(string((<-triggered)[0].Content)).To(Equal("# Alpha v2")) + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(1)) + + // Stop gating so the follow-up re-read can finish. + _, _ = fake.blockReads() + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(string(pages[0].Content)).To(Equal("# Alpha v3")) + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(2)) + }) + + for _, name := range []string{"alpha", "sub/Alpha", "../Alpha", "[[]]", "Missing"} { + name := name + It("falls back to a folder-level mark for the id "+name, func() { + fake.setPages(key, "Alpha", "Beta") + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + listings := fake.lister.ListFilesCallCount() + reads := fake.reader.ReadEntryCallCount() + idx.MarkFileDirty(key, name) + + pages, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(fake.lister.ListFilesCallCount() - listings).To(Equal(1)) + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(0)) + Expect(titles(pages)).To(Equal([]string{"Alpha", "Beta"})) + }) + } + + It("counts the write, rescan and reload read sites under their own reason", func() { + fake.setPages(key, "Alpha", "Beta") + Expect(os.WriteFile(pageFilePath(key, "Alpha"), []byte("x"), 0600)).To(Succeed()) + idx := newIndex(fake) + _, err := idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + + write := index.FilesReadTotal("write") + fake.putPage(key, "Alpha", "# Alpha v2") + idx.MarkFileDirty(key, "Alpha") + _, err = idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(index.FilesReadTotal("write") - write).To(Equal(1.0)) + + rescan := index.FilesReadTotal("rescan") + fake.putPage(key, "Beta", "# Beta v2") + Expect(idx.Refresh(ctx, key)).To(BeNil()) + Expect(index.FilesReadTotal("rescan") - rescan).To(Equal(1.0)) + + reload := index.FilesReadTotal("reload") + idx.ForceReload() + _, err = idx.ListPages(ctx, key.VaultPath, key.PagesDir) + Expect(err).To(BeNil()) + Expect(index.FilesReadTotal("reload") - reload).To(Equal(2.0)) + }) + + It("AC5(iii) reads every file exactly once after ForceReload", func() { + aTasks := index.NewKey("/vault-a", "24 Tasks") + bTasks := index.NewKey("/vault-b", "24 Tasks") + fake.setPages(aTasks, "a1", "a2", "a3") + fake.setPages(bTasks, "b1", "b2") + idx := newIndex(fake) + Expect(idx.Build(ctx, []index.Key{aTasks, bTasks})).To(BeNil()) + + reads := fake.reader.ReadEntryCallCount() + idx.ForceReload() + + for _, k := range []index.Key{aTasks, bTasks} { + _, err := idx.ListPages(ctx, k.VaultPath, k.PagesDir) + Expect(err).To(BeNil()) + } + // Five files across the two keys, each read exactly once. + Expect(fake.reader.ReadEntryCallCount() - reads).To(Equal(5)) + }) +}) diff --git a/pkg/index/reader_test.go b/pkg/index/reader_test.go index 05d168b..ad47c9a 100644 --- a/pkg/index/reader_test.go +++ b/pkg/index/reader_test.go @@ -106,7 +106,7 @@ var _ = Describe("DirectoryLister", func() { entries, err := index.NewDirectoryLister().ListFiles(ctx, vaultDir, readerFolder) Expect(err).NotTo(HaveOccurred()) - names := []string{} + names := make([]string, 0, len(entries)) for _, entry := range entries { names = append(names, entry.Name) Expect(entry.Fingerprint.Size).To(Equal(int64(1))) From af0f0262f896fe80ea10194bf4b83b04a3378789 Mon Sep 17 00:00:00 2001 From: Benjamin Borbe Date: Tue, 17 Nov 2026 18:19:08 +0100 Subject: [PATCH 3/3] feat(index): add metadata-only bolt store (format v2) with hydration and write-through --- docs/page-index.md | 37 ++ go.mod | 4 +- go.sum | 4 +- mocks/index-store.go | 336 +++++++++++++++ pkg/index/export_test.go | 21 + pkg/index/index.go | 39 +- pkg/index/index_build.go | 90 +++- pkg/index/index_changedpages_test.go | 31 ++ pkg/index/index_file.go | 4 +- pkg/index/store.go | 453 ++++++++++++++++++++ pkg/index/store_codec.go | 157 +++++++ pkg/index/store_hydration_test.go | 430 +++++++++++++++++++ pkg/index/store_test.go | 608 +++++++++++++++++++++++++++ pkg/index/store_v2_test.go | 238 +++++++++++ pkg/index/store_writethrough_test.go | 414 ++++++++++++++++++ 15 files changed, 2853 insertions(+), 13 deletions(-) create mode 100644 mocks/index-store.go create mode 100644 pkg/index/store.go create mode 100644 pkg/index/store_codec.go create mode 100644 pkg/index/store_hydration_test.go create mode 100644 pkg/index/store_test.go create mode 100644 pkg/index/store_v2_test.go create mode 100644 pkg/index/store_writethrough_test.go diff --git a/docs/page-index.md b/docs/page-index.md index 61d7624..b4f2ee7 100644 --- a/docs/page-index.md +++ b/docs/page-index.md @@ -76,6 +76,8 @@ injected `Metrics` interface (`FileRead(reason)`, - A failed listing keeps serving the previous snapshot and logs the error with the key; the mark stays pending, so the next read retries. A failed first build returns an error. +- With a store, a key is hydrated from it and resolves exactly one stat-diff + before it is served; the stored records are never served as written. ## Incremental updates @@ -112,3 +114,38 @@ A key is `NewKey(vault path, folder)`, which applies `filepath.Clean` to both parts so equivalent paths produce identical keys. A consumer derives the read side and the event side from the same vault config entry; a mismatch silently disables event refresh and leaves the folder on its rescan cadence. + +## Store format version 2 + +An optional bolt store (`NewPageIndexWithStore`) persists the index as a cache, +so a restart does not re-parse unchanged files. It is never a source of truth. + +- **Record.** One record per indexed file: filename, has-page flag, full + frontmatter, file metadata, Open Questions and fingerprint. **No body.** An + excluded file stores its fingerprint and no page. +- **Frozen names**, byte-identical to vault-ui's v1 store: meta bucket + `vault-ui-page-index-meta`, identity key `identity`, identity JSON fields + `StoreFormat` / `ParserVersion`, entry bucket name + `vault-ui-page-index\x00\x00`. A v1 file is therefore + recognised and discarded as a format mismatch, entry buckets deleted. +- **Identity.** `StoreFormat` is `2`; `ParserVersion` is the vault-cli module + version from build info (the dependency entry, or the main module when + vault-cli is itself main, else `(unknown)`). +- **Discard reasons.** On open, a store whose identity is missing or empty, + damaged, of another store format, or of another parser version is discarded: + its entry buckets are deleted and one warning names the path and the reason — + `store format mismatch (stored N, current 2)`, `parser version mismatch`, + `store file missing or empty`, or `damaged`. Format is checked before parser + version. Opening waits at most 5 s for the file lock; any open or reconcile + failure yields a discarded store that serves nothing and never fails + construction. +- **Writes.** Each publication writes exactly its changed and vanished entries + in one transaction, outside the index mutex. A failed write leaves the + previous content intact and logs one warning. +- **Location.** `DefaultStorePath(app)` returns + `//page-index.bolt`; `NewBoltStore` creates the + directory with mode 0700. The store never lives inside a vault. +- **Codec.** Frontmatter is encoded through `FrontmatterMap.RawMap()` and pages + are rebuilt with `domain.NewPage`: a JSON codec on `domain.Page` would drop + every frontmatter key. Decoding never preallocates from stored counts, and + any decode failure discards the store. diff --git a/go.mod b/go.mod index 0f81a04..a6507a8 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module github.com/bborbe/vault-cli go 1.27.4 require ( + github.com/bborbe/boltkv v1.17.0 github.com/bborbe/collection v1.21.7 github.com/bborbe/cqrs v0.6.16 github.com/bborbe/errors v1.6.7 github.com/bborbe/kafka v1.29.14 + github.com/bborbe/kv v1.21.30 github.com/bborbe/log v1.7.9 github.com/bborbe/notification v0.7.13 github.com/bborbe/run v1.11.12 @@ -17,6 +19,7 @@ require ( github.com/onsi/ginkgo/v2 v2.34.0 github.com/onsi/gomega v1.44.0 github.com/spf13/cobra v1.10.2 + go.etcd.io/bbolt v1.5.0 golang.org/x/sys v0.50.0 golang.org/x/term v0.48.0 gopkg.in/yaml.v3 v3.0.1 @@ -27,7 +30,6 @@ require ( github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/bborbe/http v1.26.34 // indirect github.com/bborbe/k8s v1.15.1 // indirect - github.com/bborbe/kv v1.21.30 // indirect github.com/bborbe/math v1.4.20 // indirect github.com/bborbe/parse v1.12.7 // indirect github.com/bborbe/sentry v1.10.8 // indirect diff --git a/go.sum b/go.sum index e4a7cf8..6b873db 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/IBM/sarama v1.61.1 h1:I59MWPHQUWqJNdRpsDUcbeCriog8SxjQaPfHNWxidEg= github.com/IBM/sarama v1.61.1/go.mod h1:dITlGHIiCQL/maGtBfDHNMDvyWgC9Ww//8pmlsU3RUs= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/bborbe/boltkv v1.16.8 h1:L+YyepgvRjYKkDg9CSQJvOweXdCcKiqsl40g+CHXWJQ= -github.com/bborbe/boltkv v1.16.8/go.mod h1:IjPczouQRUnWcGbIBXQMyXoyDic0qgLL9HmFQxBLkGo= +github.com/bborbe/boltkv v1.17.0 h1:IRfO5oBznAuGzVYz7HrJjPPKWuUj4Df0yLcI9/ZL+jw= +github.com/bborbe/boltkv v1.17.0/go.mod h1:RyD+DM/KROJrbEseBRUpIguQV4UepTwNFN+0rVHko9c= github.com/bborbe/collection v1.21.7 h1:VNbMotiaTZy4b4Mk9gZ5UrDXeYiywJGP95Jhp+dqXZ8= github.com/bborbe/collection v1.21.7/go.mod h1:nMOfUE29djizqSTytT+OGvl41NvFfU1HpIjRnpl8Zco= github.com/bborbe/cqrs v0.6.16 h1:wwEnswzU6jbfJMPp1qlPnkvhxusVV92zEkQiE/MmvS8= diff --git a/mocks/index-store.go b/mocks/index-store.go new file mode 100644 index 0000000..23887a5 --- /dev/null +++ b/mocks/index-store.go @@ -0,0 +1,336 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package mocks + +import ( + "context" + "sync" + + "github.com/bborbe/vault-cli/pkg/index" +) + +type IndexStore struct { + CloseStub func() error + closeMutex sync.RWMutex + closeArgsForCall []struct { + } + closeReturns struct { + result1 error + } + closeReturnsOnCall map[int]struct { + result1 error + } + LoadStub func(context.Context, index.Key) ([]index.StoredEntry, bool, error) + loadMutex sync.RWMutex + loadArgsForCall []struct { + arg1 context.Context + arg2 index.Key + } + loadReturns struct { + result1 []index.StoredEntry + result2 bool + result3 error + } + loadReturnsOnCall map[int]struct { + result1 []index.StoredEntry + result2 bool + result3 error + } + PathStub func() string + pathMutex sync.RWMutex + pathArgsForCall []struct { + } + pathReturns struct { + result1 string + } + pathReturnsOnCall map[int]struct { + result1 string + } + WriteStub func(context.Context, index.Key, []index.StoredEntry, []string) error + writeMutex sync.RWMutex + writeArgsForCall []struct { + arg1 context.Context + arg2 index.Key + arg3 []index.StoredEntry + arg4 []string + } + writeReturns struct { + result1 error + } + writeReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *IndexStore) Close() error { + fake.closeMutex.Lock() + ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] + fake.closeArgsForCall = append(fake.closeArgsForCall, struct { + }{}) + stub := fake.CloseStub + fakeReturns := fake.closeReturns + fake.recordInvocation("Close", []interface{}{}) + fake.closeMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexStore) CloseCallCount() int { + fake.closeMutex.RLock() + defer fake.closeMutex.RUnlock() + return len(fake.closeArgsForCall) +} + +func (fake *IndexStore) CloseCalls(stub func() error) { + fake.closeMutex.Lock() + defer fake.closeMutex.Unlock() + fake.CloseStub = stub +} + +func (fake *IndexStore) CloseReturns(result1 error) { + fake.closeMutex.Lock() + defer fake.closeMutex.Unlock() + fake.CloseStub = nil + fake.closeReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexStore) CloseReturnsOnCall(i int, result1 error) { + fake.closeMutex.Lock() + defer fake.closeMutex.Unlock() + fake.CloseStub = nil + if fake.closeReturnsOnCall == nil { + fake.closeReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.closeReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexStore) Load(arg1 context.Context, arg2 index.Key) ([]index.StoredEntry, bool, error) { + fake.loadMutex.Lock() + ret, specificReturn := fake.loadReturnsOnCall[len(fake.loadArgsForCall)] + fake.loadArgsForCall = append(fake.loadArgsForCall, struct { + arg1 context.Context + arg2 index.Key + }{arg1, arg2}) + stub := fake.LoadStub + fakeReturns := fake.loadReturns + fake.recordInvocation("Load", []interface{}{arg1, arg2}) + fake.loadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *IndexStore) LoadCallCount() int { + fake.loadMutex.RLock() + defer fake.loadMutex.RUnlock() + return len(fake.loadArgsForCall) +} + +func (fake *IndexStore) LoadCalls(stub func(context.Context, index.Key) ([]index.StoredEntry, bool, error)) { + fake.loadMutex.Lock() + defer fake.loadMutex.Unlock() + fake.LoadStub = stub +} + +func (fake *IndexStore) LoadArgsForCall(i int) (context.Context, index.Key) { + fake.loadMutex.RLock() + defer fake.loadMutex.RUnlock() + argsForCall := fake.loadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *IndexStore) LoadReturns(result1 []index.StoredEntry, result2 bool, result3 error) { + fake.loadMutex.Lock() + defer fake.loadMutex.Unlock() + fake.LoadStub = nil + fake.loadReturns = struct { + result1 []index.StoredEntry + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *IndexStore) LoadReturnsOnCall(i int, result1 []index.StoredEntry, result2 bool, result3 error) { + fake.loadMutex.Lock() + defer fake.loadMutex.Unlock() + fake.LoadStub = nil + if fake.loadReturnsOnCall == nil { + fake.loadReturnsOnCall = make(map[int]struct { + result1 []index.StoredEntry + result2 bool + result3 error + }) + } + fake.loadReturnsOnCall[i] = struct { + result1 []index.StoredEntry + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *IndexStore) Path() string { + fake.pathMutex.Lock() + ret, specificReturn := fake.pathReturnsOnCall[len(fake.pathArgsForCall)] + fake.pathArgsForCall = append(fake.pathArgsForCall, struct { + }{}) + stub := fake.PathStub + fakeReturns := fake.pathReturns + fake.recordInvocation("Path", []interface{}{}) + fake.pathMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexStore) PathCallCount() int { + fake.pathMutex.RLock() + defer fake.pathMutex.RUnlock() + return len(fake.pathArgsForCall) +} + +func (fake *IndexStore) PathCalls(stub func() string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = stub +} + +func (fake *IndexStore) PathReturns(result1 string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = nil + fake.pathReturns = struct { + result1 string + }{result1} +} + +func (fake *IndexStore) PathReturnsOnCall(i int, result1 string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = nil + if fake.pathReturnsOnCall == nil { + fake.pathReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.pathReturnsOnCall[i] = struct { + result1 string + }{result1} +} + +func (fake *IndexStore) Write(arg1 context.Context, arg2 index.Key, arg3 []index.StoredEntry, arg4 []string) error { + var arg3Copy []index.StoredEntry + if arg3 != nil { + arg3Copy = make([]index.StoredEntry, len(arg3)) + copy(arg3Copy, arg3) + } + var arg4Copy []string + if arg4 != nil { + arg4Copy = make([]string, len(arg4)) + copy(arg4Copy, arg4) + } + fake.writeMutex.Lock() + ret, specificReturn := fake.writeReturnsOnCall[len(fake.writeArgsForCall)] + fake.writeArgsForCall = append(fake.writeArgsForCall, struct { + arg1 context.Context + arg2 index.Key + arg3 []index.StoredEntry + arg4 []string + }{arg1, arg2, arg3Copy, arg4Copy}) + stub := fake.WriteStub + fakeReturns := fake.writeReturns + fake.recordInvocation("Write", []interface{}{arg1, arg2, arg3Copy, arg4Copy}) + fake.writeMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *IndexStore) WriteCallCount() int { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + return len(fake.writeArgsForCall) +} + +func (fake *IndexStore) WriteCalls(stub func(context.Context, index.Key, []index.StoredEntry, []string) error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = stub +} + +func (fake *IndexStore) WriteArgsForCall(i int) (context.Context, index.Key, []index.StoredEntry, []string) { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + argsForCall := fake.writeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *IndexStore) WriteReturns(result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + fake.writeReturns = struct { + result1 error + }{result1} +} + +func (fake *IndexStore) WriteReturnsOnCall(i int, result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + if fake.writeReturnsOnCall == nil { + fake.writeReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.writeReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *IndexStore) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *IndexStore) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ index.Store = new(IndexStore) diff --git a/pkg/index/export_test.go b/pkg/index/export_test.go index 694cea6..0e64e3f 100644 --- a/pkg/index/export_test.go +++ b/pkg/index/export_test.go @@ -6,6 +6,7 @@ package index import ( "fmt" + "reflect" "sync" libtime "github.com/bborbe/time" @@ -77,6 +78,21 @@ func NewPageIndexWithWarnf( return idx } +// NewPageIndexWithStoreAndWarnf builds a store-backed page index whose +// warnings go to warnf. +func NewPageIndexWithStoreAndWarnf( + reader PageReader, + lister DirectoryLister, + currentDateTimeGetter libtime.CurrentDateTimeGetter, + waiter libtime.WaiterDuration, + store Store, + warnf func(format string, args ...any), +) PageIndex { + idx := NewPageIndexWithStore(reader, lister, currentDateTimeGetter, waiter, TestMetrics, store) + SetWarnf(idx, warnf) + return idx +} + // FingerprintSetIdentity returns an identity token for the key's recorded // fingerprint set. An unknown key reports "". func FingerprintSetIdentity(idx PageIndex, key Key) string { @@ -108,3 +124,8 @@ func SnapshotIdentity(idx PageIndex, key Key) string { } return fmt.Sprintf("%p", e.snapshot.entries) } + +// PersistedRecordType returns the store codec's on-disk record type. +func PersistedRecordType() reflect.Type { + return reflect.TypeOf(storeRecord{}) +} diff --git a/pkg/index/index.go b/pkg/index/index.go index ac7df8b..807eda7 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -110,10 +110,15 @@ const ( // buildReload lists the folder and re-reads every file, ignoring // fingerprints. buildReload + // buildHydrate loads the key's stored entries, seeds them as the stat-diff + // baseline and lists the folder. It is used only for a key with no published + // snapshot whose store has not been consulted yet. + buildHydrate ) // storeDelta is one publication's change: the entries it re-read and the names -// it dropped. +// it dropped. It is computed under the index mutex and written to the store +// after the build is released, so no reader waits on store I/O. // StoredEntry is one indexed file's record: the metadata-only entry, or nil when // the file was excluded, plus the pre-read fingerprint. It never holds a body. type StoredEntry struct { @@ -166,6 +171,8 @@ func (b *build) covers(kind buildKind, needSeq uint64) bool { switch b.kind { case buildCold, buildReload: return true + case buildHydrate: + return kind == buildCold || kind == buildStatDiff || kind == buildHydrate case buildFileReads: return kind == buildFileReads case buildStatDiff: @@ -180,6 +187,14 @@ type entry struct { hasSnapshot bool fingerprints map[string]FileFingerprint + // storeLoaded records that this key has consulted the store, so the store + // is never re-read on a later start of the same process. + storeLoaded bool + // storedBaseline records that snapshot and fingerprints hold entries seeded + // from the store that have not yet been stat-diffed and must never be served + // as they are. + storedBaseline bool + // fileReadSeq holds, per base filename, the read sequence of the read whose // result is currently applied — the page present or removed. A name whose // page was removed keeps its entry as a tombstone; it is never deleted. @@ -264,19 +279,36 @@ type pageIndex struct { currentDateTimeGetter libtime.CurrentDateTimeGetter waiter libtime.WaiterDuration metrics Metrics + // store hydrates each key from the on-disk cache before its first stat-diff. + // It is written once at construction and read under the mutex. + store Store // warnf emits the per-file exclusion warnings. It is a field so a consumer // can redirect them per index instance. warnf func(format string, args ...any) } -// NewPageIndex creates an empty page index over the given reader and lister. -// A nil metrics records nothing. +// NewPageIndex creates an empty page index over the given reader and lister, +// with no store. A nil metrics records nothing. func NewPageIndex( reader PageReader, lister DirectoryLister, currentDateTimeGetter libtime.CurrentDateTimeGetter, waiter libtime.WaiterDuration, metrics Metrics, +) PageIndex { + return NewPageIndexWithStore(reader, lister, currentDateTimeGetter, waiter, metrics, nil) +} + +// NewPageIndexWithStore creates an empty page index that hydrates each key from +// store before its first stat-diff and writes each publication through to it. +// A nil metrics records nothing; a nil store disables persistence. +func NewPageIndexWithStore( + reader PageReader, + lister DirectoryLister, + currentDateTimeGetter libtime.CurrentDateTimeGetter, + waiter libtime.WaiterDuration, + metrics Metrics, + store Store, ) PageIndex { if metrics == nil { metrics = NewNoopMetrics() @@ -288,6 +320,7 @@ func NewPageIndex( currentDateTimeGetter: currentDateTimeGetter, waiter: waiter, metrics: metrics, + store: store, warnf: slogWarnf, } } diff --git a/pkg/index/index_build.go b/pkg/index/index_build.go index 9800929..1e5febb 100644 --- a/pkg/index/index_build.go +++ b/pkg/index/index_build.go @@ -94,6 +94,9 @@ func (p *pageIndex) ensure( // hold the mutex. func (p *pageIndex) pendingLocked(e *entry, refresh bool) (buildKind, uint64, bool) { if !e.hasSnapshot { + if p.store != nil && !e.storeLoaded { + return buildHydrate, e.requestSeq, true + } return buildCold, e.requestSeq, true } if refresh { @@ -201,10 +204,73 @@ func (p *pageIndex) execute(ctx context.Context, key Key, b *build) { switch b.kind { case buildFileReads: p.executeFileReads(readCtx, key, b) + case buildHydrate: + p.executeHydrate(readCtx, key, b) default: p.executeListing(readCtx, key, b) } p.release(key, b) + p.writeThrough(readCtx, key, b.delta) +} + +// writeThrough persists one publication's delta outside the index mutex. It is +// called after the build is released, so no reader waits on store I/O. A failed +// write leaves the previous store content intact, does not affect what is +// served, and is reported with exactly one warning naming the failure. +func (p *pageIndex) writeThrough(ctx context.Context, key Key, delta *storeDelta) { + if p.store == nil || delta == nil || (len(delta.puts) == 0 && len(delta.deletes) == 0) { + return + } + if err := p.store.Write(ctx, key, delta.puts, delta.deletes); err != nil { + p.warnf( + "page index store write failed for %q/%q: %v", + key.VaultPath, + key.PagesDir, + err, + ) + } +} + +// executeHydrate loads the key's stored entries, seeds them as the stat-diff +// baseline and runs one listing over them. The store read runs outside the +// index mutex. A store that cannot supply entries leaves the key on the +// full-parse path. A failed listing clears the seeded baseline so a stored +// snapshot is never served without its stat-diff. +func (p *pageIndex) executeHydrate(ctx context.Context, key Key, b *build) { + entries, ok, err := p.store.Load(ctx, key) + if err != nil { + // A store that cannot supply entries leaves the key on the full-parse + // path: a store failure never stops the index from starting or serving. + ok = false + } + + p.mu.Lock() + e := p.entryLocked(key) + if ok { + seedStoredBaseline(e, entries) + } + e.storeLoaded = true + p.mu.Unlock() + + p.executeListing(ctx, key, b) +} + +// seedStoredBaseline fills the entry's snapshot and fingerprints from the stored +// entries, in the store's key order (byte-ascending file names, which equals +// os.ReadDir order). The caller must hold the mutex. hasSnapshot stays false: +// the baseline is an input to the stat-diff, never a published value. +func seedStoredBaseline(e *entry, stored []StoredEntry) { + entries := make([]*Entry, 0, len(stored)) + fingerprints := make(map[string]FileFingerprint, len(stored)) + for _, item := range stored { + fingerprints[item.Filename] = item.Fingerprint + if item.Entry != nil { + entries = append(entries, item.Entry) + } + } + e.snapshot = publish(entries) + e.fingerprints = fingerprints + e.storedBaseline = true } // executeFileReads re-reads exactly the write-marked files. The marks are taken @@ -281,14 +347,14 @@ func (p *pageIndex) executeListing(ctx context.Context, key Key, b *build) { e := p.entryLocked(key) startSeq := e.requestSeq b.startSeq = startSeq - baseline := e.hasSnapshot + baseline := e.hasSnapshot || e.storedBaseline full := b.kind == buildCold || b.kind == buildReload || !baseline forced := takeWriteMarksLocked(e, startSeq, full) p.mu.Unlock() reason := b.reason switch { - case !baseline, b.kind == buildCold: + case !baseline, b.kind == buildCold, b.kind == buildHydrate: reason = ReadReasonBuild case b.kind == buildReload: reason = ReadReasonReload @@ -327,6 +393,7 @@ func (p *pageIndex) executeListing(ctx context.Context, key Key, b *build) { e = p.entryLocked(key) b.delta = publishListingLocked(e, entries, reads, full) b.snap = e.snapshot + e.storedBaseline = false e.dirtyResolvedSeq = startSeq if full { e.reloadResolvedSeq = startSeq @@ -356,7 +423,9 @@ func takeWriteMarksLocked(e *entry, startSeq uint64, full bool) map[string]bool // diffListingLocked compares a listing with the recorded fingerprints and // returns the names to re-read, and whether the pass is unchanged. An -// unchanged pass requires a published snapshot to keep serving. The name set +// unchanged pass requires a published snapshot to keep serving: a hydrated +// baseline has recorded fingerprints but nothing published, so the fast path +// would publish nothing and leave the key serving nothing. The name set // is equal exactly when the lengths match and every listed entry is present // and equal, and the missing/empty state did not flip. The caller must hold // the mutex. @@ -403,7 +472,8 @@ func publishListingLocked( // A publication is attributable to a bounded set of single files only when // it merged into an already-published snapshot. A key's first publication // publishes the whole snapshot whatever the build's kind, so it stays - // unbounded. + // unbounded: the hydrate path reaches here with full false but nothing + // published, which e.hasSnapshot still reports before it is set below. bounded := !full && e.hasSnapshot e.recordChangeLocked(listingChangedNames(full, delta), bounded, unboundedListingReason(e, bounded)) e.hasSnapshot = true @@ -424,13 +494,21 @@ func unboundedListingReason(e *entry, bounded bool) string { } // failListing records a listing or collection failure: nothing is published, -// the error is logged with the key, and the mark stays pending so the next -// read retries. +// the error is logged with the key, a stored baseline that was never +// stat-diffed is dropped, and the mark stays pending so the next read retries. func (p *pageIndex) failListing(key Key, b *build, err error) { p.mu.Lock() defer p.mu.Unlock() b.snap = published{} b.err = err + e := p.entryLocked(key) + if e.storedBaseline { + // The stored baseline was never stat-diffed; never serve it. + e.snapshot = published{} + e.fingerprints = nil + e.storedBaseline = false + e.hasSnapshot = false + } slog.Error( "page index rebuild failed", "vault", key.VaultPath, diff --git a/pkg/index/index_changedpages_test.go b/pkg/index/index_changedpages_test.go index 7a97252..4d013d3 100644 --- a/pkg/index/index_changedpages_test.go +++ b/pkg/index/index_changedpages_test.go @@ -139,4 +139,35 @@ var _ = Describe("ChangedPagesSince", func() { Expect(ok).To(BeFalse()) Expect(names).To(BeEmpty()) }) + + It("reports unknown after a hydrate publishes the whole snapshot", func() { + // The store-backed hydrate is also a first publication, but it seeds the + // stored pages as its stat-diff baseline, so its delta names only the files + // it re-read and never the pages it carried forward. It must not claim a + // bound the cold path refuses: both first publications are unbounded. + storeVaultDir := newStoreVault(3) + storeKey := index.NewKey(storeVaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, storeKey, storeVaultDir, equivalenceFolder) + + settle() + writeFixtureFile( + storeVaultDir, + "Page01.md", + "---\ntitle: Page01\n---\n# Page01 v2\n", + ) + + start := newStoreStart(store) + pages, err := start.idx.ListPages(ctx, storeVaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(pages).To(HaveLen(3)) + // The hydrate re-read exactly the one changed file, so the delta it would + // have reported is a partial set — the under-reporting case, not an empty + // one. + Expect(start.reader.Names()).To(Equal([]string{"Page01.md"})) + + names, ok := start.idx.ChangedPagesSince(storeKey, 0) + Expect(ok).To(BeFalse()) + Expect(names).To(BeEmpty()) + }) }) diff --git a/pkg/index/index_file.go b/pkg/index/index_file.go index 887f783..fa7c4a9 100644 --- a/pkg/index/index_file.go +++ b/pkg/index/index_file.go @@ -54,9 +54,11 @@ func (p *pageIndex) RefreshFile(ctx context.Context, key Key, filename string) e p.metrics.FileRead(ReadReasonEvent) p.mu.Lock() - p.applyFileReadLocked(e, key, filename, seq, read, fingerprint, readErr) + delta := p.applyFileReadLocked(e, key, filename, seq, read, fingerprint, readErr) p.mu.Unlock() + p.writeThrough(readCtx, key, delta) + if err := ctx.Err(); err != nil { return errors.Wrap(ctx, err, "refresh file") } diff --git a/pkg/index/store.go b/pkg/index/store.go new file mode 100644 index 0000000..7fcbdba --- /dev/null +++ b/pkg/index/store.go @@ -0,0 +1,453 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file adds the on-disk page index store: a metadata-only cache of the +// page index outside every vault. It is a cache only — a store that is +// missing, empty, damaged, unreadable or written by a different writer is +// discarded, never served, and never stops the process from starting. + +package index + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "runtime/debug" + "sync/atomic" + "time" + + "github.com/bborbe/boltkv" + "github.com/bborbe/errors" + "github.com/bborbe/kv" + bolt "go.etcd.io/bbolt" +) + +// StoreFormatVersion is the on-disk layout version. Version 2 records carry no +// page body. A store written under a different version is discarded, never +// converted. +const StoreFormatVersion = 2 + +// storeOpenTimeout bounds the wait for the store file's lock. The zero value of +// bolt's Timeout waits forever, which would let a second process on the same +// cache path hang startup. +const storeOpenTimeout = 5 * time.Second + +// storeMetaBucketName holds the writer identity. It carries no NUL-separated +// key prefix, so it can never collide with an entry bucket. +const storeMetaBucketName = "vault-ui-page-index-meta" + +// storeBucketPrefix starts every entry bucket name. The NUL bytes around the +// key parts cannot appear in a path, so two keys cannot produce one bucket. +const storeBucketPrefix = "vault-ui-page-index\x00" + +// storeIdentityKey is the meta-bucket key holding the JSON writer identity. +const storeIdentityKey = "identity" + +// vaultCLIModulePath is the dependency whose version identifies the parser that +// produced a stored page. +const vaultCLIModulePath = "github.com/bborbe/vault-cli" + +// unknownParserVersion is reported when the parser version cannot be read. +const unknownParserVersion = "(unknown)" + +// WriterIdentity identifies the process that wrote the store: the store-format +// version plus the vault-cli parser version this process was built against. +type WriterIdentity struct { + StoreFormat int + ParserVersion string +} + +//counterfeiter:generate -o ../../mocks/index-store.go --fake-name IndexStore . Store + +// Store persists the parsed page index on local disk. It is a cache: a Load +// that cannot supply entries never fails the caller, it reports them absent. +type Store interface { + // Load returns the key's stored entries. ok is false when there is nothing + // usable to load for the key, which is the full-parse path. err is non-nil + // only when ctx was cancelled. + Load(ctx context.Context, key Key) (entries []StoredEntry, ok bool, err error) + // Write applies one publication's delta for the key in a single + // transaction: puts replaces or inserts entries keyed by Filename, deletes + // removes them. + Write(ctx context.Context, key Key, puts []StoredEntry, deletes []string) error + // Path returns the store file path, for logging and warnings. + Path() string + // Close releases the store file. + Close() error +} + +// DefaultStorePath returns //page-index.bolt. It is +// a pure path helper and creates nothing. +// +// It has no context parameter by contract, so the single stdlib lookup's error +// is returned unwrapped. +func DefaultStorePath(app string) (string, error) { + cacheDir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(cacheDir, app, "page-index.bolt"), nil +} + +// CurrentWriterIdentity returns this process's writer identity. +func CurrentWriterIdentity(ctx context.Context) WriterIdentity { + return WriterIdentity{ + StoreFormat: StoreFormatVersion, + ParserVersion: vaultCLIParserVersion(), + } +} + +// vaultCLIParserVersion reads the vault-cli version this binary was built +// against: the dependency entry when vault-cli is a dependency, the main +// module when vault-cli is itself main, else a marker. +func vaultCLIParserVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return unknownParserVersion + } + for _, dep := range info.Deps { + if dep.Path == vaultCLIModulePath { + return dep.Version + } + } + if info.Main.Path == vaultCLIModulePath && info.Main.Version != "" { + return info.Main.Version + } + return unknownParserVersion +} + +// NewBoltStore opens the embedded-Bolt store at path under the given writer +// identity. It never returns nil and never fails. +func NewBoltStore( + ctx context.Context, + path string, + identity WriterIdentity, + warnf func(format string, args ...any), + vlogf func(format string, args ...any), +) Store { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return &discardedStore{path: path, reason: "create store directory", warnf: warnf} + } + + db, err := boltkv.OpenFile(ctx, path, func(opts *bolt.Options) { + opts.Timeout = storeOpenTimeout + }) + if err != nil { + return &discardedStore{path: path, reason: "open store", warnf: warnf} + } + + store := &boltStore{ + db: db, + path: path, + identity: identity, + warnf: warnf, + vlogf: vlogf, + } + if err := store.reconcile(ctx); err != nil { + if closeErr := db.Close(); closeErr != nil { + slog.Debug("close unusable page index store", "path", path, "error", closeErr) + } + return &discardedStore{path: path, reason: "record writer identity", warnf: warnf} + } + + keys, entries := store.count(ctx) + store.empty.Store(keys == 0) + vlogf("page index store %s: %d keys, %d entries", path, keys, entries) + return store +} + +// boltStore is the live embedded-Bolt store. +type boltStore struct { + db boltkv.DB + path string + identity WriterIdentity + // empty is true while the store holds no entry bucket, so a Load can answer + // without a transaction. + empty atomic.Bool + warnf func(format string, args ...any) + vlogf func(format string, args ...any) +} + +// reconcile reads the stored writer identity and either keeps the store or +// discards its entries, then records the current identity. It runs in one +// transaction. Format is checked before parser version, so a store differing +// in both reports a format mismatch. +func (s *boltStore) reconcile(ctx context.Context) error { + return s.db.Update(ctx, func(ctx context.Context, tx kv.Tx) error { + meta, err := tx.CreateBucketIfNotExists(ctx, kv.NewBucketName(storeMetaBucketName)) + if err != nil { + return errors.Wrap(ctx, err, "create store meta bucket") + } + item, err := meta.Get(ctx, []byte(storeIdentityKey)) + if err != nil { + return errors.Wrap(ctx, err, "read store writer identity") + } + var raw []byte + if err := item.Value(func(value []byte) error { + raw = append([]byte(nil), value...) + return nil + }); err != nil { + return errors.Wrap(ctx, err, "read store writer identity") + } + + reason := s.discardReason(raw) + if reason == "" { + return nil + } + s.warnf("page index store discarded (%s): %s", reason, s.path) + if err := deleteEntryBuckets(ctx, tx); err != nil { + return err + } + encoded, err := json.Marshal(s.identity) + if err != nil { + return errors.Wrap(ctx, err, "marshal store writer identity") + } + if err := meta.Put(ctx, []byte(storeIdentityKey), encoded); err != nil { + return errors.Wrap(ctx, err, "write store writer identity") + } + return nil + }) +} + +// discardReason returns why a stored identity cannot be used, or "" when it +// matches this process's identity. +func (s *boltStore) discardReason(raw []byte) string { + if len(raw) == 0 { + return "store file missing or empty" + } + var existing WriterIdentity + if err := json.Unmarshal(raw, &existing); err != nil { + return "damaged" + } + switch { + case existing.StoreFormat != s.identity.StoreFormat: + return fmt.Sprintf( + "store format mismatch (stored %d, current %d)", + existing.StoreFormat, + s.identity.StoreFormat, + ) + case existing.ParserVersion != s.identity.ParserVersion: + return "parser version mismatch" + } + return "" +} + +// deleteEntryBuckets removes every bucket except the meta bucket. +func deleteEntryBuckets(ctx context.Context, tx kv.Tx) error { + names, err := tx.ListBucketNames(ctx) + if err != nil { + return errors.Wrap(ctx, err, "list store buckets") + } + for _, name := range names { + if name.String() == storeMetaBucketName { + continue + } + if err := tx.DeleteBucket(ctx, name); err != nil { + return errors.Wrapf(ctx, err, "delete store bucket %s", name.String()) + } + } + return nil +} + +// count reports the number of entry buckets and the total number of stored +// entries. It runs once at startup. +func (s *boltStore) count(ctx context.Context) (int, int) { + keys, entries := 0, 0 + err := s.db.View(ctx, func(ctx context.Context, tx kv.Tx) error { + names, err := tx.ListBucketNames(ctx) + if err != nil { + return err + } + for _, name := range names { + if name.String() == storeMetaBucketName { + continue + } + bucket, err := tx.Bucket(ctx, name) + if err != nil { + return err + } + keys++ + entries += bucketEntryCount(bucket) + } + return nil + }) + if err != nil { + return 0, 0 + } + return keys, entries +} + +// bucketEntryCount counts the items in one bucket by iterating it. +func bucketEntryCount(bucket kv.Bucket) int { + count := 0 + iterator := bucket.Iterator() + defer iterator.Close() + for iterator.Rewind(); iterator.Valid(); iterator.Next() { + count++ + } + return count +} + +// Load returns the key's stored entries. Any failure to supply them is a +// discard: it warns once and reports the key absent, so the caller full-parses. +func (s *boltStore) Load(ctx context.Context, key Key) ([]StoredEntry, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, errors.Wrap(ctx, err, "load page index store") + } + if s.empty.Load() { + return nil, false, nil + } + + var entries []StoredEntry + err := s.db.View(ctx, func(ctx context.Context, tx kv.Tx) error { + bucket, err := tx.Bucket(ctx, storeBucketName(key)) + if err != nil { + return err + } + return kv.ForEach(ctx, bucket, func(item kv.Item) error { + return item.Value(func(value []byte) error { + entry, err := decodeRecord(ctx, value) + if err != nil { + return err + } + // Append rather than preallocate from a stored count, so a + // crafted count cannot drive an unbounded allocation. + entries = append(entries, entry) + return nil + }) + }) + }) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, false, errors.Wrap(ctx, ctxErr, "load page index store") + } + if errors.Is(err, kv.BucketNotFoundError) { + s.warnf("page index store discarded (no stored entries for key): %s", s.path) + return nil, false, nil + } + s.warnf("page index store discarded (damaged): %s", s.path) + return nil, false, nil + } + return entries, true, nil +} + +// Write applies one publication's delta in a single transaction. A failed write +// leaves the previous content intact; the caller owns the failure warning. +func (s *boltStore) Write( + ctx context.Context, + key Key, + puts []StoredEntry, + deletes []string, +) error { + err := s.db.Update(ctx, func(ctx context.Context, tx kv.Tx) error { + if err := s.writeIdentity(ctx, tx); err != nil { + return err + } + bucket, err := tx.CreateBucketIfNotExists(ctx, storeBucketName(key)) + if err != nil { + return errors.Wrap(ctx, err, "create store entry bucket") + } + return applyDelta(ctx, bucket, puts, deletes) + }) + if err != nil { + return errors.Wrap(ctx, err, "write page index store") + } + s.empty.Store(false) + return nil +} + +// writeIdentity records this process's writer identity in the meta bucket. +func (s *boltStore) writeIdentity(ctx context.Context, tx kv.Tx) error { + meta, err := tx.CreateBucketIfNotExists(ctx, kv.NewBucketName(storeMetaBucketName)) + if err != nil { + return errors.Wrap(ctx, err, "create store meta bucket") + } + encoded, err := json.Marshal(s.identity) + if err != nil { + return errors.Wrap(ctx, err, "marshal store writer identity") + } + if err := meta.Put(ctx, []byte(storeIdentityKey), encoded); err != nil { + return errors.Wrap(ctx, err, "write store writer identity") + } + return nil +} + +// applyDelta puts and deletes one publication's entries in a bucket. +func applyDelta( + ctx context.Context, + bucket kv.Bucket, + puts []StoredEntry, + deletes []string, +) error { + for _, entry := range puts { + value, err := encodeRecord(ctx, entry) + if err != nil { + return err + } + if err := bucket.Put(ctx, []byte(entry.Filename), value); err != nil { + return errors.Wrapf(ctx, err, "put store entry %s", entry.Filename) + } + } + for _, name := range deletes { + if err := bucket.Delete(ctx, []byte(name)); err != nil { + return errors.Wrapf(ctx, err, "delete store entry %s", name) + } + } + return nil +} + +// Path returns the store file path. +func (s *boltStore) Path() string { + return s.path +} + +// Close releases the store file. +func (s *boltStore) Close() error { + return s.db.Close() +} + +// storeBucketName returns the entry bucket name for one key. +func storeBucketName(key Key) kv.BucketName { + return kv.NewBucketName(storeBucketPrefix + key.VaultPath + "\x00" + key.PagesDir) +} + +// discardedStore stands in for a store that could not be opened. It serves +// nothing, warns exactly once per Load, and fails Write so the caller warns. +type discardedStore struct { + path string + reason string + warnf func(format string, args ...any) +} + +// Load warns once and reports the key absent. +func (d *discardedStore) Load( + ctx context.Context, + key Key, +) ([]StoredEntry, bool, error) { + d.warnf("page index store discarded (%s): %s", d.reason, d.path) + return nil, false, nil +} + +// Write fails; the caller owns the warning. +func (d *discardedStore) Write( + ctx context.Context, + key Key, + puts []StoredEntry, + deletes []string, +) error { + return errors.Errorf(ctx, "page index store discarded (%s): %s", d.reason, d.path) +} + +// Path returns the intended store file path. +func (d *discardedStore) Path() string { + return d.path +} + +// Close is a no-op: nothing was opened. +func (d *discardedStore) Close() error { + return nil +} diff --git a/pkg/index/store_codec.go b/pkg/index/store_codec.go new file mode 100644 index 0000000..d24836d --- /dev/null +++ b/pkg/index/store_codec.go @@ -0,0 +1,157 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index + +import ( + "bytes" + "context" + "encoding/gob" + "time" + + "github.com/bborbe/errors" + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" +) + +// storeRecord is the on-disk shape of one stored entry. It holds no page body. +// It is the gob codec's own type: domain.Page is deliberately not encoded +// directly, because its +// embedded FrontmatterMap has only an unexported field and both gob and JSON +// would silently drop every frontmatter key. The page is rebuilt from +// FrontmatterMap.RawMap() on the way in and domain.NewPage on the way out. +type storeRecord struct { + Filename string + HasPage bool + Frontmatter map[string]any + Metadata domain.FileMetadata + OpenQuestions []domain.OpenQuestion + Fingerprint FileFingerprint +} + +// init registers every concrete type that can appear as a value in a decoded +// YAML frontmatter map. gob encodes interface values with their concrete type +// name, so an unregistered value makes the encode fail. The list is the YAML +// decoder's value set: scalars, sequences, string-keyed maps, and the generic +// map[any]any a nested mapping with non-string keys produces. A new type in a +// fixture fails the codec test loudly; extend this list rather than dropping +// the value. +// +// libtime.DateOrDateTime is deliberately NOT registered: it implements +// BinaryMarshaler but not BinaryUnmarshaler, so gob encodes it and then fails +// to decode. encodeValue converts it to time.Time first. +func init() { + gob.Register("") + gob.Register(false) + gob.Register(int(0)) + gob.Register(int64(0)) + gob.Register(float64(0)) + gob.Register(time.Time{}) + gob.Register([]any{}) + gob.Register(map[string]any{}) + gob.Register(map[any]any{}) +} + +// encodeRecord encodes one entry into its stored bytes. +func encodeRecord(ctx context.Context, entry StoredEntry) ([]byte, error) { + record := storeRecord{ + Filename: entry.Filename, + Fingerprint: entry.Fingerprint, + } + if entry.Entry != nil { + converted, err := encodeValue(ctx, entry.Entry.Page.RawMap()) + if err != nil { + return nil, err + } + frontmatter, ok := converted.(map[string]any) + if !ok { + return nil, errors.Errorf( + ctx, + "encode page index record %s: unexpected frontmatter type %T", + entry.Filename, + converted, + ) + } + record.HasPage = true + record.Frontmatter = frontmatter + record.Metadata = entry.Entry.Page.FileMetadata + record.OpenQuestions = entry.Entry.OpenQuestions + } + + var buffer bytes.Buffer + if err := gob.NewEncoder(&buffer).Encode(record); err != nil { + return nil, errors.Wrapf(ctx, err, "encode page index record %s", entry.Filename) + } + return buffer.Bytes(), nil +} + +// decodeRecord decodes one entry's stored bytes. A decode error is returned, +// never a partially-filled record. +func decodeRecord(ctx context.Context, data []byte) (StoredEntry, error) { + var record storeRecord + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&record); err != nil { + return StoredEntry{}, errors.Wrap(ctx, err, "decode page index record") + } + + entry := StoredEntry{ + Filename: record.Filename, + Fingerprint: record.Fingerprint, + } + if record.HasPage { + questions := record.OpenQuestions + if questions == nil { + questions = []domain.OpenQuestion{} + } + entry.Entry = &Entry{ + Page: domain.NewPage(record.Frontmatter, record.Metadata, ""), + OpenQuestions: questions, + } + } + return entry, nil +} + +// encodeValue converts a frontmatter value into a gob-encodable one. Only +// libtime.DateOrDateTime needs rewriting: gob cannot decode it, so it becomes +// the time.Time it wraps. Maps and slices are walked so a nested value is +// converted too; every other value is returned unchanged. +func encodeValue(ctx context.Context, value any) (any, error) { + switch typed := value.(type) { + case libtime.DateOrDateTime: + return typed.Time(), nil + case *libtime.DateOrDateTime: + if typed == nil { + return nil, nil + } + return typed.Time(), nil + case map[string]any: + return encodeMap(ctx, typed) + case map[any]any: + return encodeMap(ctx, typed) + case []any: + converted := make([]any, len(typed)) + for index, item := range typed { + value, err := encodeValue(ctx, item) + if err != nil { + return nil, err + } + converted[index] = value + } + return converted, nil + default: + return value, nil + } +} + +// encodeMap converts every value of a frontmatter map with encodeValue. +func encodeMap[K comparable](ctx context.Context, typed map[K]any) (map[K]any, error) { + converted := make(map[K]any, len(typed)) + for key, item := range typed { + value, err := encodeValue(ctx, item) + if err != nil { + return nil, err + } + converted[key] = value + } + return converted, nil +} diff --git a/pkg/index/store_hydration_test.go b/pkg/index/store_hydration_test.go new file mode 100644 index 0000000..99d7185 --- /dev/null +++ b/pkg/index/store_hydration_test.go @@ -0,0 +1,430 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/mocks" + "github.com/bborbe/vault-cli/pkg/index" +) + +// newStore opens a real store at a temp path and closes it when the spec ends. +func newHydrationStore(t GinkgoTInterface) index.Store { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "page-index.bolt") + warns := &warnSink{} + store := index.NewBoltStore( + ctx, + path, + index.CurrentWriterIdentity(ctx), + warns.warnf, + func(format string, args ...any) {}, + ) + DeferCleanup(func() { + Expect(store.Close()).To(Succeed()) + }) + return store +} + +// populateStore lists the folder with the production seams and writes one +// stored entry per listed file, standing in for prompt 3's write-through. Every +// listed file is stored, including the excluded ones, which are stored as +// fingerprint-only entries so an unchanged excluded file is neither re-read nor +// re-warned on the next start. +func populateStore( + ctx context.Context, + store index.Store, + key index.Key, + vaultDir string, + folder string, +) { + lister := index.NewDirectoryLister() + reader := index.NewPageReader(storage.NewPageStorage(nil)) + entries, err := lister.ListFiles(ctx, vaultDir, folder) + Expect(err).NotTo(HaveOccurred()) + + puts := make([]index.StoredEntry, 0, len(entries)) + for _, entry := range entries { + read, fingerprint, readErr := reader.ReadEntry(ctx, vaultDir, folder, entry.Name) + stored := index.StoredEntry{Filename: entry.Name, Fingerprint: fingerprint} + if readErr == nil { + stored.Entry = read + } + puts = append(puts, stored) + } + Expect(store.Write(ctx, key, puts, nil)).To(Succeed()) +} + +// newStoreIndex builds a store-backed index with captured warnings. +func newStoreIndex( + store index.Store, + reader index.PageReader, + lister index.DirectoryLister, + warns *warnSink, +) index.PageIndex { + return index.NewPageIndexWithStoreAndWarnf( + reader, lister, libtime.NewCurrentDateTime(), fastWaiter(), store, warns.warnf, + ) +} + +// storeStart is one process start: fresh seams over a shared store. +type storeStart struct { + reader *recordingReader + lister *recordingLister + idx index.PageIndex +} + +// newStoreStart builds a fresh start over the store with counting seams. +func newStoreStart(store index.Store) *storeStart { + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + return &storeStart{ + reader: reader, + lister: lister, + idx: newStoreIndex(store, reader, lister, &warnSink{}), + } +} + +// newStoreVault builds a real vault folder holding n parseable pages named +// Page00.md upward, so their names sort in listing order. +func newStoreVault(n int) string { + vaultDir := GinkgoT().TempDir() + Expect(os.MkdirAll( + filepath.Join(vaultDir, equivalenceFolder), 0750, + )).To(Succeed()) + for i := 0; i < n; i++ { + writeFixtureFile(vaultDir, fmt.Sprintf("Page%02d.md", i), fmt.Sprintf( + "---\ntitle: Page%02d\n---\n# Page%02d\n", i, i, + )) + } + return vaultDir +} + +// blockingLister blocks every listing until the test closes release. +type blockingLister struct { + inner index.DirectoryLister + release chan struct{} +} + +func (l *blockingLister) ListFiles( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]index.FileEntry, error) { + <-l.release + return l.inner.ListFiles(ctx, vaultPath, pagesDir) +} + +// togglableLister fails every listing while err is set. +type togglableLister struct { + inner index.DirectoryLister + + mu sync.Mutex + err error +} + +func (l *togglableLister) ListFiles( + ctx context.Context, + vaultPath string, + pagesDir string, +) ([]index.FileEntry, error) { + l.mu.Lock() + err := l.err + l.mu.Unlock() + if err != nil { + return nil, err + } + return l.inner.ListFiles(ctx, vaultPath, pagesDir) +} + +func (l *togglableLister) setErr(err error) { + l.mu.Lock() + defer l.mu.Unlock() + l.err = err +} + +// listResult carries one ListPages outcome out of a goroutine. +type listResult struct { + pages []*domain.Page + err error +} + +var _ = Describe("Store hydration", func() { + It("AC1: the store round-trip is lossless", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + key := index.NewKey(vaultDir, equivalenceFolder) + + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + second := newStoreStart(store) + got, err := second.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + + want := referenceListing(ctx, vaultDir) + // Positive control: a fixture that silently excluded everything could + // not pass this. + Expect(indexedNames(want)).To(Equal([]string{"Inside", "Plain", "Wikilink", "Ünïcode"})) + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + Expect(second.reader.Count()).To(Equal(0)) + Expect(second.lister.Count()).To(Equal(1)) + + // Change a file that is not the fixture's in-vault symlink target: + // statFingerprint follows symlinks, so rewriting Plain.md would change + // Inside.md's fingerprint too and the third start would read 2 files. + settle() + writeFixtureFile( + vaultDir, + "Wikilink.md", + "---\ntitle: Wikilink\nrelated: [[Other]]\n---\n# Wikilink v2\n", + ) + + third := newStoreStart(store) + got, err = third.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(third.reader.Names()).To(Equal([]string{"Wikilink.md"})) + + want = referenceListing(ctx, vaultDir) + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + }) + + It("AC2: a stored snapshot is never served before its stat-diff", func() { + ctx := context.Background() + vaultDir := buildEquivalenceVault() + key := index.NewKey(vaultDir, equivalenceFolder) + + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &blockingLister{ + inner: index.NewDirectoryLister(), + release: make(chan struct{}), + } + idx := newStoreIndex(store, reader, lister, &warnSink{}) + + done := make(chan listResult, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + done <- listResult{pages: pages, err: err} + }() + + // The stored snapshot exists from the store read, but the read must not + // return while the stat-diff's listing is blocked. + Consistently(func() int { return len(done) }, "150ms").Should(Equal(0)) + + close(lister.release) + var served listResult + Eventually(done, "2s").Should(Receive(&served)) + Expect(served.err).NotTo(HaveOccurred()) + Expect(served.pages).To(HaveLen(4)) + Expect(reader.Count()).To(Equal(0)) + }) + + It("AC2 control: a key with no stored entries blocks on the full parse", func() { + ctx := context.Background() + store := newHydrationStore(GinkgoT()) + // A different vault dir has no stored entries for its key. + controlDir := newStoreVault(3) + + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &blockingLister{ + inner: index.NewDirectoryLister(), + release: make(chan struct{}), + } + idx := newStoreIndex(store, reader, lister, &warnSink{}) + + done := make(chan listResult, 1) + go func() { + defer GinkgoRecover() + pages, err := idx.ListPages(ctx, controlDir, equivalenceFolder) + done <- listResult{pages: pages, err: err} + }() + + Consistently(func() int { return len(done) }, "150ms").Should(Equal(0)) + + close(lister.release) + var served listResult + Eventually(done, "2s").Should(Receive(&served)) + Expect(served.err).NotTo(HaveOccurred()) + + want := referenceListing(ctx, controlDir) + Expect(reflect.DeepEqual(served.pages, want)).To(BeTrue()) + Expect(reader.Count()).To(Equal(3)) + }) + + It("AC3(a): an unchanged folder lists once and reads nothing", func() { + ctx := context.Background() + vaultDir := newStoreVault(6) + key := index.NewKey(vaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + start := newStoreStart(store) + before := index.FilesReadTotal("build") + pages, err := start.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(indexedNames(pages)).To(HaveLen(6)) + Expect(start.lister.Count()).To(Equal(1)) + Expect(start.reader.Count()).To(Equal(0)) + Expect(index.FilesReadTotal("build") - before).To(Equal(float64(0))) + }) + + It("AC3(b): re-reads exactly the K changed files", func() { + ctx := context.Background() + vaultDir := newStoreVault(6) + key := index.NewKey(vaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + changed := []string{"Page01.md", "Page03.md", "Page05.md"} + settle() + for i, name := range changed { + writeFixtureFile(vaultDir, name, fmt.Sprintf( + "---\ntitle: Changed%d\n---\n# Changed%d\n", i, i, + )) + } + + start := newStoreStart(store) + before := index.FilesReadTotal("build") + pages, err := start.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(pages).To(HaveLen(6)) + Expect(start.lister.Count()).To(Equal(1)) + Expect(start.reader.Names()).To(ConsistOf(changed)) + Expect(index.FilesReadTotal("build") - before).To(Equal(float64(3))) + }) + + It("AC3(c): re-reads only the added file and drops the removed one", func() { + ctx := context.Background() + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + settle() + writeFixtureFile(vaultDir, "Page05.md", "---\ntitle: Page05\n---\n# Page05\n") + Expect(os.Remove(pageFile(vaultDir, "Page04.md"))).To(Succeed()) + + start := newStoreStart(store) + before := index.FilesReadTotal("build") + pages, err := start.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(start.reader.Names()).To(Equal([]string{"Page05.md"})) + Expect(indexedNames(pages)).To(ConsistOf( + "Page00", "Page01", "Page02", "Page03", "Page05", + )) + + want := referenceListing(ctx, vaultDir) + Expect(reflect.DeepEqual(pages, want)).To(BeTrue()) + Expect(index.FilesReadTotal("build") - before).To(Equal(float64(1))) + }) + + It("AC3(d): detects a same-size rewrite with a restored modification time", func() { + ctx := context.Background() + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + target := pageFile(vaultDir, "Page02.md") + info, err := os.Stat(target) + Expect(err).NotTo(HaveOccurred()) + + settle() + original := "---\ntitle: Page02\n---\n# Page02\n" + replacement := "---\ntitle: Page99\n---\n# Page02\n" + Expect(len(replacement)).To(Equal(len(original))) + writeFixtureFile(vaultDir, "Page02.md", replacement) + Expect(os.Chtimes(target, info.ModTime(), info.ModTime())).To(Succeed()) + + start := newStoreStart(store) + before := index.FilesReadTotal("build") + pages, err := start.idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(start.reader.Names()).To(Equal([]string{"Page02.md"})) + Expect(index.FilesReadTotal("build") - before).To(Equal(float64(1))) + Expect(pages[2].GetString("title")).To(Equal("Page99")) + }) + + It("AC4: falls back to a full parse when the store supplies nothing", func() { + ctx := context.Background() + vaultDir := newStoreVault(3) + + fake := &mocks.IndexStore{} + fake.LoadReturns(nil, false, nil) + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + warns := &warnSink{} + idx := newStoreIndex(fake, reader, lister, warns) + + got, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + want := referenceListing(ctx, vaultDir) + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + Expect(reader.Count()).To(Equal(3)) + Expect(warns.naming("")).To(BeEmpty()) + }) + + It("AC4: serves correctly when the store read fails", func() { + ctx := context.Background() + vaultDir := newStoreVault(3) + + fake := &mocks.IndexStore{} + fake.LoadReturns(nil, false, errors.New("boom")) + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + warns := &warnSink{} + idx := newStoreIndex(fake, reader, lister, warns) + + got, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + want := referenceListing(ctx, vaultDir) + Expect(reflect.DeepEqual(got, want)).To(BeTrue()) + Expect(reader.Count()).To(Equal(3)) + Expect(warns.naming("")).To(BeEmpty()) + }) + + It("AC2/failure: a failed stat-diff of a store-loaded key publishes nothing", func() { + ctx := context.Background() + vaultDir := newStoreVault(4) + key := index.NewKey(vaultDir, equivalenceFolder) + store := newHydrationStore(GinkgoT()) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &togglableLister{inner: index.NewDirectoryLister()} + idx := newStoreIndex(store, reader, lister, &warnSink{}) + + lister.setErr(errors.New("listing boom")) + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).To(HaveOccurred()) + + // The next read takes the cold path, since the store was already + // consulted and the seeded baseline was discarded. + lister.setErr(nil) + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(indexedNames(pages)).To(HaveLen(4)) + + want := referenceListing(ctx, vaultDir) + Expect(reflect.DeepEqual(pages, want)).To(BeTrue()) + }) +}) diff --git a/pkg/index/store_test.go b/pkg/index/store_test.go new file mode 100644 index 0000000..b7bd64e --- /dev/null +++ b/pkg/index/store_test.go @@ -0,0 +1,608 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "time" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/domain" + "github.com/bborbe/vault-cli/pkg/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// cacheDirEnv points the user cache directory at dir so the real cache is +// never touched. linux reads XDG_CACHE_HOME, darwin reads HOME. +func cacheDirEnv(dir string) { + GinkgoT().Setenv("XDG_CACHE_HOME", dir) + GinkgoT().Setenv("HOME", dir) +} + +// vlogSink captures the store's V(2) lines. +type vlogSink struct { + mu sync.Mutex + lines []string +} + +func (v *vlogSink) vlogf(format string, args ...any) { + v.mu.Lock() + defer v.mu.Unlock() + v.lines = append(v.lines, fmt.Sprintf(format, args...)) +} + +func (v *vlogSink) all() []string { + v.mu.Lock() + defer v.mu.Unlock() + return append([]string(nil), v.lines...) +} + +// currentIdentity is this process's writer identity. +func currentIdentity() index.WriterIdentity { + return index.CurrentWriterIdentity(context.Background()) +} + +// newStoreAt opens a store at an explicit path with explicit seams. +func newStoreAt( + path string, + identity index.WriterIdentity, + warns *warnSink, + vlogs *vlogSink, +) index.Store { + return index.NewBoltStore( + context.Background(), path, identity, warns.warnf, vlogs.vlogf, + ) +} + +// newStore opens a store at path with this process's identity. +func newStore(path string, warns *warnSink, vlogs *vlogSink) index.Store { + return newStoreAt(path, currentIdentity(), warns, vlogs) +} + +// closeStore closes a store, ignoring the close error in test cleanup. +func closeStore(store index.Store) { + _ = store.Close() +} + +// readFixtureEntries reads every .md entry the production lister reports, +// returning the entries to store and the fingerprint each read reported. +func readFixtureEntries( + ctx context.Context, + vaultDir string, +) ([]index.StoredEntry, map[string]index.FileFingerprint) { + lister := index.NewDirectoryLister() + entries, err := lister.ListFiles(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).NotTo(BeEmpty()) + + reader := index.NewPageReader(storage.NewPageStorage(nil)) + puts := make([]index.StoredEntry, 0, len(entries)) + fingerprints := make(map[string]index.FileFingerprint, len(entries)) + for _, entry := range entries { + read, fingerprint, readErr := reader.ReadEntry( + ctx, vaultDir, equivalenceFolder, entry.Name, + ) + fingerprints[entry.Name] = fingerprint + if readErr != nil { + puts = append(puts, index.StoredEntry{ + Filename: entry.Name, + Fingerprint: fingerprint, + }) + continue + } + puts = append(puts, index.StoredEntry{ + Filename: entry.Name, + Entry: read, + Fingerprint: fingerprint, + }) + } + return puts, fingerprints +} + +// simpleEntry builds a stored entry with a minimal page. +func simpleEntry(name, title string) index.StoredEntry { + return index.StoredEntry{ + Filename: name, + Entry: &index.Entry{ + Page: domain.NewPage(map[string]any{"title": title}, domain.FileMetadata{Name: strings.TrimSuffix(name, ".md")}, ""), + OpenQuestions: []domain.OpenQuestion{}, + }, + Fingerprint: index.FileFingerprint{Size: int64(len(name))}, + } +} + +var _ = Describe("Page index store", func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + Describe("codec", func() { + It("AC1 round-trips every awkward file losslessly", func() { + vaultDir := buildEquivalenceVault() + key := index.NewKey(vaultDir, equivalenceFolder) + puts, fingerprints := readFixtureEntries(ctx, vaultDir) + + warns := &warnSink{} + vlogs := &vlogSink{} + store := newStore(filepath.Join(GinkgoT().TempDir(), "page-index.bolt"), warns, vlogs) + defer closeStore(store) + + Expect(store.Write(ctx, key, puts, nil)).To(Succeed()) + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(len(puts))) + + gotByName := make(map[string]index.StoredEntry, len(loaded)) + pageNames := make([]string, 0, len(loaded)) + for _, entry := range loaded { + gotByName[entry.Filename] = entry + if entry.Entry != nil { + pageNames = append(pageNames, entry.Filename) + } + } + // Positive control: a fixture that silently excluded everything + // could not produce this ordered non-empty list. + Expect(pageNames).To(Equal( + []string{"Inside.md", "Plain.md", "Wikilink.md", "Ünïcode.md"}, + )) + + reference := storage.NewPageStorage(nil) + for _, entry := range puts { + got, found := gotByName[entry.Filename] + Expect(found).To(BeTrue(), "missing stored entry for %s", entry.Filename) + Expect(got.Fingerprint).To(Equal(fingerprints[entry.Filename])) + + want, readErr := reference.ReadPage( + ctx, vaultDir, equivalenceFolder, strings.TrimSuffix(entry.Filename, ".md"), + ) + if readErr != nil { + Expect(got.Entry).To(BeNil(), "excluded %s must store no page", entry.Filename) + continue + } + Expect(got.Entry).NotTo(BeNil(), "readable %s must store a page", entry.Filename) + Expect(got.Entry.OpenQuestions).To(Equal(entry.Entry.OpenQuestions)) + want.Content = "" + Expect(reflect.DeepEqual(got.Entry.Page, want)).To( + BeTrue(), "%s page is not losslessly round-tripped", entry.Filename, + ) + } + }) + + It("converts a DateOrDateTime frontmatter value to time.Time", func() { + key := index.NewKey("/vault-a", "24 Tasks") + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + store := newStore(path, &warnSink{}, &vlogSink{}) + defer closeStore(store) + + date := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + page := domain.NewPage(map[string]any{ + "defer_date": libtime.DateOrDateTime(date), + "nested": map[string]any{"due_date": libtime.DateOrDateTime(date)}, + "list": []any{"a", libtime.DateOrDateTime(date)}, + "generic": map[any]any{"k": "v"}, + "title": "Dates", + }, domain.FileMetadata{Name: "Dates"}, "") + Expect(store.Write(ctx, key, []index.StoredEntry{ + {Filename: "Dates.md", Entry: &index.Entry{Page: page, OpenQuestions: []domain.OpenQuestion{}}}, + }, nil)).To(Succeed()) + + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + + want := domain.NewPage(map[string]any{ + "defer_date": date, + "nested": map[string]any{"due_date": date}, + "list": []any{"a", date}, + "generic": map[any]any{"k": "v"}, + "title": "Dates", + }, domain.FileMetadata{Name: "Dates"}, "") + Expect(reflect.DeepEqual(loaded[0].Entry.Page, want)).To( + BeTrue(), "DateOrDateTime was not converted to time.Time", + ) + }) + }) + + Describe("discard on mismatch", func() { + keyA := index.NewKey("/vault-a", "24 Tasks") + keyB := index.NewKey("/vault-b", "24 Tasks") + + It("AC4(a) treats a store file that never existed as absent", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + + entries, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(entries).To(BeNil()) + Expect(warns.naming("store file missing or empty")).To(HaveLen(1)) + Expect(warns.naming(path)).To(HaveLen(1)) + }) + + It("AC4(b) opens a zero-byte store file empty without panicking", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + Expect(os.WriteFile(path, nil, 0600)).To(Succeed()) + + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming("store file missing or empty")).To(HaveLen(1)) + }) + + It("AC4(c) discards a store file holding garbage bytes", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + Expect(os.WriteFile(path, []byte("not a bolt database at all"), 0600)).To(Succeed()) + + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + // A discarded store warns only from Load, never from the open. + Expect(warns.naming("open store")).To(BeEmpty()) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming("open store")).To(HaveLen(1)) + Expect(warns.naming(path)).To(HaveLen(1)) + Expect(store.Path()).To(Equal(path)) + Expect(store.Write(ctx, keyA, nil, nil)).To(HaveOccurred()) + }) + + It("discards a store whose directory cannot be created", func() { + blocker := filepath.Join(GinkgoT().TempDir(), "blocker") + Expect(os.WriteFile(blocker, []byte("x"), 0600)).To(Succeed()) + path := filepath.Join(blocker, "nested", "page-index.bolt") + + warns := &warnSink{} + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming("create store directory")).To(HaveLen(1)) + }) + + It("AC4(d) discards a store file whose read permission was removed", func() { + if os.Geteuid() == 0 { + GinkgoT().Skip("a root process can still read a 0000 file") + } + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + valid := newStore(path, &warnSink{}, &vlogSink{}) + Expect(valid.Write(ctx, keyA, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + Expect(valid.Close()).To(Succeed()) + Expect(os.Chmod(path, 0)).To(Succeed()) + defer func() { _ = os.Chmod(path, 0600) }() + + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming(path)).To(HaveLen(1)) + }) + + It("AC4(e) discards a store written under a different store format", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + + first := newStoreAt( + path, + index.WriterIdentity{StoreFormat: 1, ParserVersion: "x"}, + &warnSink{}, &vlogSink{}, + ) + Expect(first.Write(ctx, keyA, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + Expect(first.Close()).To(Succeed()) + + store := newStoreAt( + path, + index.WriterIdentity{StoreFormat: 2, ParserVersion: "x"}, + warns, &vlogSink{}, + ) + defer closeStore(store) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming("store format mismatch (stored 1, current 2)")).To(HaveLen(1)) + Expect(warns.naming(path)).To(HaveLen(1)) + + // The mismatch wipes and takes the full parse; the store is usable. + Expect(store.Write(ctx, keyA, []index.StoredEntry{simpleEntry("B.md", "B")}, nil)). + To(Succeed()) + loaded, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + Expect(loaded[0].Filename).To(Equal("B.md")) + }) + + It("AC4(f) discards a store written by a different parser version", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + + first := newStoreAt( + path, + index.WriterIdentity{StoreFormat: 1, ParserVersion: "v0.1.0"}, + &warnSink{}, &vlogSink{}, + ) + Expect(first.Write(ctx, keyA, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + Expect(first.Close()).To(Succeed()) + + store := newStoreAt( + path, + index.WriterIdentity{StoreFormat: 1, ParserVersion: "v0.2.0"}, + warns, &vlogSink{}, + ) + defer closeStore(store) + + _, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(warns.naming("parser version mismatch")).To(HaveLen(1)) + Expect(warns.naming("store format mismatch")).To(BeEmpty()) + }) + + It("AC4(g) serves one key while another full-parses", func() { + warns := &warnSink{} + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + store := newStore(path, warns, &vlogSink{}) + defer closeStore(store) + + Expect(store.Write(ctx, keyA, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + + loaded, ok, err := store.Load(ctx, keyA) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + Expect(warns.naming("no stored entries for key")).To(BeEmpty()) + + entries, ok, err := store.Load(ctx, keyB) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(entries).To(BeNil()) + missing := warns.naming("no stored entries for key") + Expect(missing).To(HaveLen(1)) + Expect(missing[0]).To(ContainSubstring(path)) + }) + }) + + Describe("write", func() { + var ( + key index.Key + path string + warns *warnSink + store index.Store + ) + + BeforeEach(func() { + key = index.NewKey("/vault-a", "24 Tasks") + path = filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + warns = &warnSink{} + store = newStore(path, warns, &vlogSink{}) + }) + + AfterEach(func() { + closeStore(store) + }) + + It("AC5 puts and deletes the delta in one transaction", func() { + Expect(store.Path()).To(Equal(path)) + Expect(store.Write(ctx, key, []index.StoredEntry{ + simpleEntry("A.md", "A"), + simpleEntry("B.md", "B"), + }, nil)).To(Succeed()) + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(2)) + + Expect(store.Write(ctx, key, nil, []string{"A.md"})).To(Succeed()) + loaded, ok, err = store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + Expect(loaded[0].Filename).To(Equal("B.md")) + }) + + It("AC5 leaves the previous content intact when a write fails", func() { + Expect(store.Write(ctx, key, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + + broken := index.StoredEntry{ + Filename: "Broken.md", + Entry: &index.Entry{Page: domain.NewPage( + map[string]any{"bad": make(chan int)}, + domain.FileMetadata{Name: "Broken"}, + "", + )}, + } + Expect(store.Write(ctx, key, []index.StoredEntry{broken}, nil)). + To(HaveOccurred()) + + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + Expect(loaded[0].Filename).To(Equal("A.md")) + }) + + It("AC5 makes a written delta observable through a second store", func() { + Expect(store.Write(ctx, key, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + Expect(store.Close()).To(Succeed()) + + secondWarns := &warnSink{} + second := newStore(path, secondWarns, &vlogSink{}) + defer closeStore(second) + + loaded, ok, err := second.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + Expect(loaded[0].Filename).To(Equal("A.md")) + }) + }) + + Describe("location", func() { + It("AC6 resolves the fixed cache path", func() { + cacheDir, err := os.UserCacheDir() + Expect(err).NotTo(HaveOccurred()) + + path, err := index.DefaultStorePath("vault-ui") + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(filepath.Join(cacheDir, "vault-ui", "page-index.bolt"))) + }) + + It("AC6 opens the default store at the cache path", func() { + cacheDir := GinkgoT().TempDir() + cacheDirEnv(cacheDir) + + expectedCacheDir, err := os.UserCacheDir() + Expect(err).NotTo(HaveOccurred()) + Expect(strings.HasPrefix(expectedCacheDir, cacheDir)).To(BeTrue()) + + key := index.NewKey("/vault-a", "24 Tasks") + warns := &warnSink{} + defaultPath, err := index.DefaultStorePath("vault-ui") + Expect(err).NotTo(HaveOccurred()) + store := index.NewBoltStore(ctx, defaultPath, index.CurrentWriterIdentity(ctx), warns.warnf, (&vlogSink{}).vlogf) + defer closeStore(store) + + Expect(store.Path()).To(Equal( + filepath.Join(expectedCacheDir, "vault-ui", "page-index.bolt"), + )) + Expect(store.Write(ctx, key, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(1)) + }) + + It("AC6 creates a missing store directory", func() { + path := filepath.Join(GinkgoT().TempDir(), "missing", "page-index.bolt") + store := newStore(path, &warnSink{}, &vlogSink{}) + defer closeStore(store) + + info, err := os.Stat(filepath.Dir(path)) + Expect(err).NotTo(HaveOccurred()) + Expect(info.IsDir()).To(BeTrue()) + _, err = os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + }) + + It("AC6 never writes the store under a vault directory", func() { + vaultDir := buildEquivalenceVault() + key := index.NewKey(vaultDir, equivalenceFolder) + puts, _ := readFixtureEntries(ctx, vaultDir) + + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + store := newStore(path, &warnSink{}, &vlogSink{}) + defer closeStore(store) + Expect(store.Write(ctx, key, puts, nil)).To(Succeed()) + + var bolts []string + Expect(filepath.WalkDir(vaultDir, func(p string, _ os.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasSuffix(p, ".bolt") { + bolts = append(bolts, p) + } + return nil + })).To(Succeed()) + Expect(bolts).To(BeEmpty()) + }) + }) + + Describe("reporting", func() { + It("AC7 warns nothing on a hit and reports keys and entries at V(2)", func() { + key := index.NewKey("/vault-a", "24 Tasks") + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + + first := newStore(path, &warnSink{}, &vlogSink{}) + Expect(first.Write(ctx, key, []index.StoredEntry{ + simpleEntry("A.md", "A"), + simpleEntry("B.md", "B"), + }, nil)).To(Succeed()) + Expect(first.Close()).To(Succeed()) + + warns := &warnSink{} + vlogs := &vlogSink{} + store := newStore(path, warns, vlogs) + defer closeStore(store) + + Expect(vlogs.all()).To(HaveLen(1)) + Expect(vlogs.all()[0]).To(ContainSubstring(path)) + Expect(vlogs.all()[0]).To(ContainSubstring("1 keys")) + Expect(vlogs.all()[0]).To(ContainSubstring("2 entries")) + + loaded, ok, err := store.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeTrue()) + Expect(loaded).To(HaveLen(2)) + Expect(warns.naming("")).To(BeEmpty()) + }) + }) + + Describe("bounded open", func() { + It("AC-safety times out instead of hanging on a locked store", NodeTimeout(30*time.Second), func(specCtx SpecContext) { + ctx = specCtx + key := index.NewKey("/vault-a", "24 Tasks") + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + + first := newStore(path, &warnSink{}, &vlogSink{}) + defer closeStore(first) + Expect(first.Write(ctx, key, []index.StoredEntry{simpleEntry("A.md", "A")}, nil)). + To(Succeed()) + + warns := &warnSink{} + var second index.Store + done := make(chan struct{}) + start := time.Now() + go func() { + defer GinkgoRecover() + defer close(done) + second = newStore(path, warns, &vlogSink{}) + }() + + Eventually(done, "20s").Should(BeClosed()) + Expect(time.Since(start)).To(BeNumerically("<", 15*time.Second)) + Expect(second).NotTo(BeNil()) + + entries, ok, err := second.Load(ctx, key) + Expect(err).NotTo(HaveOccurred()) + Expect(ok).To(BeFalse()) + Expect(entries).To(BeNil()) + Expect(warns.naming("open store")).To(HaveLen(1)) + }) + }) +}) diff --git a/pkg/index/store_v2_test.go b/pkg/index/store_v2_test.go new file mode 100644 index 0000000..65e7b88 --- /dev/null +++ b/pkg/index/store_v2_test.go @@ -0,0 +1,238 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "bytes" + "context" + "os" + "path/filepath" + "reflect" + "time" + + libtime "github.com/bborbe/time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + bolt "go.etcd.io/bbolt" + + "github.com/bborbe/vault-cli/pkg/index" + "github.com/bborbe/vault-cli/pkg/storage" +) + +// fieldNames walks a type and returns every struct field name at any depth. +func fieldNames(t reflect.Type, seen map[reflect.Type]bool) []string { + for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Map { + t = t.Elem() + } + if t.Kind() != reflect.Struct || seen[t] { + return nil + } + seen[t] = true + names := []string{} + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + names = append(names, field.Name) + names = append(names, fieldNames(field.Type, seen)...) + } + return names +} + +// writeLiteralStore writes a bolt file holding one identity and one entry +// bucket, using literal names rather than package constants. +func writeLiteralStore(path, identity, vaultDir, pagesDir string) { + db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: time.Second}) + Expect(err).NotTo(HaveOccurred()) + Expect(db.Update(func(tx *bolt.Tx) error { + meta, err := tx.CreateBucket([]byte("vault-ui-page-index-meta")) + if err != nil { + return err + } + if err := meta.Put([]byte("identity"), []byte(identity)); err != nil { + return err + } + entries, err := tx.CreateBucket( + []byte("vault-ui-page-index\x00" + vaultDir + "\x00" + pagesDir), + ) + if err != nil { + return err + } + return entries.Put([]byte("A.md"), []byte("v1 record with a body")) + })).To(Succeed()) + Expect(db.Close()).To(Succeed()) +} + +// bucketNames lists the bolt file's top-level bucket names. +func bucketNames(path string) []string { + db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: time.Second}) + Expect(err).NotTo(HaveOccurred()) + defer func() { Expect(db.Close()).To(Succeed()) }() + names := []string{} + Expect(db.View(func(tx *bolt.Tx) error { + return tx.ForEach(func(name []byte, _ *bolt.Bucket) error { + names = append(names, string(name)) + return nil + }) + })).To(Succeed()) + return names +} + +var _ = Describe("Store format version 2", func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + It("persists no Content field in the store record", func() { + names := fieldNames(index.PersistedRecordType(), map[reflect.Type]bool{}) + Expect(names).NotTo(ContainElement("Content")) + Expect(names).To(ContainElement("OpenQuestions")) + }) + + It("writes no body bytes to the store file", func() { + vaultDir := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(vaultDir, equivalenceFolder), 0750)).To(Succeed()) + writeFixtureFile(vaultDir, "Body.md", + "---\ntitle: Body\n---\n## Open Questions\n\n- kept question\n\n## Notes\nBODY-SENTINEL-7f3a\n") + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + store := newStore(path, &warnSink{}, &vlogSink{}) + + idx := index.NewPageIndexWithStore( + index.NewPageReader(storage.NewPageStorage(nil)), + index.NewDirectoryLister(), + libtime.NewCurrentDateTime(), fastWaiter(), nil, store, + ) + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(pages).To(HaveLen(1)) + Expect(store.Close()).To(Succeed()) + + raw, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(bytes.Contains(raw, []byte("kept question"))).To(BeTrue()) + Expect(bytes.Contains(raw, []byte("BODY-SENTINEL-7f3a"))).To(BeFalse()) + }) + + It("discards a v1 store, deletes its entry buckets and warns store format mismatch with the path", func() { + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + writeLiteralStore(path, `{"StoreFormat":1,"ParserVersion":"v0.171.0"}`, "/vault", "24 Tasks") + Expect(bucketNames(path)).To(HaveLen(2)) + + warns := &warnSink{} + store := newStore(path, warns, &vlogSink{}) + Expect(store.Close()).To(Succeed()) + + Expect(bucketNames(path)).To(Equal([]string{"vault-ui-page-index-meta"})) + mismatch := warns.naming("store format mismatch") + Expect(mismatch).To(HaveLen(1)) + Expect(mismatch[0]).To(ContainSubstring("stored 1, current 2")) + Expect(mismatch[0]).To(ContainSubstring(path)) + }) + + It("reports parser version mismatch as a distinct reason", func() { + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + writeLiteralStore(path, `{"StoreFormat":2,"ParserVersion":"v0.0.0-stale"}`, "/vault", "24 Tasks") + + warns := &warnSink{} + store := newStore(path, warns, &vlogSink{}) + Expect(store.Close()).To(Succeed()) + + Expect(warns.naming("parser version mismatch")).To(HaveLen(1)) + Expect(warns.naming(path)).To(HaveLen(1)) + Expect(warns.naming("store format mismatch")).To(BeEmpty()) + Expect(bucketNames(path)).To(Equal([]string{"vault-ui-page-index-meta"})) + }) + + It("reports a damaged identity as a distinct reason", func() { + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + writeLiteralStore(path, `{not json`, "/vault", "24 Tasks") + + warns := &warnSink{} + store := newStore(path, warns, &vlogSink{}) + Expect(store.Close()).To(Succeed()) + + Expect(warns.naming("damaged")).To(HaveLen(1)) + Expect(bucketNames(path)).To(Equal([]string{"vault-ui-page-index-meta"})) + }) + + It("serves without a store when the bolt file is locked by another process", NodeTimeout(30*time.Second), func(specCtx SpecContext) { + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + holder, err := bolt.Open(path, 0600, &bolt.Options{Timeout: time.Second}) + Expect(err).NotTo(HaveOccurred()) + defer func() { Expect(holder.Close()).To(Succeed()) }() + + vaultDir := buildEquivalenceVault() + warns := &warnSink{} + start := time.Now() + store := index.NewBoltStore(specCtx, path, index.CurrentWriterIdentity(specCtx), warns.warnf, (&vlogSink{}).vlogf) + Expect(time.Since(start)).To(BeNumerically("<", 10*time.Second)) + + idx := index.NewPageIndexWithStoreAndWarnf( + index.NewPageReader(storage.NewPageStorage(nil)), + index.NewDirectoryLister(), + libtime.NewCurrentDateTime(), fastWaiter(), store, warns.warnf, + ) + pages, err := idx.ListPages(specCtx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(indexedNames(pages)).To(Equal([]string{"Inside", "Plain", "Wikilink", "Ünïcode"})) + Expect(warns.naming("discarded")).NotTo(BeEmpty()) + Expect(warns.naming(path)).NotTo(BeEmpty()) + }) + + It("resolves the store path under the user cache directory per app", func() { + cacheDir, err := os.UserCacheDir() + Expect(err).NotTo(HaveOccurred()) + path, err := index.DefaultStorePath("vault-api") + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(filepath.Join(cacheDir, "vault-api", "page-index.bolt"))) + }) + + It("creates the store directory with mode 0700", func() { + dir := filepath.Join(GinkgoT().TempDir(), "nested") + store := newStore(filepath.Join(dir, "page-index.bolt"), &warnSink{}, &vlogSink{}) + defer closeStore(store) + info, err := os.Stat(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0700))) + }) +}) + +var _ = Describe("Store hydration", func() { + It("restores open questions from the store without re-reading the file", func() { + ctx := context.Background() + vaultDir := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(vaultDir, equivalenceFolder), 0750)).To(Succeed()) + writeFixtureFile(vaultDir, "Questions.md", + "---\ntitle: Questions\n---\n## Open Questions\n\n- first\n- second → answered\n") + writeFixtureFile(vaultDir, "Plain.md", "---\ntitle: Plain\n---\n# Plain\n") + path := filepath.Join(GinkgoT().TempDir(), "page-index.bolt") + + storeA := newStore(path, &warnSink{}, &vlogSink{}) + a := index.NewPageIndexWithStore( + index.NewPageReader(storage.NewPageStorage(nil)), index.NewDirectoryLister(), + libtime.NewCurrentDateTime(), fastWaiter(), nil, storeA, + ) + entriesA, err := a.ListEntries(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(entriesA).To(HaveLen(2)) + Expect(storeA.Close()).To(Succeed()) + + storeB := newStore(path, &warnSink{}, &vlogSink{}) + defer closeStore(storeB) + metricsB := index.NewCountingMetrics() + b := index.NewPageIndexWithStore( + index.NewPageReader(storage.NewPageStorage(nil)), index.NewDirectoryLister(), + libtime.NewCurrentDateTime(), fastWaiter(), metricsB, storeB, + ) + entriesB, err := b.ListEntries(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(entriesB).To(HaveLen(2)) + for i := range entriesA { + Expect(entriesB[i].OpenQuestions).To(Equal(entriesA[i].OpenQuestions)) + } + Expect(entriesB[1].OpenQuestions).To(HaveLen(2)) + Expect(metricsB.TotalReads()).To(Equal(0)) + }) +}) diff --git a/pkg/index/store_writethrough_test.go b/pkg/index/store_writethrough_test.go new file mode 100644 index 0000000..9d34584 --- /dev/null +++ b/pkg/index/store_writethrough_test.go @@ -0,0 +1,414 @@ +// Copyright (c) 2026 Benjamin Borbe All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package index_test + +import ( + "context" + "errors" + "fmt" + "os" + "sort" + "sync" + "time" + + libtime "github.com/bborbe/time" + "github.com/bborbe/vault-cli/pkg/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/bborbe/vault-cli/pkg/index" +) + +// storeWrite is one recorded Store.Write call: the entries it put and the names +// it deleted, exactly as the caller passed them. +type storeWrite struct { + puts []index.StoredEntry + deletes []string +} + +// recordingStore is a Store seam that records every transaction and keeps an +// in-memory per-key map so Load reflects what was applied. It can be configured +// to fail or to block a write, so the write-through path is observable. +type recordingStore struct { + mu sync.Mutex + entries map[index.Key]map[string]index.StoredEntry + writes []storeWrite + + fail error + blockWrites bool + writeEntered chan struct{} + writeRelease chan struct{} +} + +func (s *recordingStore) Load( + ctx context.Context, + key index.Key, +) ([]index.StoredEntry, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + key = index.NewKey(key.VaultPath, key.PagesDir) + s.mu.Lock() + defer s.mu.Unlock() + stored := s.entries[key] + if len(stored) == 0 { + return nil, false, nil + } + names := make([]string, 0, len(stored)) + for name := range stored { + names = append(names, name) + } + sort.Strings(names) + entries := make([]index.StoredEntry, 0, len(names)) + for _, name := range names { + entries = append(entries, stored[name]) + } + return entries, true, nil +} + +func (s *recordingStore) Write( + ctx context.Context, + key index.Key, + puts []index.StoredEntry, + deletes []string, +) error { + if err := ctx.Err(); err != nil { + return err + } + key = index.NewKey(key.VaultPath, key.PagesDir) + + s.mu.Lock() + s.writes = append(s.writes, storeWrite{ + puts: append([]index.StoredEntry(nil), puts...), + deletes: append([]string(nil), deletes...), + }) + block := s.blockWrites + entered := s.writeEntered + release := s.writeRelease + fail := s.fail + s.mu.Unlock() + + if block { + if entered != nil { + entered <- struct{}{} + } + <-release + } + if fail != nil { + return fail + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.entries == nil { + s.entries = map[index.Key]map[string]index.StoredEntry{} + } + if s.entries[key] == nil { + s.entries[key] = map[string]index.StoredEntry{} + } + for _, put := range puts { + s.entries[key][put.Filename] = put + } + for _, name := range deletes { + delete(s.entries[key], name) + } + return nil +} + +func (s *recordingStore) Path() string { return "recording-store" } + +func (s *recordingStore) Close() error { return nil } + +// recorded returns every Write call made so far, in order. +func (s *recordingStore) recorded() []storeWrite { + s.mu.Lock() + defer s.mu.Unlock() + return append([]storeWrite(nil), s.writes...) +} + +// stored returns a copy of the in-memory map for one key. +func (s *recordingStore) stored(key index.Key) map[string]index.StoredEntry { + key = index.NewKey(key.VaultPath, key.PagesDir) + s.mu.Lock() + defer s.mu.Unlock() + stored := map[string]index.StoredEntry{} + for name, entry := range s.entries[key] { + stored[name] = entry + } + return stored +} + +// reset forgets the recorded transactions but keeps the applied content. +func (s *recordingStore) reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.writes = nil +} + +// setFail makes every subsequent Write fail with err. +func (s *recordingStore) setFail(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.fail = err +} + +// setBlocking makes every subsequent Write signal entered and wait until +// release is closed. +func (s *recordingStore) setBlocking(entered, release chan struct{}) { + s.mu.Lock() + defer s.mu.Unlock() + s.blockWrites = true + s.writeEntered = entered + s.writeRelease = release +} + +// putNames returns the filenames of one write's puts. +func putNames(writes []storeWrite, i int) []string { + names := make([]string, 0, len(writes[i].puts)) + for _, put := range writes[i].puts { + names = append(names, put.Filename) + } + return names +} + +// newWriteThroughIndex builds a recording-store index with counting seams. +func newWriteThroughIndex( + store index.Store, +) (*recordingReader, *recordingLister, index.PageIndex) { + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + return reader, lister, newStoreIndex(store, reader, lister, &warnSink{}) +} + +var _ = Describe("Store write-through", func() { + var ( + ctx context.Context + store *recordingStore + ) + + BeforeEach(func() { + ctx = context.Background() + store = &recordingStore{} + }) + + It("AC5: a cold build writes N entries in one transaction", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + reader, _, idx := newWriteThroughIndex(store) + + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(indexedNames(pages)).To(HaveLen(5)) + Expect(reader.Count()).To(Equal(5)) + + writes := store.recorded() + Expect(writes).To(HaveLen(1)) + Expect(putNames(writes, 0)).To(ConsistOf( + "Page00.md", "Page01.md", "Page02.md", "Page03.md", "Page04.md", + )) + Expect(writes[0].deletes).To(BeEmpty()) + Expect(store.stored(key)).To(HaveLen(5)) + }) + + It("AC5: a per-file update writes exactly one put", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + _, _, idx := newWriteThroughIndex(store) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + before := store.stored(key) + store.reset() + + settle() + writeFixtureFile(vaultDir, "Page02.md", "---\ntitle: Page02 v2\n---\n# Page02 v2\n") + Expect(idx.RefreshFile(ctx, key, "Page02.md")).To(Succeed()) + + writes := store.recorded() + Expect(writes).To(HaveLen(1)) + Expect(putNames(writes, 0)).To(Equal([]string{"Page02.md"})) + Expect(writes[0].deletes).To(BeEmpty()) + + after := store.stored(key) + Expect(after).To(HaveLen(5)) + Expect(after["Page02.md"].Entry.Page.GetString("title")).To(Equal("Page02 v2")) + for name, entry := range before { + if name == "Page02.md" { + continue + } + Expect(after[name].Fingerprint).To(Equal(entry.Fingerprint)) + Expect(after[name].Entry).To(BeIdenticalTo(entry.Entry)) + } + }) + + It("AC5: a deletion writes exactly one delete", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + _, _, idx := newWriteThroughIndex(store) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + store.reset() + + Expect(os.Remove(pageFile(vaultDir, "Page02.md"))).To(Succeed()) + Expect(idx.RefreshFile(ctx, key, "Page02.md")).To(Succeed()) + + writes := store.recorded() + Expect(writes).To(HaveLen(1)) + Expect(writes[0].puts).To(BeEmpty()) + Expect(writes[0].deletes).To(Equal([]string{"Page02.md"})) + Expect(store.stored(key)).NotTo(HaveKey("Page02.md")) + }) + + It("AC5: a stat-diff that drops a removed file writes exactly one delete", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + _, _, idx := newWriteThroughIndex(store) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + store.reset() + + // The listing path, never RefreshFile: the removed name is dropped by the + // merge, so the delete must be derived before the in-place fingerprint + // update erases the name it would have reported. + Expect(os.Remove(pageFile(vaultDir, "Page02.md"))).To(Succeed()) + Expect(idx.Refresh(ctx, key)).To(Succeed()) + + writes := store.recorded() + Expect(writes).To(HaveLen(1)) + Expect(writes[0].puts).To(BeEmpty()) + Expect(writes[0].deletes).To(Equal([]string{"Page02.md"})) + Expect(store.stored(key)).NotTo(HaveKey("Page02.md")) + }) + + It("AC5: a failed write leaves serving and the store intact", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + warns := &warnSink{} + idx := newStoreIndex(store, reader, lister, warns) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + before := store.stored(key) + store.reset() + + settle() + writeFixtureFile(vaultDir, "Page02.md", "---\ntitle: Page02 v2\n---\n# Page02 v2\n") + store.setFail(errors.New("disk full")) + + Expect(idx.RefreshFile(ctx, key, "Page02.md")).To(Succeed()) + + // The publication completed and the next read serves the new content. + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(indexedNames(pages)).To(HaveLen(5)) + Expect(pages[2].GetString("title")).To(Equal("Page02 v2")) + + // The previous store content is untouched and exactly one warning names + // the failure. + Expect(store.stored(key)).To(Equal(before)) + Expect(warns.naming("store write failed")).To(HaveLen(1)) + }) + + It("AC5: a blocking write does not block a reader", func() { + vaultDir := newStoreVault(5) + key := index.NewKey(vaultDir, equivalenceFolder) + _, _, idx := newWriteThroughIndex(store) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + store.reset() + + entered := make(chan struct{}, 1) + release := make(chan struct{}) + store.setBlocking(entered, release) + + settle() + writeFixtureFile(vaultDir, "Page02.md", "---\ntitle: Page02 v2\n---\n# Page02 v2\n") + refreshDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + refreshDone <- idx.RefreshFile(ctx, key, "Page02.md") + }() + Eventually(entered, "2s").Should(Receive()) + + start := time.Now() + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(time.Since(start)).To(BeNumerically("<", 100*time.Millisecond)) + Expect(pages[2].GetString("title")).To(Equal("Page02 v2")) + + close(release) + Eventually(refreshDone, "2s").Should(Receive(BeNil())) + }) + + It("AC5: an unchanged rescan writes nothing", func() { + clock := libtime.NewCurrentDateTime() + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock.SetNow(libtime.DateTime(start)) + + vaultDir := newStoreVault(4) + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + idx := index.NewPageIndexWithStoreAndWarnf( + reader, lister, clock, fastWaiter(), store, (&warnSink{}).warnf, + ) + + _, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + store.reset() + lister.reset() + + rescanCtx, cancel := context.WithCancel(ctx) + rescanDone := make(chan error, 1) + go func() { + defer GinkgoRecover() + rescanDone <- idx.Rescan(rescanCtx) + }() + + // Let the loop record its start time before the clock moves. + Consistently(func() int { return len(store.recorded()) }, "100ms").Should(Equal(0)) + + clock.SetNow(libtime.DateTime(start.Add(time.Minute))) + Eventually(lister.Count, "2s").Should(Equal(1)) + Consistently(func() int { return len(store.recorded()) }, "150ms").Should(Equal(0)) + Expect(index.RescanInterval).To(Equal(50 * time.Second)) + + cancel() + Eventually(rescanDone, "2s").Should(Receive(BeNil())) + }) + + It("AC5: a hydrate's stat-diff writes only what it re-read", func() { + vaultDir := newStoreVault(6) + key := index.NewKey(vaultDir, equivalenceFolder) + populateStore(ctx, store, key, vaultDir, equivalenceFolder) + store.reset() + + changed := []string{"Page01.md", "Page03.md", "Page05.md"} + settle() + for i, name := range changed { + writeFixtureFile(vaultDir, name, fmt.Sprintf( + "---\ntitle: Changed%d\n---\n# Changed%d\n", i, i, + )) + } + + reader := &recordingReader{inner: index.NewPageReader(storage.NewPageStorage(nil))} + lister := &recordingLister{inner: index.NewDirectoryLister()} + idx := newStoreIndex(store, reader, lister, &warnSink{}) + + pages, err := idx.ListPages(ctx, vaultDir, equivalenceFolder) + Expect(err).NotTo(HaveOccurred()) + Expect(pages).To(HaveLen(6)) + Expect(reader.Names()).To(ConsistOf(changed)) + + writes := store.recorded() + Expect(writes).To(HaveLen(1)) + Expect(putNames(writes, 0)).To(ConsistOf(changed)) + Expect(writes[0].deletes).To(BeEmpty()) + }) +})