Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/components/modal/IndexingReportModal.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const skipReasonLabels: Record<SkipReason, string> = {
"not-indexed": "Not yet indexed",
"read-error": "Failed to read file",
"embed-error": "Embedding failed",
empty: "No text to embed",
};

const skipReasonIcons: Record<SkipReason, string> = {
Expand All @@ -29,6 +30,7 @@ const skipReasonIcons: Record<SkipReason, string> = {
"not-indexed": "clock",
"read-error": "alert-triangle",
"embed-error": "zap-off",
empty: "file-x",
};

const skippedByReason = $derived.by(() => {
Expand Down
22 changes: 16 additions & 6 deletions src/search/LexicalSearchService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,13 @@ import {
} from "../search/lexicalScoring";
import { Logger } from "../utils/logging";
import { StartupProfiler } from "../utils/startupProfiler";
import { getIndexableVaultFiles, isIndexableFile, readIndexableContent } from "../utils/fileFiltering";
import {
DRAWING_MODIFY_DEBOUNCE_MS,
getIndexableVaultFiles,
isExcalidrawPath,
isIndexableFile,
readIndexableContent,
} from "../utils/fileFiltering";
import {
MiniSearchService,
type AutocompleteCacheSnapshot,
Expand All @@ -36,7 +42,8 @@ const MAX_SNIPPET_LENGTH = 180;
* user types, and every save re-read, re-tokenized and re-added the note;
* the embedding path already waits (5 s, a provider round trip per save). The
* lexical index is cheap to update, so it waits less — an edit is searchable
* a couple of seconds after the last keystroke.
* a couple of seconds after the last keystroke. Drawings wait longer
* (`DRAWING_MODIFY_DEBOUNCE_MS`).
*/
const MODIFY_DEBOUNCE_MS = 2_000;

Expand Down Expand Up @@ -429,10 +436,13 @@ export class LexicalSearchService {
this.cancelPendingModify(file.path);
this.modifyTimers.set(
file.path,
window.setTimeout(() => {
this.modifyTimers.delete(file.path);
void this.reindexModifiedFile(file);
}, MODIFY_DEBOUNCE_MS),
window.setTimeout(
() => {
this.modifyTimers.delete(file.path);
void this.reindexModifiedFile(file);
},
isExcalidrawPath(file.path) ? DRAWING_MODIFY_DEBOUNCE_MS : MODIFY_DEBOUNCE_MS,
),
);
}

Expand Down
8 changes: 8 additions & 0 deletions src/stores/dataStore.svelte.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1449,6 +1449,7 @@ export class PluginDataStore {
documentCount?: number;
dimensions?: number;
failedNotes?: Record<string, number>;
emptyNotes?: Record<string, number>;
},
): void {
const config = this.#data.embeddingIndexes.find((i) => i.id === indexId);
Expand Down Expand Up @@ -1478,6 +1479,13 @@ export class PluginDataStore {
changed = true;
}
}
if (stats.emptyNotes !== undefined) {
const next = Object.keys(stats.emptyNotes).length > 0 ? stats.emptyNotes : undefined;
if (next !== undefined || config.emptyNotes !== undefined) {
config.emptyNotes = next;
changed = true;
}
}
if (changed) void this.saveSettings();
}

Expand Down
6 changes: 6 additions & 0 deletions src/types/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ export interface EmbeddingIndexConfig {
* that mtime changes; absent when nothing failed.
*/
failedNotes?: Record<string, number>;
/**
* Notes with no text to embed (a blank note, a drawing with nothing on the back
* of the note), keyed by path with the mtime they were read at. Same lifecycle
* as `failedNotes`; absent when there are none.
*/
emptyNotes?: Record<string, number>;
}

// ============================================================================
Expand Down
77 changes: 61 additions & 16 deletions src/utils/fileFiltering.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,13 @@ const BINARY_TEXT_EXTENSIONS = new Set(["pdf"]);
* - images: no text to extract. Embedding them would require a multimodal
* embedding model and a separate image pipeline; until that exists they are
* pure noise rather than a missing feature.
* - `excalidraw`: a bare Excalidraw scene — JSON shapes, no prose. (The
* `.excalidraw.md` form is a markdown note and is decided by its content:
* see {@link readIndexableContent}.)
*/
const NON_EMBEDDABLE_EXTENSIONS = new Set([
"base",
"excalidraw",
"png",
"jpg",
"jpeg",
Expand Down Expand Up @@ -225,6 +229,24 @@ export function isEmbeddableFile(file: TFile): boolean {
return !NON_EMBEDDABLE_EXTENSIONS.has(extension);
}

/** Returns `true` for an Excalidraw drawing, in either its `.excalidraw.md` or bare `.excalidraw` form. */
export function isExcalidrawPath(path: string): boolean {
const lower = path.toLowerCase();
return lower.endsWith(".excalidraw.md") || lower.endsWith(".excalidraw");
}

/**
* Debounce before re-indexing a modified Excalidraw drawing.
*
* Excalidraw autosaves every few seconds while the user draws, and each save
* rewrites the whole file — the scene JSON included. The indexers' normal
* debounce (a couple of seconds) fell between those saves, so every one of them
* re-read the full file on the main thread mid-stroke (#541). What a drawing
* contributes to search — its title and the text on the back of the note — rarely
* changes while drawing, so it is picked up once the drawing has been left alone.
*/
export const DRAWING_MODIFY_DEBOUNCE_MS = 30_000;

/**
* Enumerate all vault files eligible for indexing (replaces
* `vault.getMarkdownFiles()` in search / vectorstore callers).
Expand All @@ -245,8 +267,8 @@ export function getEmbeddableVaultFiles(vault: Vault): TFile[] {
* - `.chat` files: extracts only human/AI message content (not LangChain metadata)
* - `.pdf` files: extracts the document text via pdfjs, so a PDF is chunked and
* embedded like any other long document rather than reduced to its filename
* - `.excalidraw.md` files: extracts the "back of the note" and text elements,
* stripping the huge embedded JSON drawing data
* - `.excalidraw.md` files: extracts the frontmatter and the user-written "back
* of the note", stripping Excalidraw's banner and the huge drawing data.
*
* The caller is responsible for filtering via `isIndexableFile`
* before calling this helper.
Expand Down Expand Up @@ -323,6 +345,26 @@ async function readIndexableContentRaw(vault: Vault, file: TFile): Promise<strin
return raw;
}

/**
* Whether `content`, as {@link readIndexableContent} returned it for `file`, means
* the file has nothing worth embedding — so it is left out rather than stored as a
* title-only vector at the similarity noise floor.
*
* Only a successful plain-text read can say so. An empty extraction from a
* container (a PDF, a `.chat` thread) or a file over the mobile size gate may be a
* failed or skipped read, and is not a verdict on the file. A drawing's
* frontmatter is plugin bookkeeping (`excalidraw-plugin: parsed`) and does not
* count as text — every drawing has it, so with it a blank drawing never read as
* empty (#541). A plain note's frontmatter does: a TaskNotes task keeps
* everything in it.
*/
export function hasNoEmbeddableText(file: TFile, content: string): boolean {
if (!isTextIndexableFile(file) || file.extension === "chat") return false;
if (Platform.isMobile && file.stat.size > MOBILE_MAX_READ_BYTES) return false;
const text = isExcalidrawPath(file.path) ? content.replace(RE_LEADING_FRONTMATTER, "") : content;
return text.trim() === "";
}

/**
* Read a text file's raw content, resilient to Obsidian mobile's `cachedRead`.
*
Expand Down Expand Up @@ -662,22 +704,25 @@ function extractMessageContent(msg: unknown): string {
*/
const RE_EXCALIDRAW_DATA_START = /^#+ (?:Excalidraw Data|Text Elements|Drawing)\s*$/m;

/** A leading frontmatter block. */
const RE_LEADING_FRONTMATTER = /^---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/;

/**
* Extract the user-authored "back of the note" from an `.excalidraw.md` file.
* Strips the frontmatter-tagged Excalidraw sections and the huge JSON drawing.
* The "==⚠ Switch to EXCALIDRAW VIEW … ⚠==" warning Excalidraw writes into every
* drawing for readers who open it as markdown — the same boilerplate in each file.
*/
function extractExcalidrawContent(raw: string): string {
const match = RE_EXCALIDRAW_DATA_START.exec(raw);
if (!match) {
// No Excalidraw markers found — treat the whole file as content
return raw;
}

// Everything before the first Excalidraw section header
let content = raw.substring(0, match.index);
const RE_EXCALIDRAW_BANNER = /^==⚠.*$/gm;

/**
* Extract the frontmatter and the user-authored "back of the note" from an
* `.excalidraw.md` file, stripping the banner and everything from the drawing
* data on. The frontmatter stays for lexical search, which reads `aliases` from
* it; {@link hasNoEmbeddableText} looks past it. Exported for tests.
*/
export function extractExcalidrawContent(raw: string): string {
const match = RE_EXCALIDRAW_DATA_START.exec(raw);
// Without the data section it is not a drawing in the format we know; keep it all.
const content = (match ? raw.substring(0, match.index) : raw).replace(RE_EXCALIDRAW_BANNER, "");
// Strip Obsidian comment markers that Excalidraw uses as section fences
content = content.replace(/%%\s*$/g, "").trim();

return content;
return content.replace(/%%\s*$/g, "").trim();
}
Loading
Loading