Check analyses against the whole source text (#143)

__mocks__/lucide-react.tsx

@@ -26,6 +26,13 @@ export function GripVertical(props: Readonly<{ className?: string }>): ReactElem
   return <svg data-testid="grip-vertical-icon" {...props} />;
 }
 
+/**
+ * Stub for the SearchX icon.
+ */
+export function SearchX(props: Readonly<{ className?: string }>): ReactElement {
+  return <svg data-testid="search-x-icon" {...props} />;
+}
+
 /**
  * Stub for the Info icon.
  */

contributions/localizedStrings.json

@@ -45,9 +45,10 @@
       "%interlinearizer_concordance_title%": "Concordance",
       "%interlinearizer_concordance_close%": "Close concordance",
       "%interlinearizer_concordance_refresh%": "Read the source text again",
-      "%interlinearizer_concordance_loading%": "Reading books… {read} of {total}",
-      "%interlinearizer_concordance_error%": "The text of this project could not be read.",
+      "%interlinearizer_textReading_loading%": "Reading books… {read} of {total}",
+      "%interlinearizer_textReading_error%": "The text of this project could not be read.",
       "%interlinearizer_concordance_empty%": "The source text has no words to list.",
+      "%interlinearizer_concordance_partial%": "Some books could not be read, so their words are missing from this list.",
       "%interlinearizer_concordance_occurrenceCount%": "Occurrences in the whole text",
       "%interlinearizer_concordance_occurrenceCountInBook%": "Occurrences in {book}",
       "%interlinearizer_concordance_status_analyzed%": "Analyzed",
@@ -82,6 +83,8 @@
       "%interlinearizer_analysisCatalog_filter_morphemes_lacks%": "Not broken down",
       "%interlinearizer_analysisCatalog_filter_zeroUsages%": "Used nowhere",
       "%interlinearizer_analysisCatalog_filter_stale%": "Stale (found when a book is opened)",
+      "%interlinearizer_analysisCatalog_filter_staleEverywhere%": "Stale",
+      "%interlinearizer_analysisCatalog_filter_notInText%": "Word not in the text",
       "%interlinearizer_analysisCatalog_confidence_high%": "High",
       "%interlinearizer_analysisCatalog_confidence_medium%": "Medium",
       "%interlinearizer_analysisCatalog_confidence_low%": "Low",
@@ -94,6 +97,7 @@
       "%interlinearizer_analysisCatalog_noUsages%": "Not used anywhere",
       "%interlinearizer_analysisCatalog_showAllUsages%": "Show {count} more",
       "%interlinearizer_analysisCatalog_staleCount%": "Stale places",
+      "%interlinearizer_analysisCatalog_notInText%": "Not in the text",
       "%interlinearizer_analysisCatalog_staleHeading%": "Stale: the text changed here after approval",
       "%interlinearizer_analysisCatalog_staleApply%": "Apply to “{word}”",
       "%interlinearizer_analysisCatalog_staleApplyNoTarget%": "Select a word to apply to",
@@ -183,6 +187,7 @@
       "%interlinearizer_freeTranslationInput_staleDiscard%": "Discard",
       "%interlinearizer_freeTranslationInput_staleNoText%": "(no translation in this language)",
       "%interlinearizer_freeTranslationInput_staleKeepBlocked%": "Clear your translation to keep one of these instead",
+      "%interlinearizer_freeTranslationInput_orphaned%": "Stale: verse {verse} is no longer in the text",
       "%interlinearizer_morphemeEditor_splitLabel%": "Split into morphemes",
       "%interlinearizer_morphemeEditor_reset%": "Reset",
       "%interlinearizer_morphemeEditor_cancel%": "Cancel",

docs/reanchoring.md

@@ -21,7 +21,12 @@ Do not grow this into a general diff, fuzzy matching, or cross-verse tracking. F
 
 ## Algorithm
 
-When it runs: on every book load, in `useReanchorToBook` (`src/components/AnalysisStore.tsx`), for an editable project only. An imported, read-only project is a record of what was imported and is never healed. A pass that moves nothing leaves the analysis identical, so opening a book neither dirties the draft nor writes storage.
+When it runs, for an editable project only:
+
+- for the loaded book, whenever its text, its boundaries, or the draft changes;
+- for every other book the draft has records in, each time the whole source text is read (for the concordance, or a catalog filter that needs the text).
+
+A book the project no longer has is treated as holding no text, so every approval in it goes stale. An imported, read-only project is a record of what was imported and is never healed. A pass that moves nothing leaves the analysis identical, so opening a book neither dirties the draft nor writes storage.
 
 Token and phrase links carry a `TokenSnapshot` (`tokenRef` plus the `surfaceText` it was written against). Approved, candidate, and stale links are the evidence the alignment weighs; rejected and suggested links align afterward, among themselves, against the tokens those leave unclaimed, so a verdict never decides which occurrence an approval names. For each verse independently:
 

src/__tests__/components/AnalysisCatalogPanel.test.tsx

@@ -11,13 +11,18 @@ import AnalysisCatalogPanel, {
   type AnalysisCatalogPanelHandle,
 } from '../../components/AnalysisCatalogPanel';
 import { AnalysisStoreProvider, useGlossDispatch } from '../../components/AnalysisStore';
+import { SourceTextContext } from '../../components/SourceTextContext';
 import { InterlinearNavProvider, useInterlinearNav } from '../../components/InterlinearNavContext';
+import type { SourceText } from '../../hooks/useSourceTextReader';
 import type { StepSummary } from '../../hooks/useDraftProject';
 import { emptyAnalysis } from '../../types/empty-factories';
 import type { HeadingPlacement } from '../../utils/analysis-query';
 import { defaultScrRef, FIXTURE_STAMPS, makeScrollGroupHook } from '../test-helpers';
 import { mockKeyAsValueLocalizedStrings } from './test-helpers';
 
+// What the reading status shows is its own concern; these cover only when the panel shows it.
+jest.mock('../../components/TextReadingStatus');
+
 const NO_HEADINGS: ReadonlyMap<string, HeadingPlacement> = new Map();
 
 /**
@@ -83,6 +88,22 @@ function FocusPublishProbe({ tokenRef }: Readonly<{ tokenRef: string | undefined
   return undefined;
 }
 
+/** Builds the source text as its hook hands it over, unread by default. */
+function makeText(overrides: Partial<SourceText> = {}): SourceText {
+  return {
+    status: 'idle',
+    booksRead: 0,
+    bookCount: 0,
+    readings: undefined,
+    liveVersions: new Map(),
+    isPartial: false,
+    textForms: undefined,
+    refresh: () => {},
+    request: () => {},
+    ...overrides,
+  };
+}
+
 /** Options every `renderPanel` call may override. */
 type PanelOptions = Partial<{
   ref: Ref<AnalysisCatalogPanelHandle>;
@@ -111,6 +132,10 @@ type PanelOptions = Partial<{
   liveSurfaceText: (tokenRef: string) => string | undefined;
   /** The word the view beside the panel has focused, or none. */
   focusedTokenRef: string;
+  /** The source text. Defaults to one not yet read. */
+  text: SourceText;
+  /** Whether every book has been checked for stale places. Defaults to only the opened ones. */
+  staleCoversDraft: boolean;
 }>;
 
 /** Reads every token as still carrying the form its analysis was recorded under. */
@@ -143,7 +168,9 @@ function PanelProviders({
       >
         <FocusRequestProbe bookCode={overrides.mountedBook ?? 'GEN'} />
         <FocusPublishProbe tokenRef={overrides.focusedTokenRef} />
-        {children}
+        <SourceTextContext.Provider value={overrides.text ?? makeText()}>
+          {children}
+        </SourceTextContext.Provider>
       </AnalysisStoreProvider>
     </InterlinearNavProvider>
   );
@@ -164,6 +191,7 @@ function renderPanel(overrides: PanelOptions = {}) {
         onShowConcordance={overrides.onShowConcordance ?? (() => {})}
         showMorphology={overrides.showMorphology ?? true}
         sourceLanguageTag="el"
+        staleCoversDraft={overrides.staleCoversDraft ?? false}
       />
     </PanelProviders>,
   );
@@ -189,6 +217,7 @@ function ReopenableCatalog() {
           onShowConcordance={() => {}}
           showMorphology
           sourceLanguageTag="el"
+          staleCoversDraft={false}
         />
       )}
     </>
@@ -229,6 +258,7 @@ function renderPanelWithGlossEditing(overrides: PanelOptions = {}) {
         onShowConcordance={overrides.onShowConcordance ?? (() => {})}
         showMorphology={overrides.showMorphology ?? true}
         sourceLanguageTag="el"
+        staleCoversDraft={false}
       />
     </PanelProviders>,
   );
@@ -1222,6 +1252,7 @@ describe('AnalysisCatalogPanel', () => {
             onShowConcordance={() => {}}
             showMorphology={false}
             sourceLanguageTag="el"
+            staleCoversDraft={false}
           />
         </PanelProviders>,
       );
@@ -1379,6 +1410,7 @@ describe('AnalysisCatalogPanel', () => {
             onShowConcordance={() => {}}
             showMorphology
             sourceLanguageTag="el"
+            staleCoversDraft={false}
           />
         </PanelProviders>,
       );
@@ -1403,6 +1435,7 @@ describe('AnalysisCatalogPanel', () => {
             onShowConcordance={() => {}}
             showMorphology
             sourceLanguageTag="el"
+            staleCoversDraft={false}
           />
         </PanelProviders>,
       );
@@ -1422,6 +1455,7 @@ describe('AnalysisCatalogPanel', () => {
             onShowConcordance={() => {}}
             showMorphology
             sourceLanguageTag="el"
+            staleCoversDraft={false}
           />
         </PanelProviders>
       );
@@ -1998,6 +2032,38 @@ describe('AnalysisCatalogPanel', () => {
       });
     });
 
+    it('asks for the text to be read when filtering for them', async () => {
+      const request = jest.fn();
+      renderPanel({ analysis: ONLY_STALE, text: makeText({ request }) });
+      await openFilters();
+
+      await userEvent.click(
+        screen.getByRole('checkbox', { name: '%interlinearizer_analysisCatalog_filter_stale%' }),
+      );
+
+      expect(request).toHaveBeenCalled();
+    });
+
+    it('says the stale filter finds them only in opened books until every book is checked', async () => {
+      renderPanel({ analysis: ONLY_STALE });
+      await openFilters();
+
+      expect(
+        screen.getByRole('checkbox', { name: '%interlinearizer_analysisCatalog_filter_stale%' }),
+      ).toBeInTheDocument();
+    });
+
+    it('names the stale filter plainly once every book is checked', async () => {
+      renderPanel({ analysis: ONLY_STALE, staleCoversDraft: true });
+      await openFilters();
+
+      expect(
+        screen.getByRole('checkbox', {
+          name: '%interlinearizer_analysisCatalog_filter_staleEverywhere%',
+        }),
+      ).toBeInTheDocument();
+    });
+
     describe('over an unsaved breakdown', () => {
       /** Types a re-segmentation into an expanded row's breakdown without saving it. */
       async function typeUnsavedBreakdown(analysisId = 'ta-1'): Promise<void> {
@@ -2124,6 +2190,104 @@ describe('AnalysisCatalogPanel', () => {
     expect(onClose).toHaveBeenCalled();
   });
 
+  describe('forms not in the text', () => {
+    /** One analysis whose form the text still holds, and one whose form it no longer does. */
+    const analysis: TextAnalysis = {
+      ...emptyAnalysis(),
+      tokenAnalyses: [
+        { ...FIXTURE_STAMPS, id: 'kept', surfaceText: 'λόγος' },
+        { ...FIXTURE_STAMPS, id: 'gone', surfaceText: 'ἦν' },
+      ],
+      tokenAnalysisLinks: [
+        link('kept', 'GEN 1:1:0', 'approved', 'λόγος'),
+        link('gone', 'GEN 1:2:0', 'stale', 'ἦν'),
+      ],
+    };
+
+    /** The filter keeping only analyses whose form the text no longer holds. */
+    function notInTextFilter(): HTMLElement {
+      return screen.getByRole('checkbox', {
+        name: '%interlinearizer_analysisCatalog_filter_notInText%',
+      });
+    }
+
+    it('asks for the text to be read when filtering for them', async () => {
+      const request = jest.fn();
+      renderPanel({ analysis, text: makeText({ request }) });
+      await openFilters();
+
+      await userEvent.click(notInTextFilter());
+
+      expect(request).toHaveBeenCalled();
+    });
+
+    it('asks for the text to be read when narrowed to them from outside', () => {
+      const request = jest.fn();
+      const ref = createRef<AnalysisCatalogPanelHandle>();
+      renderPanel({ ref, analysis, text: makeText({ request }) });
+
+      act(() => ref.current?.filterTo({ notInText: true }));
+
+      expect(request).toHaveBeenCalled();
+    });
+
+    it('keeps only the analyses whose form the text lacks once it is read', async () => {
+      const text = makeText({ status: 'ready', textForms: new Set(['λόγος']) });
+      renderPanel({ analysis, text });
+      await openFilters();
+
+      await userEvent.click(notInTextFilter());
+
+      expect(listedAnalysisIds()).toEqual(['gone']);
+    });
+
+    it('shows how far reading the text has got while filtering for them', async () => {
+      renderPanel({ analysis, text: makeText({ status: 'loading' }) });
+      await openFilters();
+
+      await userEvent.click(notInTextFilter());
+
+      expect(screen.getByTestId('text-reading-status')).toBeInTheDocument();
+      expect(screen.queryByTestId('catalog-row')).not.toBeInTheDocument();
+    });
+
+    it('says so when a book could not be read', async () => {
+      renderPanel({ analysis, text: makeText({ status: 'ready', isPartial: true }) });
+      await openFilters();
+
+      await userEvent.click(notInTextFilter());
+
+      expect(screen.getByTestId('text-reading-status')).toHaveAttribute('data-partial', 'true');
+      expect(screen.queryByTestId('catalog-row')).not.toBeInTheDocument();
+    });
+
+    it('marks an analysis whose form the text lacks once it is read', () => {
+      renderPanel({
+        analysis,
+        text: makeText({ status: 'ready', textForms: new Set(['λόγος']) }),
+      });
+
+      expect(within(rowFor('gone')).getByTestId('catalog-row-not-in-text')).toBeInTheDocument();
+    });
+
+    it('does not mark an analysis whose form the text holds', () => {
+      renderPanel({
+        analysis,
+        text: makeText({ status: 'ready', textForms: new Set(['λόγος']) }),
+      });
+
+      expect(
+        within(rowFor('kept')).queryByTestId('catalog-row-not-in-text'),
+      ).not.toBeInTheDocument();
+    });
+
+    it('marks no analysis while the text is unread', () => {
+      renderPanel({ analysis });
+
+      expect(screen.queryByTestId('catalog-row-not-in-text')).not.toBeInTheDocument();
+    });
+  });
+
   describe('editing a row', () => {
     /** One analysis, shared by two tokens, so an edit here is visibly an edit to both. */
     const SHARED: TextAnalysis = {

src/__tests__/components/AnalysisStore.test.tsx

@@ -29,7 +29,7 @@ import {
   useSegmentHasApprovedTranslation,
   useSegmentsWithApprovedTranslation,
   useStaleFreeTranslationDispatch,
-  useStaleFreeTranslationsBySegment,
+  usePlacedStaleFreeTranslations,
   useStaleTokenLinks,
   useGloss,
   useGlossDispatch,
@@ -1074,14 +1074,14 @@ describe('useSegmentsWithApprovedTranslation', () => {
   });
 });
 
-describe('useStaleFreeTranslationsBySegment', () => {
+describe('usePlacedStaleFreeTranslations', () => {
   it("files a stale translation under the book's segment showing it", () => {
     const book = makeVerseBook([{ sid: 'GEN 1:1', text: 'In the beginning' }]);
-    const { result } = renderStoreHook(() => useStaleFreeTranslationsBySegment(book), {
+    const { result } = renderStoreHook(() => usePlacedStaleFreeTranslations(book), {
       initialAnalysis: withSegmentTranslation('GEN 1:1', 'stale'),
     });
 
-    expect(result.current).toEqual(
+    expect(result.current.bySegment).toEqual(
       new Map([['GEN 1:1', [{ analysisId: 'sa-1', segmentId: 'GEN 1:1', text: 'old' }]]]),
     );
   });

src/__tests__/components/CatalogMergeModal.test.tsx

@@ -32,6 +32,7 @@ function row(analysisId: string, overrides: Partial<CatalogRow> = {}): CatalogRo
   return {
     analysisId,
     surfaceText: 'λόγος',
+    form: 'λόγος',
     gloss: '',
     glosses: gloss ? { [analysisLanguage]: gloss } : undefined,
     morphemes: [],