fix: preserve offline access and tolerate slow mirrors

README.md

@@ -24,6 +24,8 @@ const matches = await docs.search('repair', { pathPrefix: 'repair', limit: 10 })
 
 After a transient failure the client serves stale cached data. With none, it throws
 `DocsFetchError`, which exposes `path` and HTTP `status`.
+With a persistent `store`, listing and searching cached documents also work after
+restarting offline. The first load still needs a network connection.
 
 ## API
 
@@ -64,8 +66,8 @@ refetched. Store errors and corrupt entries are ignored; eviction is up to the s
 `mirror` is tried before GitHub. It serves `index.json` in the shape of GitHub's recursive tree
 response, each file at `raw/<path>`, and optionally `bundle.json` as
 `{ files: [{ path, sha, content }] }`. A search that would fetch many uncached documents loads the
-bundle first. Any mirror failure, invalid index, or wait past 5 seconds falls back to GitHub. The
-GitHub token is never sent to the mirror.
+bundle first. Mirror index and file requests wait up to 15 seconds before falling back to GitHub;
+the optional bundle gets 30 seconds. The GitHub token is never sent to the mirror.
 
 ## License
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nbtca/docs",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "description": "GitHub-backed document client for NBTCA",
   "type": "module",
   "main": "./dist/index.js",

src/__tests__/client.test.ts

@@ -830,6 +830,44 @@ describe('persistent store', () => {
     expect(spy).not.toHaveBeenCalled();
   });
 
+  it('lists and searches persisted documents after restarting offline', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi
+        .fn()
+        .mockResolvedValueOnce({ ok: true, json: async () => tree(guideSha) })
+        .mockResolvedValueOnce({ ok: true, text: async () => '# Guide' }),
+    );
+    const store = memoryStore();
+    const online = createDocsClient({ store });
+    const items = await online.listAll();
+    await online.getDocument('repair/guide.md');
+
+    vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new TypeError('fetch failed')));
+    const offline = createDocsClient({ store });
+    const cached = await offline.listAll();
+    expect(cached).toEqual(items);
+    cached.length = 0;
+    await expect(offline.listAll()).resolves.toEqual(items);
+    await expect(offline.listSections()).resolves.toEqual([{ path: 'repair', count: 1 }]);
+    await expect(offline.search('Guide')).resolves.toMatchObject([
+      { path: 'repair/guide.md', title: 'Guide' },
+    ]);
+  });
+
+  it.each([408, 429, 503])('uses the persisted tree after a transient HTTP %s', async (status) => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status })));
+    const items = [{ name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha }];
+    const store = memoryStore({ tree: JSON.stringify(items) });
+    await expect(createDocsClient({ store }).listAll()).resolves.toEqual(items);
+  });
+
+  it.each([401, 403, 404])('does not hide HTTP %s behind a persisted tree', async (status) => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status })));
+    const store = memoryStore({ tree: '[]' });
+    await expect(createDocsClient({ store }).listAll()).rejects.toMatchObject({ status });
+  });
+
   it('refetches a changed file and stores it under its own sha', async () => {
     vi.stubGlobal(
       'fetch',
@@ -1187,7 +1225,7 @@ describe('mirror', () => {
     expect(fetchMock).not.toHaveBeenCalled();
   });
 
-  it('gives up on a slow mirror after five seconds', async () => {
+  it('falls back to GitHub when the mirror exceeds fifteen seconds', async () => {
     vi.useFakeTimers();
     try {
       const fetchMock = vi.fn().mockImplementation((url: string, init: RequestInit) => {
@@ -1201,7 +1239,7 @@ describe('mirror', () => {
       });
       vi.stubGlobal('fetch', fetchMock);
       const request = createDocsClient({ mirror }).listAll();
-      await vi.advanceTimersByTimeAsync(5_000);
+      await vi.advanceTimersByTimeAsync(15_000);
       await expect(request).resolves.toHaveLength(3);
     } finally {
       vi.useRealTimers();
@@ -1221,6 +1259,35 @@ describe('mirror', () => {
       });
     }
 
+    it('loads a bundle through a slow mirror without contacting GitHub', async () => {
+      vi.useFakeTimers();
+      try {
+        const fetchMock = routes({
+          mirror: (url) =>
+            new Promise((resolve) => {
+              setTimeout(() => {
+                resolve(
+                  url.endsWith('/index.json')
+                    ? { ok: true, json: async () => mirrorTree }
+                    : bundleOf([{ path: 'repair/guide.md', sha: guideSha, content: '# Guide' }]),
+                );
+              }, 8_000);
+            }),
+        });
+        const client = createDocsClient({ mirror });
+        const request = client.prefetch();
+        await vi.advanceTimersByTimeAsync(16_000);
+        await expect(request).resolves.toBe(1);
+        await expect(client.getFile('repair/guide.md')).resolves.toBe('# Guide');
+        expect(fetchMock.mock.calls.map(([url]) => String(url))).toEqual([
+          'https://docs.example.org/docs-api/index.json',
+          'https://docs.example.org/docs-api/bundle.json',
+        ]);
+      } finally {
+        vi.useRealTimers();
+      }
+    });
+
     it('loads every verified file from the bundle in one request', async () => {
       const fetchMock = mirrorWith(
         bundleOf([{ path: 'repair/guide.md', sha: guideSha, content: '# Guide' }]),

src/client.ts

@@ -44,7 +44,7 @@ const SKIP = new Set([
 
 const TREE_KEY = 'tree';
 const BUNDLE_KEY = 'bundle';
-const MIRROR_TIMEOUT_MS = 5_000;
+const MIRROR_TIMEOUT_MS = 15_000;
 const BUNDLE_TIMEOUT_MS = 30_000;
 const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
@@ -491,13 +491,11 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   }
 
   function recoverFailure<T>(
-    cache: TtlCache<T>,
-    key: string,
+    stale: T | undefined,
     path: string,
     error: unknown,
     copy: (value: T) => T = (value) => value,
   ): T {
-    const stale = cache.getStale(key);
     if (stale !== undefined) return copy(stale);
     const message =
       error instanceof Error && error.name === 'AbortError' ? 'Request timed out' : String(error);
@@ -536,7 +534,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
       });
     } catch (error) {
       if (error instanceof DocsFetchError) throw error;
-      return recoverFailure(dirCache, path, path, error, copyItems);
+      return recoverFailure(dirCache.getStale(path), path, error, copyItems);
     }
   }
 
@@ -552,7 +550,6 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   }
 
   async function loadAll(generation: number): Promise<DocItem[]> {
-    const key = TREE_KEY;
     if (mirror !== undefined) {
       const mirrored = await fromMirror(mirror, 'index.json', async (response) => {
         const data = parseTreeResponse(await response.json());
@@ -564,13 +561,13 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
     try {
       return await withResponse(url, 20_000, async (response) => {
         if (!response.ok) {
-          const stale = treeCache.getStale(key);
+          const stale = knownTree();
           if (stale !== undefined && (await isTransientResponse(response))) return copyItems(stale);
           throw new DocsFetchError('', response.status, `HTTP ${String(response.status)}`);
         }
         const data = parseTreeResponse(await response.json());
         if (data.truncated) {
-          const stale = treeCache.getStale(key);
+          const stale = knownTree();
           if (stale !== undefined) return copyItems(stale);
           throw new DocsFetchError(
             '',
@@ -582,7 +579,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
       });
     } catch (error) {
       if (error instanceof DocsFetchError) throw error;
-      return recoverFailure(treeCache, key, '', error, copyItems);
+      return recoverFailure(knownTree(), '', error, copyItems);
     }
   }
 
@@ -632,7 +629,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
       });
     } catch (error) {
       if (error instanceof DocsFetchError) throw error;
-      return recoverFailure(fileCache, path, path, error);
+      return recoverFailure(fileCache.getStale(path), path, error);
     }
   }