From a785091db4ef1c8800cdbd67b14eca180ebb6524 Mon Sep 17 00:00:00 2001 From: Pocket Shark Date: Fri, 18 Sep 2026 13:53:01 +0800 Subject: [PATCH 1/4] feat(drive): remember permanent upload rejections across sync rounds --- .../commands/drive/path-executor.ts | 34 ++- src/handwritten/commands/drive/retry.ts | 12 ++ src/handwritten/commands/drive/state.ts | 32 +++ src/handwritten/commands/drive/sync.ts | 41 +++- test/handwritten/drive/retry.test.ts | 21 +- test/handwritten/drive/state.test.ts | 57 +++++ test/handwritten/drive/sync.test.ts | 202 +++++++++++++++++- 7 files changed, 390 insertions(+), 9 deletions(-) diff --git a/src/handwritten/commands/drive/path-executor.ts b/src/handwritten/commands/drive/path-executor.ts index a8f3d04..36e772b 100644 --- a/src/handwritten/commands/drive/path-executor.ts +++ b/src/handwritten/commands/drive/path-executor.ts @@ -18,8 +18,10 @@ import { } from "./local-mutations.js" import { classifyMergeText, conflictCopyPath, mergeText3 } from "./merge.js" import { resolveInsideRoot, validateDrivePath } from "./path-policy.js" +import { VERSION } from "../../../version.js" import { isDriveAuthFailure, + isPermanentUploadRejection, isRetryableDriveFailure, type DrivePathErrorSummary, } from "./retry.js" @@ -64,7 +66,14 @@ export async function executeDrivePathAction( if (action.type === "upload_create" || action.type === "upload_update") { const localPath = resolveInsideRoot(root, path) const { body, digest: uploadDigest } = await readStableUploadBody(localPath, local) - const uploaded = await api.uploadFile(state.library_id, path, body, uploadDigest, action.expectedEntryVersion) + let uploaded: UploadDriveFileResponse + try { + uploaded = await api.uploadFile(state.library_id, path, body, uploadDigest, action.expectedEntryVersion) + } catch (error) { + if (!isPermanentUploadRejection(error)) throw error + await recordDrivePathError(summary, undefined, path, error, { debug, op: "process" }) + return { state: await rememberUploadRejection(root, state, path, uploadDigest, error, clock), stop: false } + } durableStateRequired = true const nextState = cloneDriveState(state) nextState.entries[path] = stateEntryFromRemote(uploaded.entry, uploadDigest, clock) @@ -504,9 +513,32 @@ export function cloneDriveState(state: DriveState): DriveState { entries: { ...state.entries }, conflicts: { ...state.conflicts }, ...(state.scan_errors === undefined ? {} : { scan_errors: { ...state.scan_errors } }), + ...(state.upload_rejections === undefined ? {} : { upload_rejections: { ...state.upload_rejections } }), } } +// The scan cache is the fingerprint; it must describe the bytes just rejected. +async function rememberUploadRejection( + root: string, + state: DriveState, + path: string, + uploadDigest: string, + error: unknown, + clock: DriveClock, +): Promise { + const scanned = state.scan_cache?.[path] + if (scanned === undefined || scanned.sha256 !== uploadDigest) return state + const { mtime_ms, size_bytes, sha256 } = scanned + const { code, message } = drivePathErrorSummary(path, error) + const nextState = cloneDriveState(state) + nextState.upload_rejections = { + ...nextState.upload_rejections, + [path]: { mtime_ms, size_bytes, sha256, code, message, cli_version: VERSION, rejected_at: driveIsoTimestamp(clock) }, + } + await writeDriveState(root, nextState, clock) + return nextState +} + async function recordConflict( root: string, state: DriveState, diff --git a/src/handwritten/commands/drive/retry.ts b/src/handwritten/commands/drive/retry.ts index 32978e9..dfde51f 100644 --- a/src/handwritten/commands/drive/retry.ts +++ b/src/handwritten/commands/drive/retry.ts @@ -98,6 +98,18 @@ export function isRetryableDriveFailure(error: unknown): boolean { return classifyDriveRetry(error, 0) !== undefined } +const NON_PERMANENT_UPLOAD_STATUSES = new Set([401, 403, 408, 409, 429]) + +export function isPermanentUploadRejection(error: unknown): boolean { + return ( + error instanceof DriveHttpError && + error.status >= 400 && + error.status < 500 && + !NON_PERMANENT_UPLOAD_STATUSES.has(error.status) && + error.code !== "VERSION_CONFLICT" + ) +} + export function isDriveAuthFailure(error: unknown): boolean { const failure = error instanceof DriveRetryableSyncError ? error.cause : error const status = failure instanceof DriveHttpError ? failure.status : structuredStatus(failure) diff --git a/src/handwritten/commands/drive/state.ts b/src/handwritten/commands/drive/state.ts index 4412aa9..514985b 100644 --- a/src/handwritten/commands/drive/state.ts +++ b/src/handwritten/commands/drive/state.ts @@ -49,6 +49,16 @@ export interface DriveScanError { retryable: boolean } +export interface DriveUploadRejection { + mtime_ms: number + size_bytes: number + sha256: string + code: string + message: string + cli_version: string + rejected_at: string +} + export interface DriveState { schema_version: 1 library_id: string @@ -59,6 +69,7 @@ export interface DriveState { realtime?: DriveRealtimeState scan_cache?: Record scan_errors?: Record + upload_rejections?: Record // Latest manifest cursor seen; enables since_cursor delta fetches. manifest_cursor?: string } @@ -313,6 +324,7 @@ function isValidDriveState(value: unknown): value is DriveState { (value.realtime !== undefined && !isDriveRealtimeState(value.realtime)) || (value.scan_cache !== undefined && !isRecord(value.scan_cache)) || (value.scan_errors !== undefined && !isRecord(value.scan_errors)) || + (value.upload_rejections !== undefined && !isRecord(value.upload_rejections)) || (value.manifest_cursor !== undefined && typeof value.manifest_cursor !== "string") ) { return false @@ -333,6 +345,11 @@ function isValidDriveState(value: unknown): value is DriveState { if (!isDriveScanError(error)) return false } } + if (value.upload_rejections !== undefined) { + for (const rejection of Object.values(value.upload_rejections)) { + if (!isDriveUploadRejection(rejection)) return false + } + } return true } @@ -355,3 +372,18 @@ function isDriveScanError(value: unknown): value is DriveScanError { typeof value.retryable === "boolean" ) } + +function isDriveUploadRejection(value: unknown): value is DriveUploadRejection { + const allowedKeys = new Set(["mtime_ms", "size_bytes", "sha256", "code", "message", "cli_version", "rejected_at"]) + return ( + isRecord(value) && + Object.keys(value).every((key) => allowedKeys.has(key)) && + typeof value.mtime_ms === "number" && + typeof value.size_bytes === "number" && + typeof value.sha256 === "string" && + typeof value.code === "string" && + typeof value.message === "string" && + typeof value.cli_version === "string" && + typeof value.rejected_at === "string" + ) +} diff --git a/src/handwritten/commands/drive/sync.ts b/src/handwritten/commands/drive/sync.ts index 5977e11..3697851 100644 --- a/src/handwritten/commands/drive/sync.ts +++ b/src/handwritten/commands/drive/sync.ts @@ -17,6 +17,7 @@ import { type DriveStateEntry, } from "./state.js" import { render } from "../../output/render.js" +import { VERSION } from "../../../version.js" import { DriveRetryableSyncError, isRetryableDriveFailure, @@ -177,19 +178,47 @@ export async function runDriveSyncOnce( throw error } + const rejectedPaths = new Set() + const uploadRejections = { ...state.upload_rejections } + for (const [path, rejection] of Object.entries(uploadRejections)) { + const scanned = state.scan_cache?.[path] + if ( + scanned?.mtime_ms !== rejection.mtime_ms || + scanned.size_bytes !== rejection.size_bytes || + scanned.sha256 !== rejection.sha256 || + rejection.cli_version !== VERSION + ) { + delete uploadRejections[path] + continue + } + const action = decideDriveAction(state.entries[path], localFiles[path], remoteFiles[path]) + if (action.type !== "upload_create" && action.type !== "upload_update") continue + rejectedPaths.add(path) + const pathError = { path, code: rejection.code, message: rejection.message, retryable: false } + await recordDrivePathError(summary, undefined, path, undefined, { + appendPathResult: true, + debug, + op: "upload_rejected", + pathError, + }) + } + if (Object.keys(uploadRejections).length !== Object.keys(state.upload_rejections ?? {}).length) { + state = { ...state, upload_rejections: uploadRejections } + if (Object.keys(uploadRejections).length === 0) delete state.upload_rejections + await writeDriveState(root, state, clock) + } + + const pendingPaths = paths.filter((path) => !movedPaths.has(path) && !rejectedPaths.has(path)) // decideDriveAction is pure and reads only this path's slices of the // initial state, so this pre-pass total matches the loop's actions. - const total = paths.filter( - (path) => - !movedPaths.has(path) && - isActionableAction(decideDriveAction(state.entries[path], localFiles[path], remoteFiles[path])), + const total = pendingPaths.filter((path) => + isActionableAction(decideDriveAction(state.entries[path], localFiles[path], remoteFiles[path])), ).length let processed = 0 onProgress?.(processed, total) const processStartedMs = Date.now() - for (const path of paths) { - if (movedPaths.has(path)) continue + for (const path of pendingPaths) { const remote = remoteFiles[path] const local = localFiles[path] const action = decideDriveAction(state.entries[path], local, remote) diff --git a/test/handwritten/drive/retry.test.ts b/test/handwritten/drive/retry.test.ts index 482c60e..b864efc 100644 --- a/test/handwritten/drive/retry.test.ts +++ b/test/handwritten/drive/retry.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { DateTime } from "luxon" -import { DriveHttpError, classifyDriveRetry, parseRetryAfter } from "../../../src/handwritten/commands/drive/retry.js" +import { DriveHttpError, classifyDriveRetry, isPermanentUploadRejection, parseRetryAfter } from "../../../src/handwritten/commands/drive/retry.js" describe("Drive retry policy", () => { it("parses Retry-After delta seconds", () => { @@ -47,4 +47,23 @@ describe("Drive retry policy", () => { expect(classifyDriveRetry(reset, 1_000)).toEqual({ reason: "transient", delayMs: 1_000 }) }) + + it.each([ + [new DriveHttpError(400), true], + [new DriveHttpError(413), true], + [new DriveHttpError(422, { code: "FILE_TOO_LARGE" }), true], + [new DriveHttpError(401), false], + [new DriveHttpError(403), false], + [new DriveHttpError(408), false], + [new DriveHttpError(409), false], + [new DriveHttpError(429), false], + [new DriveHttpError(412, { code: "VERSION_CONFLICT" }), false], + [new DriveHttpError(399), false], + [new DriveHttpError(500), false], + [new DriveHttpError(502), false], + [Object.assign(new Error("HTTP 413"), { status: 413 }), false], + [new Error("local file changed after scan"), false], + ])("classifies %s as a permanent upload rejection: %s", (error, expected) => { + expect(isPermanentUploadRejection(error)).toBe(expected) + }) }) diff --git a/test/handwritten/drive/state.test.ts b/test/handwritten/drive/state.test.ts index fb6e969..e2d3f28 100644 --- a/test/handwritten/drive/state.test.ts +++ b/test/handwritten/drive/state.test.ts @@ -259,6 +259,63 @@ describe("drive state", () => { } }) + it("round-trips upload rejections in schema version 1", async () => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-state-upload-rejections-")) + const state = await initDriveState(root, "lib_1") + state.upload_rejections = { + "big.jsonl": { + mtime_ms: 1_789_000_000_000, + size_bytes: 291_000_000, + sha256: "a".repeat(64), + code: "DRIVE_PATH_ERROR", + message: "HTTP 413", + cli_version: "0.12.0", + rejected_at: "2026-09-18T00:00:00.000Z", + }, + } + + await writeDriveState(root, state) + + expect(await readDriveState(root)).toMatchObject({ + schema_version: 1, + upload_rejections: state.upload_rejections, + }) + }) + + it("rejects malformed upload rejection records", async () => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-state-bad-upload-rejections-")) + const path = join(root, ".wspc-drive", "state.json") + await mkdir(join(root, ".wspc-drive"), { recursive: true }) + const baseState = { + schema_version: 1, + library_id: "lib_1", + created_at: "2026-06-21T00:00:00.000Z", + updated_at: "2026-06-21T00:00:00.000Z", + entries: {}, + conflicts: {}, + } + const valid = { + mtime_ms: 1, + size_bytes: 2, + sha256: "a".repeat(64), + code: "DRIVE_PATH_ERROR", + message: "HTTP 413", + cli_version: "0.12.0", + rejected_at: "2026-09-18T00:00:00.000Z", + } + const malformed = [ + ["big.jsonl"], + { "big.jsonl": { ...valid, size_bytes: "2" } }, + { "big.jsonl": { ...valid, cli_version: undefined } }, + { "big.jsonl": { ...valid, response_body: "secret" } }, + ] + + for (const uploadRejections of malformed) { + await writeFile(path, JSON.stringify({ ...baseState, upload_rejections: uploadRejections }), { mode: 0o600 }) + await expect(readDriveState(root)).rejects.toThrow(/unsupported \.wspc-drive\/state\.json schema/) + } + }) + it("rejects malformed extended conflict metadata", async () => { const root = await mkdtemp(join(tmpdir(), "wspc-drive-state-bad-conflict-meta-")) await mkdir(join(root, ".wspc-drive"), { recursive: true }) diff --git a/test/handwritten/drive/sync.test.ts b/test/handwritten/drive/sync.test.ts index 008fabe..6f52cf0 100644 --- a/test/handwritten/drive/sync.test.ts +++ b/test/handwritten/drive/sync.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto" import { spawnSync } from "node:child_process" -import { mkdir, mkdtemp, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, readdir, rename, unlink, utimes, writeFile } from "node:fs/promises" import { join } from "node:path" import { tmpdir } from "node:os" import { DateTime } from "luxon" @@ -11,6 +11,7 @@ import { render } from "../../../src/handwritten/output/render.js" import type { UploadDriveFileResponse } from "../../../src/generated/sdk/index.js" import type { DriveClock } from "../../../src/handwritten/commands/drive/clock.js" import { DriveHttpError } from "../../../src/handwritten/commands/drive/retry.js" +import { VERSION } from "../../../src/version.js" const stateWriteControl = vi.hoisted(() => ({ failNext: undefined as undefined | ((state: unknown) => Error | undefined), @@ -182,6 +183,21 @@ function mkApi( return api } +function rejectUploadsOf(api: TestDriveSyncApi, rejectedPath: string, error: () => unknown = () => new DriveHttpError(413)): void { + const uploadFile = api.uploadFile.bind(api) + api.uploadFile = async (id, path, body, digest, expectedEntryVersion) => { + if (path !== rejectedPath) return uploadFile(id, path, body, digest, expectedEntryVersion) + api.uploads.push({ id, path, sha256: digest, expectedEntryVersion }) + throw error() + } +} + +function uploadCount(api: TestDriveSyncApi, path: string): number { + return api.uploads.filter((upload) => upload.path === path).length +} + +const rejected413 = { path: "big.jsonl", code: "DRIVE_PATH_ERROR", message: "HTTP 413", retryable: false } + describe("drive sync once", () => { beforeEach(() => { process.exitCode = undefined @@ -1755,6 +1771,190 @@ describe("drive sync once", () => { expect(errorEvents[0]?.fields).toMatchObject({ path: "gone.txt", message: expect.any(String) }) }) + describe("permanent upload rejection", () => { + it("uploads an unchanged rejected file only once across rounds and keeps reporting it", async () => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-sync-rejected-")) + await initDriveState(root, "lib_1") + await writeFile(join(root, "big.jsonl"), "huge") + const api = mkApi([{ entries: [] }, { entries: [] }, { entries: [] }]) + rejectUploadsOf(api, "big.jsonl") + + const summaries = [] + const progress: number[] = [] + summaries.push(await runDriveSyncOnce(root, api)) + const recorded = (await readDriveState(root)).upload_rejections?.["big.jsonl"] + summaries.push(await runDriveSyncOnce(root, api, undefined, (_processed, total) => progress.push(total))) + summaries.push(await runDriveSyncOnce(root, api)) + + expect(uploadCount(api, "big.jsonl")).toBe(1) + for (const summary of summaries) { + expect(summary.path_errors).toEqual([rejected413]) + expect(summary.errors).toBe(1) + expect(summary.paths).toEqual([{ path: "big.jsonl", action: "error" }]) + } + expect(progress).toEqual([0]) + const scanned = (await readDriveState(root)).scan_cache?.["big.jsonl"] + expect(recorded).toEqual({ + mtime_ms: scanned?.mtime_ms, + size_bytes: 4, + sha256: sha256("huge"), + code: "DRIVE_PATH_ERROR", + message: "HTTP 413", + cli_version: VERSION, + rejected_at: expect.any(String), + }) + }) + + it("keeps syncing other files around a rejected file", async () => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-sync-rejected-others-")) + await initDriveState(root, "lib_1") + await writeFile(join(root, "big.jsonl"), "huge") + await writeFile(join(root, "notes.txt"), "hello") + const api = mkApi([{ entries: [] }, { entries: [entry("notes.txt", "hello", 1)] }]) + rejectUploadsOf(api, "big.jsonl") + + const first = await runDriveSyncOnce(root, api) + await writeFile(join(root, "notes.txt"), "hello again") + const second = await runDriveSyncOnce(root, api) + + expect(first.uploaded).toBe(1) + expect(second.uploaded).toBe(1) + expect(second.path_errors).toEqual([rejected413]) + expect(api.uploads.map((upload) => upload.path)).toEqual(["big.jsonl", "notes.txt", "notes.txt"]) + }) + + async function rejectedLibrary(prefix: string, rounds: number): Promise<{ root: string; api: TestDriveSyncApi }> { + const root = await mkdtemp(join(tmpdir(), prefix)) + await initDriveState(root, "lib_1") + await writeFile(join(root, "big.jsonl"), "huge") + const api = mkApi(Array.from({ length: rounds }, () => ({ entries: [] }))) + rejectUploadsOf(api, "big.jsonl") + await runDriveSyncOnce(root, api) + return { root, api } + } + + it("retries once after the content changes", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-content-", 3) + await writeFile(join(root, "big.jsonl"), "huger") + + await runDriveSyncOnce(root, api) + await runDriveSyncOnce(root, api) + + expect(uploadCount(api, "big.jsonl")).toBe(2) + expect((await readDriveState(root)).upload_rejections?.["big.jsonl"]?.sha256).toBe(sha256("huger")) + }) + + it("retries once after only the mtime changes", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-mtime-", 3) + await utimes(join(root, "big.jsonl"), new Date("2030-01-01T00:00:00Z"), new Date("2030-01-01T00:00:00Z")) + + await runDriveSyncOnce(root, api) + await runDriveSyncOnce(root, api) + + expect(uploadCount(api, "big.jsonl")).toBe(2) + }) + + it("forgets the rejection when the file is deleted", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-delete-", 2) + await unlink(join(root, "big.jsonl")) + + const result = await runDriveSyncOnce(root, api) + + expect(result.path_errors ?? []).toEqual([]) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + + it("forgets the rejection when the file is renamed and uploads the new path once", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-rename-", 2) + await rename(join(root, "big.jsonl"), join(root, "renamed.jsonl")) + + const result = await runDriveSyncOnce(root, api) + + expect(result.path_errors ?? []).toEqual([]) + expect(uploadCount(api, "renamed.jsonl")).toBe(1) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + + it("forgets the rejection when the file becomes excluded", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-exclude-", 2) + await writeFile(join(root, ".wspc-drive", "ignore"), "big.jsonl\n") + + const result = await runDriveSyncOnce(root, api) + + expect(result.path_errors ?? []).toEqual([]) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + + it("retries once when the rejection was recorded by another CLI version", async () => { + const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-version-", 3) + const state = await readDriveState(root) + state.upload_rejections!["big.jsonl"]!.cli_version = "0.0.0-older" + await writeDriveState(root, state) + + await runDriveSyncOnce(root, api) + await runDriveSyncOnce(root, api) + + expect(uploadCount(api, "big.jsonl")).toBe(2) + expect((await readDriveState(root)).upload_rejections?.["big.jsonl"]?.cli_version).toBe(VERSION) + }) + + it("removes the rejection once a changed file uploads successfully", async () => { + const { root } = await rejectedLibrary("wspc-drive-sync-rejected-success-", 1) + await writeFile(join(root, "big.jsonl"), "small") + const api = mkApi([{ entries: [] }]) + + const result = await runDriveSyncOnce(root, api) + + expect(result.uploaded).toBe(1) + expect(result.path_errors ?? []).toEqual([]) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + + it.each([ + ["502", () => new DriveHttpError(502)], + ["429", () => new DriveHttpError(429)], + ["408", () => new DriveHttpError(408)], + ["401", () => new DriveHttpError(401)], + ["403", () => new DriveHttpError(403)], + ["VERSION_CONFLICT", () => new DriveHttpError(409, { code: "VERSION_CONFLICT" })], + ])("does not remember a %s upload failure", async (_name, error) => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-sync-not-rejected-")) + await initDriveState(root, "lib_1") + await writeFile(join(root, "big.jsonl"), "huge") + const api = mkApi([{ entries: [] }]) + rejectUploadsOf(api, "big.jsonl", error) + + await runDriveSyncOnce(root, api).catch(() => undefined) + + expect(uploadCount(api, "big.jsonl")).toBe(1) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + + it.each([ + ["a local read error", Object.assign(new Error("permission denied"), { code: "EACCES" })], + ["a local change after scan", undefined], + ])("does not remember %s before upload", async (_name, readError) => { + const root = await mkdtemp(join(tmpdir(), "wspc-drive-sync-not-rejected-read-")) + await initDriveState(root, "lib_1") + await writeFile(join(root, "big.jsonl"), "huge") + const api = mkApi([{ entries: [] }]) + rejectUploadsOf(api, "big.jsonl") + const getManifest = api.getManifest.bind(api) + api.getManifest = vi.fn(async (id, cursor) => { + const page = await getManifest(id, cursor) + if (readError === undefined) await writeFile(join(root, "big.jsonl"), "changed") + else scannerControl.afterHash = () => { throw readError } + return page + }) + + const result = await runDriveSyncOnce(root, api) + + expect(result.errors).toBe(1) + expect(uploadCount(api, "big.jsonl")).toBe(0) + expect((await readDriveState(root)).upload_rejections).toBeUndefined() + }) + }) + it("renders command summary and sets exit code for conflicts", async () => { const root = await mkdtemp(join(tmpdir(), "wspc-drive-sync-command-")) const state = await initDriveState(root, "lib_1") From 733b8dd0316d26dce5cea85715ede4575d940543 Mon Sep 17 00:00:00 2001 From: Pocket Shark Date: Fri, 18 Sep 2026 13:54:38 +0800 Subject: [PATCH 2/4] feat(drive): skip watch syncs for excluded fs event paths --- .../commands/drive/exclude-rules.ts | 2 +- src/handwritten/commands/drive/watch.ts | 16 ++++++- test/handwritten/drive/watch.test.ts | 44 +++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/handwritten/commands/drive/exclude-rules.ts b/src/handwritten/commands/drive/exclude-rules.ts index afb9dbf..43447ad 100644 --- a/src/handwritten/commands/drive/exclude-rules.ts +++ b/src/handwritten/commands/drive/exclude-rules.ts @@ -20,7 +20,7 @@ export interface DriveExcludeRules { matches(path: string, kind?: "file" | "directory"): boolean } -const emptyDriveExcludeRules: DriveExcludeRules = { +export const emptyDriveExcludeRules: DriveExcludeRules = { size: 0, matches: () => false, } diff --git a/src/handwritten/commands/drive/watch.ts b/src/handwritten/commands/drive/watch.ts index 6f17d8d..6c977bf 100644 --- a/src/handwritten/commands/drive/watch.ts +++ b/src/handwritten/commands/drive/watch.ts @@ -5,6 +5,7 @@ import { basename, relative, resolve } from "node:path" import { loadRealtimeAuthHeaders } from "../../auth/load-sdk-client.js" import { render, shouldOutputJson } from "../../output/render.js" import { createDriveDebugLogger, noopDriveDebugLogger, type DriveDebugLogger } from "./debug-log.js" +import { emptyDriveExcludeRules, loadDriveExcludeRules, type DriveExcludeRules } from "./exclude-rules.js" import { createDriveRealtimeSource } from "./realtime.js" import { isInternalSyncArtifactName } from "./scanner.js" import { DRIVE_DIR, ensureDriveRealtimeState, readDriveState, writeDriveRealtimeState } from "./state.js" @@ -96,6 +97,17 @@ export async function runDriveWatch(root: string, options: DriveWatchOptions = { let stopWatch: (() => void) | undefined let stopError: unknown let cleanupSignalListeners = () => {} + let excludeRules: DriveExcludeRules = emptyDriveExcludeRules + let excludeRulesLoad = Promise.resolve() + + // Serialized so an older ignore read can never overwrite a newer one. An + // unparsable ignore file filters nothing; sync reports the parse error. + function reloadExcludeRules(): Promise { + excludeRulesLoad = excludeRulesLoad.then(async () => { + excludeRules = await loadDriveExcludeRules(root).catch(() => emptyDriveExcludeRules) + }) + return excludeRulesLoad + } async function requestSync(): Promise { if (stopped) return @@ -252,6 +264,7 @@ export async function runDriveWatch(root: string, options: DriveWatchOptions = { writeRealtimeState: (next) => writeDriveRealtimeState(root, next), }) } + await reloadExcludeRules() source = options.source ?? createDefaultWatchSource(root) source.onChange((path) => { if (path === undefined) { @@ -264,13 +277,14 @@ export async function runDriveWatch(root: string, options: DriveWatchOptions = { if (drivePath === `${DRIVE_DIR}/ignore`) { fullReconciliationRequired = true dbg.log("fs_event", { path: drivePath, full_reconciliation: true }) - scheduleSync(debounceMs, "ignore") + void reloadExcludeRules().then(() => scheduleSync(debounceMs, "ignore")) return } if (isDriveInternalPath(root, path)) return // Download/backup/merge temp files are our own writes; reacting to them // would chain an extra sync after every applied remote change. if (isInternalSyncArtifactName(basename(path))) return + if (excludeRules.matches(drivePath)) return dirtyPaths.add(drivePath) dbg.log("fs_event", { path: drivePath }) scheduleSync(debounceMs, "local") diff --git a/test/handwritten/drive/watch.test.ts b/test/handwritten/drive/watch.test.ts index 9386a25..3951543 100644 --- a/test/handwritten/drive/watch.test.ts +++ b/test/handwritten/drive/watch.test.ts @@ -1,3 +1,6 @@ +import { mkdir, mkdtemp, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" import { createChokidarSource, @@ -625,6 +628,47 @@ describe("drive watch", () => { await watching }) + it("does not schedule syncs for fs events on excluded paths and follows ignore edits", async () => { + vi.useRealTimers() + const root = await mkdtemp(join(tmpdir(), "wspc-drive-watch-exclude-")) + const ignorePath = join(root, ".wspc-drive", "ignore") + await mkdir(join(root, ".wspc-drive")) + await writeFile(ignorePath, "**/.DS_Store*\n") + const source = fakeSource() + const timer = manualTimer() + const runSync = vi.fn(async () => syncSummary()) + const fireTimers = async () => { + for (const scheduled of timer.pending.splice(0)) scheduled.fire() + await flushMicrotasks() + } + const watching = runDriveWatch(root, { source, realtimeSource: fakeRealtimeSource(), runSync, readState, onEvent: vi.fn(), timer }) + await source.waitForSubscription() + await flushMicrotasks() + expect(runSync).toHaveBeenCalledTimes(1) + + source.emit(join(root, ".DS_Store")) + source.emit(join(root, "sub", ".DS_Store")) + await fireTimers() + expect(runSync).toHaveBeenCalledTimes(1) + + source.emit(join(root, "notes.md")) + await fireTimers() + expect(runSync).toHaveBeenCalledTimes(2) + + await writeFile(ignorePath, "") + source.emit(ignorePath) + await vi.waitFor(() => expect(timer.pending).toHaveLength(1)) + await fireTimers() + expect(runSync).toHaveBeenCalledTimes(3) + expect(runSync).toHaveBeenLastCalledWith(root, expect.any(Function), undefined) + + source.emit(join(root, ".DS_Store")) + await fireTimers() + expect(runSync).toHaveBeenCalledTimes(4) + process.emit("SIGTERM") + await watching + }) + it("ignores internal drive state events", async () => { const source = fakeSource() const onEvent = vi.fn() From b5514ecb19a4122b51cef72b8608368a79f253be Mon Sep 17 00:00:00 2001 From: Pocket Shark Date: Fri, 18 Sep 2026 13:55:18 +0800 Subject: [PATCH 3/4] docs(drive): align rejection cleanup decisions with the scan-cache pass --- .../2026-09-18-drive-permanent-upload-rejection-design.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-09-18-drive-permanent-upload-rejection-design.md b/docs/superpowers/specs/2026-09-18-drive-permanent-upload-rejection-design.md index ca438e0..eb83491 100644 --- a/docs/superpowers/specs/2026-09-18-drive-permanent-upload-rejection-design.md +++ b/docs/superpowers/specs/2026-09-18-drive-permanent-upload-rejection-design.md @@ -65,8 +65,8 @@ watch 啟動時載入 `.wspc-drive/ignore` 的 exclude rules；本機 fs event - **State 欄位**：`DriveState` 新增 optional `upload_rejections?: Record`，`DriveUploadRejection` 為 `{ mtime_ms: number; size_bytes: number; sha256: string; code: string; message: string; cli_version: string; rejected_at: string }`。`rejected_at` 用 `driveIsoTimestamp(clock)`。`isValidDriveState()` 驗證此欄位與每筆紀錄形狀，比照 `scan_errors`。空 record 時刪除欄位，不寫空物件。`schema_version` 不變。 - **CLI version**：使用 `src/version.ts` 的 `VERSION`。 - **判定**：在 `retry.ts` 新增 `isPermanentUploadRejection(error)`：`error instanceof DriveHttpError`、`400 <= status < 500`、status 不在 `{401, 403, 408, 409, 429}`、`code !== "VERSION_CONFLICT"`。只在 `executeDrivePathAction()` 的 upload 分支、`api.uploadFile()` 丟出時判定；讀檔（`readStableUploadBody`）失敗不判定。 -- **記錄**：upload 分支 catch 到 Permanent Upload Rejection 時，以 `state.scan_cache[path]` 為 fingerprint；若沒有 cache entry 或其 `sha256` 與本次 upload digest 不同，則不記錄（本輪 path error 照舊）。寫入 `state.json` 後回傳新 state，並照現行 `recordDrivePathError()` 回報。上傳成功分支刪除該 path 紀錄。 -- **略過與清除**：`runDriveSyncOnce()` 在 scan 完成、`applyRenamesAsMoves()` 之後、計算 progress `total` 之前，針對每筆紀錄：path 不在 `localFiles`、scan cache fingerprint 不同或 `cli_version !== VERSION` 就刪除；其餘若本輪決策為 `upload_create`／`upload_update`，將 path 加入略過集合，並以 `recordDrivePathError(..., { op: "upload_rejected", pathError: { path, code, message, retryable: false } })` 回報。略過集合的 path 不計入 `total`、不呼叫 `executeDrivePathAction()`。紀錄有變動才寫 `state.json`。`removeExcludedState()` 一併移除 excluded path 的紀錄。Incremental scan 下 `localFiles` 仍涵蓋完整本機 view（由 scan cache 補齊），故上述判斷在 local-trigger 輪次同樣成立。 +- **記錄**：upload 分支 catch 到 Permanent Upload Rejection 時，以 `state.scan_cache[path]` 為 fingerprint；若沒有 cache entry 或其 `sha256` 與本次 upload digest 不同，則不記錄（本輪 path error 照舊）。寫入 `state.json` 後回傳新 state，並照現行 `recordDrivePathError()` 回報。上傳成功分支不需另外刪除紀錄：會走到上傳的 path，其紀錄已在下述略過與清除步驟因 fingerprint 或 CLI version 不同而刪除。 +- **略過與清除**：`runDriveSyncOnce()` 在 scan 完成、`applyRenamesAsMoves()` 之後、計算 progress `total` 之前，針對每筆紀錄：本輪 scan cache 沒有該 path（刪除、改名、被 exclude rule 排除或 scan error；scan cache 與 `localFiles` 涵蓋相同 path）、fingerprint 不同或 `cli_version !== VERSION` 就刪除；其餘若本輪決策為 `upload_create`／`upload_update`，將 path 加入略過集合，並以 `recordDrivePathError(..., { op: "upload_rejected", pathError: { path, code, message, retryable: false } })` 回報。略過集合的 path 不計入 `total`、不呼叫 `executeDrivePathAction()`。紀錄有變動才寫 `state.json`。Excluded path 不會出現在 scan cache，由此步驟清除，`removeExcludedState()` 不需另外處理。Incremental scan 下 `localFiles` 仍涵蓋完整本機 view（由 scan cache 補齊），故上述判斷在 local-trigger 輪次同樣成立。 - **Watch**：`runDriveWatch()` 啟動時以 `loadDriveExcludeRules(root)` 載入規則，失敗（包含 `DriveIgnoreError`）時使用不過濾的 matcher。`onChange` 在 internal path 與 temp artifact 判斷之後、加入 `dirtyPaths` 之前，以 `excludeRules.matches(drivePath)` 過濾；`ignore` event 先重新載入規則再排程 full sync。`path === undefined` 的 event 維持 full reconciliation。 - **Debug log**：略過時沿用 `error` debug event，`op: "upload_rejected"`，方便與實際上傳失敗（`op: "process"`）區分。 - **Rollout／rollback**：隨下一個 CLI minor release 發佈，wspc-drive 升級內附 CLI 即生效，不需 app 改動。舊版 CLI 讀到含 `upload_rejections` 的 `state.json` 時，現行 validator 忽略未知欄位，回退只會恢復舊的重傳行為。 From 26573896acc6703d177d898065f481df37c52280 Mon Sep 17 00:00:00 2001 From: Pocket Shark Date: Fri, 18 Sep 2026 13:55:18 +0800 Subject: [PATCH 4/4] test(drive): set rejected file mtime with numeric utimes --- test/handwritten/drive/sync.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/handwritten/drive/sync.test.ts b/test/handwritten/drive/sync.test.ts index 6f52cf0..663ec79 100644 --- a/test/handwritten/drive/sync.test.ts +++ b/test/handwritten/drive/sync.test.ts @@ -1846,7 +1846,7 @@ describe("drive sync once", () => { it("retries once after only the mtime changes", async () => { const { root, api } = await rejectedLibrary("wspc-drive-sync-rejected-mtime-", 3) - await utimes(join(root, "big.jsonl"), new Date("2030-01-01T00:00:00Z"), new Date("2030-01-01T00:00:00Z")) + await utimes(join(root, "big.jsonl"), 1_893_456_000, 1_893_456_000) await runDriveSyncOnce(root, api) await runDriveSyncOnce(root, api)