diff --git a/package.json b/package.json index 80e78ec..4730dd4 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "description": "Lightweight patch-based directed acyclic graph realtime sync core", "license": "MIT", "type": "module", + "sideEffects": false, "repository": { "type": "git", "url": "git+https://github.com/sagemathinc/patchflow.git" diff --git a/src/db-document-immutable.ts b/src/db-document-immutable.ts index 23e0e3f..2456260 100644 --- a/src/db-document-immutable.ts +++ b/src/db-document-immutable.ts @@ -18,14 +18,31 @@ import { isArray, isObject, mapMergePatch, - mergeSet, - nonnullCols, toKey, toStr, type JsMap, } from "./db-util"; import type { DocCodec, Document } from "./types"; +// (immutable.js helpers, here so that db-util.ts does not import immutable) +function mergeSet( + obj: ImMap, + change: ImMap, +): ImMap { + change.forEach((v, k) => { + if (v === null || v == null) { + obj = obj.delete(k); + } else { + obj = obj.set(k, v); + } + }); + return obj; +} + +function nonnullCols(f: ImMap): ImMap { + return ImMap(f.filter((v) => v !== null)); +} + const DEFAULT_JSONL_SIZE_FACTOR = 1000; type RecordValue = ImMap | undefined; diff --git a/src/db-util.ts b/src/db-util.ts index 78932dc..7d4e47d 100644 --- a/src/db-util.ts +++ b/src/db-util.ts @@ -1,12 +1,20 @@ -import { Map as ImMap } from "immutable"; import jsonstable from "fast-json-stable-stringify"; import deepEqual from "fast-deep-equal/es6"; export { deepEqual }; export type JsMap = Record; +// An immutable.js Map (or OrderedMap), recognized as Map.isMap does, without +// importing immutable: the immer documents never need it. +const IS_IMMUTABLE_MAP = "@@__IMMUTABLE_MAP__@@"; +function isImmutableMap(value: unknown): value is { toJS(): unknown } { + return ( + !!value && typeof value === "object" && !!(value as Record)[IS_IMMUTABLE_MAP] + ); +} + export function toKey(value: unknown): string { - if (ImMap.isMap(value)) { + if (isImmutableMap(value)) { value = value.toJS(); } // must be stable, especially if value contains objects, which is technically allowed @@ -34,24 +42,6 @@ export function mapMergePatch(obj1: JsMap, obj2: JsMap): JsMap { return change; } -export function mergeSet( - obj: ImMap, - change: ImMap, -): ImMap { - change.forEach((v, k) => { - if (v === null || v == null) { - obj = obj.delete(k); - } else { - obj = obj.set(k, v); - } - }); - return obj; -} - -export function nonnullCols(f: ImMap): ImMap { - return ImMap(f.filter((v) => v !== null)); -} - export function isArray(x: unknown): x is unknown[] { return Array.isArray(x); } diff --git a/src/patch-graph.ts b/src/patch-graph.ts index 8aabe32..3f6b571 100644 --- a/src/patch-graph.ts +++ b/src/patch-graph.ts @@ -1,4 +1,3 @@ -import { List, Map } from "immutable"; import { LRUCache } from "lru-cache"; import { comparePatchId, decodePatchId } from "./patch-id"; import type { @@ -10,8 +9,9 @@ import type { PatchGraphValueOptions, } from "./types"; import { hashString, sameHashFormat } from "./value-hash"; +import { sameElements } from "./same-elements"; -type PatchMap = Map; +type PatchMap = Map; // (native maps: patch-graph does not need immutable.js) const DEFAULT_DEDUP_TOLERANCE = 3000; const DEFAULT_VALUE_CACHE_MAX_ENTRIES = 100; @@ -102,8 +102,8 @@ function patchCmp(a: Patch, b: Patch): number { } export class PatchGraph { - private patches: PatchMap = Map(); - private children: Map> = Map>(); + private patches: PatchMap = new Map(); + private children: Map> = new Map>(); private codec: DocCodec; public fileTimeDedupTolerance = DEFAULT_DEDUP_TOLERANCE; private mergeStrategy: MergeStrategy; @@ -506,7 +506,7 @@ export class PatchGraph { } private computeHeads(): string[] { - const allTimes = new Set(this.patches.keySeq().toArray()); + const allTimes = new Set(this.patches.keys()); const parents = new Set(); this.patches.forEach((patch) => { for (const p of patch.parents ?? []) { @@ -598,10 +598,9 @@ export class PatchGraph { versions(opts: { start?: string; end?: string } = {}): string[] { const { start, end } = opts; if (this.versionsCache == null) { - this.versionsCache = this.patches - .toArray() - .map(([, patch]) => patch.time) - .sort(comparePatchId); + this.versionsCache = Array.from(this.patches.values(), (patch) => patch.time).sort( + comparePatchId, + ); } return this.versionsCache.filter((t) => { if (start != null && comparePatchId(t, start) < 0) return false; @@ -1383,7 +1382,7 @@ export class PatchGraph { patch.patch && decodePatchId(patch.time).timeMs - decodePatchId(last.time).timeMs <= this.fileTimeDedupTolerance && - List(patch.patch as unknown[]).equals(List(last.patch as unknown[])) + sameElements(patch.patch, last.patch) ); } @@ -1407,9 +1406,7 @@ export class PatchGraph { history(opts: { start?: string; end?: string; includeSnapshots?: boolean } = {}): Patch[] { const { start, end, includeSnapshots = true } = opts; - return this.patches - .toArray() - .map(([, patch]) => patch) + return Array.from(this.patches.values()) .filter((p) => { if (start != null && comparePatchId(p.time, start) < 0) return false; if (end != null && comparePatchId(p.time, end) > 0) return false; diff --git a/src/same-elements.test.ts b/src/same-elements.test.ts new file mode 100644 index 0000000..f39240d --- /dev/null +++ b/src/same-elements.test.ts @@ -0,0 +1,125 @@ +// sameElements must decide exactly as immutable.js's List(a).equals(List(b)), +// which PatchGraph's file-load dedup used before it stopped importing immutable. +import { List, Map as ImMap, Set as ImSet, Record, Seq } from "immutable"; +import { sameElements } from "./same-elements"; +import { PatchGraph } from "./patch-graph"; +import { legacyPatchId } from "./patch-id"; +import type { DocCodec, Document } from "./types"; + +const listEquals = (a: unknown, b: unknown) => List(a as unknown[]).equals(List(b as unknown[])); + +describe("sameElements agrees with List(a).equals(List(b))", () => { + const shared = [1, 2]; + const Point = Record({ x: 0, y: 0 }); + function* gen(...xs: unknown[]) { + yield* xs; + } + // [name, () => [a, b]]: fresh values each call (generators are consumed) + const cases: [string, () => [unknown, unknown]][] = [ + // elements + [ + "equal numbers and strings", + () => [ + [1, "a", [0, "x"]], + [1, "a", [0, "x"]], + ], + ], + [ + "the same nested array", + () => [ + [shared, 3], + [shared, 3], + ], + ], + ["nested arrays with equal contents (by identity)", () => [[[1, 2]], [[1, 2]]]], + ["+0 and -0", () => [[0], [-0]]], + ["NaN and NaN", () => [[NaN], [NaN]]], + ["equal Dates", () => [[new Date(5)], [new Date(5)]]], + ["a Date and its time", () => [[new Date(5)], [5]]], + ["different Dates", () => [[new Date(5)], [new Date(6)]]], + ["equal immutable Maps (value objects)", () => [[ImMap({ a: 1 })], [ImMap({ a: 1 })]]], + ["different immutable Maps", () => [[ImMap({ a: 1 })], [ImMap({ a: 2 })]]], + ["boxed and primitive strings", () => [[Object("s")], ["s"]]], + ["null and undefined", () => [[null], [undefined]]], + ["0 and false", () => [[0], [false]]], + ["different lengths", () => [[1, 2], [1]]], + ["empty arrays", () => [[], []]], + // outer containers: what List() makes of them + ["strings", () => ["abc", "abc"]], + ["different strings", () => ["abc", "abd"]], + ["null and undefined patches", () => [null, undefined]], + ["array-like objects", () => [{ length: 2, 0: "a", 1: "b" }, ["a", "b"]]], + ["{length: 0}", () => [{ length: 0 }, []]], + ["native Sets", () => [new Set([1, 2]), new Set([1, 2])]], + ["a native Set and an array", () => [new Set([1, 2]), [1, 2]]], + ["native Maps (entries compare by identity)", () => [new Map([[1, 2]]), new Map([[1, 2]])]], + ["immutable Lists", () => [List([1, 2]), List([1, 2])]], + ["different immutable Lists", () => [List([1, 2]), List([2, 1])]], + ["immutable Sets", () => [ImSet([1, 2]), ImSet([1, 2])]], + ["immutable Maps (entries)", () => [ImMap({ a: 1 }), ImMap({ a: 1 })]], + ["indexed Seqs", () => [Seq([1, 2]), Seq([1, 2])]], + ["records", () => [Point({ x: 1 }), Point({ x: 1 })]], + ["generators", () => [gen(1, 2), gen(1, 2)]], + ["different generators", () => [gen(1, 2), gen(1, 3)]], + ]; + for (const [name, make] of cases) { + it(name, () => { + expect(sameElements(...make())).toBe(listEquals(...make())); + }); + } + it("a value List() cannot take is not equal (where List() throws)", () => { + expect(() => listEquals({ a: 1 }, { a: 1 })).toThrow(TypeError); + expect(sameElements({ a: 1 }, { a: 1 })).toBe(false); + }); +}); + +// A custom codec whose patches are immutable Lists of numbers to add: two +// identical file loads close in time must apply once (as before, when +// PatchGraph compared them with List.equals). +class Counter implements Document { + constructor(readonly n: number) {} + applyPatch(patch: unknown): Counter { + return new Counter(this.n + List(patch as number[]).reduce((s: number, x: number) => s + x, 0)); + } + applyPatchBatch(patches: unknown[]): Counter { + return patches.reduce((d: Counter, p) => d.applyPatch(p), this); + } + makePatch(other: Document): unknown { + return List([(other as Counter).n - this.n]); + } + isEqual(other?: Document): boolean { + return other instanceof Counter && other.n === this.n; + } + toString(): string { + return String(this.n); + } + set(value: unknown): Counter { + return new Counter(Number(value)); + } + get(): unknown { + return this.n; + } + count(): number { + return 1; + } +} +const counterCodec: DocCodec = { + fromString: (text) => new Counter(Number(text || 0)), + toString: (doc) => doc.toString(), + applyPatch: (doc, patch) => doc.applyPatch(patch), + applyPatchBatch: (doc, patches) => doc.applyPatchBatch(patches), + makePatch: (a, b) => a.makePatch(b), +}; + +describe("file-load dedup with a codec whose patches are immutable Lists", () => { + it("two identical file loads close in time apply once", () => { + const graph = new PatchGraph({ codec: counterCodec }); + const t1 = legacyPatchId(1); + const t2 = legacyPatchId(2); + graph.add([ + { time: t1, patch: List([1]), parents: [], file: true }, + { time: t2, patch: List([1]), parents: [], file: true }, + ]); + expect(graph.value().toString()).toBe("1"); + }); +}); diff --git a/src/same-elements.ts b/src/same-elements.ts new file mode 100644 index 0000000..ab81fa6 --- /dev/null +++ b/src/same-elements.ts @@ -0,0 +1,94 @@ +// Element-by-element equality of two patches' contents, exactly as +// immutable.js's List(a).equals(List(b)) decided it (immutable 4.3), without +// importing immutable (so that the immer documents do not bundle it). +// Internal: not exported from the package root. + +type Any = Record; // eslint-disable-line @typescript-eslint/no-explicit-any + +// immutable.js's `is`: SameValueZero, then valueOf() (Dates, boxed values), +// then value objects (anything with equals() and hashCode(), such as +// immutable collections). +export function immutableIs(a: unknown, b: unknown): boolean { + if (a === b || (a !== a && b !== b)) return true; + if (!a || !b) return false; + let x = a as Any; + let y = b as Any; + if (typeof x.valueOf === "function" && typeof y.valueOf === "function") { + x = x.valueOf(); + y = y.valueOf(); + if (x === y || (x !== x && y !== y)) return true; + if (!x || !y) return false; + } + return ( + typeof x.equals === "function" && + typeof x.hashCode === "function" && + typeof y.equals === "function" && + typeof y.hashCode === "function" && + !!x.equals(y) + ); +} + +// immutable.js's markers (what isList, isIndexed, ... check) +const has = (value: unknown, marker: string) => !!(value && (value as Any)[marker]); +const LIST = "@@__IMMUTABLE_LIST__@@"; +const INDEXED = "@@__IMMUTABLE_INDEXED__@@"; +const COLLECTION = "@@__IMMUTABLE_ITERABLE__@@"; +const KEYED = "@@__IMMUTABLE_KEYED__@@"; +const RECORD = "@@__IMMUTABLE_RECORD__@@"; + +// immutable.js's isArrayLike +function isArrayLike(value: unknown): boolean { + if (Array.isArray(value) || typeof value === "string") return true; + const v = value as Any; + return ( + !!v && + typeof v === "object" && + Number.isInteger(v.length) && + v.length >= 0 && + (v.length === 0 + ? Object.keys(v).length === 1 + : Object.prototype.hasOwnProperty.call(v, v.length - 1)) + ); +} + +function iteratorFn(value: unknown): (() => Iterator) | undefined { + const v = value as Any; + const f = v && ((typeof Symbol === "function" && v[Symbol.iterator]) || v["@@iterator"]); + return typeof f === "function" ? f : undefined; +} + +// The elements of List(value), in order: List() itself for immutable lists +// and indexed collections; a keyed collection's (or record's) entries; +// another collection's values; an array-like's indexed values (a string's +// UTF-16 units); an iterable's values. null where List() throws (a value of +// none of these kinds): such patches are not equal. +function elements(value: unknown): unknown[] | null { + if (value === undefined || value === null) return []; + const v = value as Any; + if (has(v, LIST) || has(v, INDEXED)) return v.toArray(); + if (has(v, COLLECTION)) + return has(v, KEYED) ? v.entrySeq().toArray() : v.toIndexedSeq().toArray(); + if (has(v, RECORD)) return v.toSeq().entrySeq().toArray(); + if (isArrayLike(v)) { + const out: unknown[] = []; + for (let i = 0; i < v.length; i++) out.push(v[i]); + return out; + } + const f = iteratorFn(v); + if (f) { + const it = f.call(v); + const out: unknown[] = []; + if (!it || typeof it.next !== "function") return out; + for (let step = it.next(); !step.done; step = it.next()) out.push(step.value); + return out; + } + return null; +} + +export function sameElements(a: unknown, b: unknown): boolean { + const x = elements(a); + const y = elements(b); + if (x == null || y == null || x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (!immutableIs(x[i], y[i])) return false; + return true; +}