From d717b1d4f38403a3b3900ba35899f026a08d6879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rg=20Lehni?= Date: Mon, 12 Oct 2026 21:45:06 +0200 Subject: [PATCH] Rewrite upsertGraph() and insertGraph() (#1268, #1378, #1694, #853) --- doc/guide/validation.md | 22 + doc/release-notes/changelog.md | 13 +- doc/release-notes/migration.md | 16 +- lib/model/graph/ModelGraph.js | 74 - lib/model/graph/ModelGraphBuilder.js | 233 ---- lib/model/graph/ModelGraphEdge.js | 46 - lib/model/graph/ModelGraphNode.js | 231 ---- lib/queryBuilder/graph/GraphAction.js | 124 -- lib/queryBuilder/graph/GraphData.js | 82 -- lib/queryBuilder/graph/GraphFetcher.js | 191 --- lib/queryBuilder/graph/GraphLoader.js | 249 ++++ lib/queryBuilder/graph/GraphNode.js | 232 ++++ .../graph/GraphNodeDbExistence.js | 178 --- lib/queryBuilder/graph/GraphOperation.js | 21 - lib/queryBuilder/graph/GraphOptions.js | 241 +--- lib/queryBuilder/graph/GraphPlan.js | 301 +++++ lib/queryBuilder/graph/GraphUpsert.js | 524 ++------ lib/queryBuilder/graph/GraphWriter.js | 1194 +++++++++++++++++ lib/queryBuilder/graph/delete/GraphDelete.js | 26 - .../graph/delete/GraphDeleteAction.js | 76 -- .../DeferredRelationGraphInsertAction.js | 52 - lib/queryBuilder/graph/insert/GraphInsert.js | 249 ---- .../graph/insert/GraphInsertAction.js | 236 ---- .../graph/insert/JoinRowGraphInsertAction.js | 87 -- lib/queryBuilder/graph/patch/GraphPatch.js | 14 - .../graph/patch/GraphPatchAction.js | 305 ----- lib/queryBuilder/graph/patch/GraphRelate.js | 94 -- .../recursiveUpsert/GraphRecursiveUpsert.js | 24 - .../GraphRecursiveUpsertAction.js | 119 -- lib/relations/Relation.js | 7 + .../belongsToMany/BelongsToManyRelation.js | 5 + .../belongsToOne/BelongsToOneRelation.js | 4 + lib/relations/hasMany/HasManyRelation.js | 4 + .../manyToMany/ManyToManyRelation.js | 4 + lib/utils/knexUtils.js | 21 + tests/integration/graph/GraphInsert.js | 58 +- tests/integration/insertGraph.js | 21 + tests/integration/misc/issue853.js | 140 ++ .../integration/misc/upsertGraphEdgeCases.js | 686 ++++++++++ tests/integration/misc/upsertGraphOrder.js | 12 + tests/integration/misc/upsertGraphQueries.js | 162 +++ .../misc/upsertGraphRefManyToMany.js | 86 ++ tests/integration/upsertGraph.js | 60 + tests/unit/queryBuilder/GraphOptions.js | 37 +- typings/objection/index.d.ts | 1 + 45 files changed, 3393 insertions(+), 3169 deletions(-) delete mode 100644 lib/model/graph/ModelGraph.js delete mode 100644 lib/model/graph/ModelGraphBuilder.js delete mode 100644 lib/model/graph/ModelGraphEdge.js delete mode 100644 lib/model/graph/ModelGraphNode.js delete mode 100644 lib/queryBuilder/graph/GraphAction.js delete mode 100644 lib/queryBuilder/graph/GraphData.js delete mode 100644 lib/queryBuilder/graph/GraphFetcher.js create mode 100644 lib/queryBuilder/graph/GraphLoader.js create mode 100644 lib/queryBuilder/graph/GraphNode.js delete mode 100644 lib/queryBuilder/graph/GraphNodeDbExistence.js delete mode 100644 lib/queryBuilder/graph/GraphOperation.js create mode 100644 lib/queryBuilder/graph/GraphPlan.js create mode 100644 lib/queryBuilder/graph/GraphWriter.js delete mode 100644 lib/queryBuilder/graph/delete/GraphDelete.js delete mode 100644 lib/queryBuilder/graph/delete/GraphDeleteAction.js delete mode 100644 lib/queryBuilder/graph/insert/DeferredRelationGraphInsertAction.js delete mode 100644 lib/queryBuilder/graph/insert/GraphInsert.js delete mode 100644 lib/queryBuilder/graph/insert/GraphInsertAction.js delete mode 100644 lib/queryBuilder/graph/insert/JoinRowGraphInsertAction.js delete mode 100644 lib/queryBuilder/graph/patch/GraphPatch.js delete mode 100644 lib/queryBuilder/graph/patch/GraphPatchAction.js delete mode 100644 lib/queryBuilder/graph/patch/GraphRelate.js delete mode 100644 lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsert.js delete mode 100644 lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsertAction.js create mode 100644 tests/integration/misc/issue853.js create mode 100644 tests/integration/misc/upsertGraphEdgeCases.js create mode 100644 tests/integration/misc/upsertGraphQueries.js create mode 100644 tests/integration/misc/upsertGraphRefManyToMany.js diff --git a/doc/guide/validation.md b/doc/guide/validation.md index 18961b83e..f836c0b0e 100644 --- a/doc/guide/validation.md +++ b/doc/guide/validation.md @@ -75,3 +75,25 @@ try { } } ``` + +## Write-only properties + +`insertGraph()` and `upsertGraph()` validate their models once the references and keys of the graph are resolved, after the models are created. A property that a setter consumes when the model is created, e.g. a `password` that a setter turns into a hash, is gone by then, and fails a `required` rule or skips its other rules. Keep such properties on the model and transform them in the `$beforeInsert` and `$beforeUpdate` hooks, which run after the validation in all queries: + +```js +class User extends Model { + static jsonSchema = { + type: 'object', + required: ['password'], + properties: { + password: { type: 'string', minLength: 8 } + }, + }; + + async $beforeInsert(context) { + await super.$beforeInsert(context); + this.passwordHash = await hash(this.password); + delete this.password; + } +} +``` diff --git a/doc/release-notes/changelog.md b/doc/release-notes/changelog.md index 50d35cb2e..4f07336d7 100644 --- a/doc/release-notes/changelog.md +++ b/doc/release-notes/changelog.md @@ -54,9 +54,13 @@ - `$fetchGraph()`, `fetchGraph()` and `$relatedQuery()` throw an error when owner models loaded from the database are missing a join property, e.g. an id left out of a partial `select()`, instead of warning and finding no related models for them. Owner models built in memory, e.g. with `fromJson()`, find no related models for a missing join property without a warning. This includes the foreign key of a `BelongsToOneRelation`, which 3.x didn't report unless it was part of the id. Models returned by `insert()` aren't loaded from the database, also with `returning()`. `$clone()` keeps whether a model was loaded from the database. See [migration](/release-notes/migration.md#missing-and-array-join-properties-throw) [#1832](https://github.com/Vincit/objection.js/issues/1832) - `withGraphFetched()` and `$fetchGraph()` throw an error when related models come back without their join property, e.g. snake_case relation mappings with `knexSnakeCaseMappers()`, instead of warning and dropping them. See [migration](/release-notes/migration.md#missing-and-array-join-properties-throw) [#2258](https://github.com/Vincit/objection.js/issues/2258) - Fetching a relation for owner models whose join property holds an array, e.g. a JSON array of keys, throws an error instead of warning and finding no related models. `$relatedQuery()` keeps working for them. See [migration](/release-notes/migration.md#missing-and-array-join-properties-throw) [#114](https://github.com/ditojs/objection/issues/114) -- `upsertGraph()` relates the rows that move to existing owners of a `HasManyRelation` or `HasOneRelation` before it deletes the removed rows, so rows moved out of a deleted parent, up a level or to a sibling are kept, where `ON DELETE CASCADE` deleted them and foreign keys without `ON DELETE` action failed. The update hooks of these relates run before the delete hooks. A relate into a `HasOneRelation` still waits for the removal of the current model, or for the relate that moves it to another owner, also a new one or one whose slot is freed first, but not into the subgraph of a related model, which is upserted last. A row related from another owner that takes the unique key of a row that the graph removes in its new place, e.g. with `unique(ownerId, name)`, fails with a `UniqueViolationError`, where 3.x removed the other row first. See [migration](/release-notes/migration.md#upsertgraph-relates-moved-rows-before-it-deletes) [#214](https://github.com/ditojs/objection/issues/214) +- `upsertGraph()` relates the rows that move to existing owners of a `HasManyRelation` or `HasOneRelation` before it deletes the removed rows, so rows moved out of a deleted parent, up a level or to a sibling are kept, where `ON DELETE CASCADE` deleted them and foreign keys without `ON DELETE` action failed. The update hooks of these relates run before the delete hooks. Rows that wait for the keys or references of models that the graph inserts are moved without them, and patched again once these are known, so a unique key at the new owner that only these values satisfy fails. A relate into a `HasOneRelation` still waits for the removal of the current model, or for the relate that moves it to another owner, also a new one or one whose slot is freed first. A row related from another owner that takes the unique key of a row that the graph removes in its new place, e.g. with `unique(ownerId, name)`, fails with a `UniqueViolationError`, where 3.x removed the other row first. See [migration](/release-notes/migration.md#upsertgraph-relates-moved-rows-before-it-deletes) [#214](https://github.com/ditojs/objection/issues/214) - `findById()` and `findByIds()` add their id condition in its own parentheses, like `$query()` and the `*ById()` writes, so `orWhere()` can't widen it: `findById(1).orWhere(b)` is `id = 1 and (b)`, where 3.x built `id = 1 or b`, and `where(a).orWhere(b).findById(1)` is `(a or b) and id = 1`. See [migration](/release-notes/migration.md#findbyid-and-findbyids-keep-their-id-condition-in-its-own-group) [#271](https://github.com/ditojs/objection/issues/271) - On PostgreSQL drivers (`pg`, `pgnative`, `cockroachdb`), JSON attributes keep the values that the driver returns, so a `json` / `jsonb` column holding a string that looks like JSON, e.g. `'123'` or `'{"a":1}'`, is no longer parsed a second time, and JSON in `text` columns is no longer parsed. Set `jsonColumnFormat = 'text'` on models that keep JSON in `text` columns on PostgreSQL. Other drivers parse JSON attributes as in 3.x. See [migration](/release-notes/migration.md#json-in-postgresql-text-columns-is-no-longer-parsed) [#205](https://github.com/ditojs/objection/issues/205) +- `upsertGraph()` upserts the models below related models in the same graph, instead of in nested `upsertGraph()` calls after all other writes: their rows are fetched together, their writes run in the same steps as the rest of the graph, and `#ref` can point between them. A related model with relations whose row doesn't exist throws a `NotFoundError` before anything is written, where 3.x wrote the rest of the graph first. See [migration](/release-notes/migration.md#upsertgraph-upserts-below-related-models-in-the-same-graph) +- `insertGraph()` and `upsertGraph()` write in fewer queries: rows are removed with one query per relation, with all owners as `items` in the static hooks, instead of one query per owner, and the changed rows of models without update hooks are patched with one query per table. See [migration](/release-notes/migration.md#insertgraph-and-upsertgraph-write-in-fewer-queries) +- Custom relation classes that extend `Relation` directly declare where their keys are stored with a `keySide` getter (`'owner'`, `'related'` or `'join'`) to be used in `insertGraph()` and `upsertGraph()`. Relations that extend the built-in relation classes inherit it +- `insertGraph()` and `upsertGraph()` never return the `#id`, `#ref`, `#dbRef`, `#unrelate` and `#delete` markers, where 3.x returned them on models that the graph ignored or removed, and graph options with values of the wrong type throw also when no model of the graph reads them ### Features @@ -75,6 +79,12 @@ - The error of `upsertGraph()` queries with other query builder calls names `$query()`, which adds such a call too [#1168](https://github.com/Vincit/objection.js/issues/1168) - `upsertGraphAndFetch()` loads the relations of a `withGraphFetched()` on its query with their modifiers, named modifiers, aliases and options into the fetched models, like `insertGraphAndFetch()`, instead of ignoring the modifiers and aliases [#839](https://github.com/Vincit/objection.js/issues/839) - On SQLite, `insert()` stores arrays in columns that aren't `jsonAttributes` as JSON, like objects, instead of `'[object Object]'` +- `upsertGraph()` keeps the join table extras of each join row when it relates the same model twice, e.g. a person as director and producer of a movie, instead of writing the last extras to both [#1268](https://github.com/Vincit/objection.js/issues/1268) +- `insertGraph()` and `upsertGraph()` resolve a `#ref{}` in a foreign key that the relation sets too, instead of failing with `replace is not a function` [#1378](https://github.com/Vincit/objection.js/issues/1378) +- `insertGraph()` and `upsertGraph()` insert several models without properties on PostgreSQL, instead of failing with "The query is empty" [#1694](https://github.com/Vincit/objection.js/issues/1694) +- `insertGraph()` and `upsertGraph()` fetch the keys that the database generates for inserted rows, like a uuid that isn't the id, when relations join through them, instead of writing join rows and foreign keys without them [#853](https://github.com/Vincit/objection.js/issues/853) +- `upsertGraph()` gives new models the keys of their existing parents that the graph leaves out, e.g. a `code` column that the relation joins through, instead of inserting them with NULL +- `insertGraph()` and `upsertGraph()` stay under the number of values that a query can bind on SQL Server, PostgreSQL and SQLite when they insert, patch and remove many rows, and fetch many roots or related models - A many-to-many `patch()` / `update()` with join table extras and `limit()` on MySQL and SQL Server can order by columns of the join table [#161](https://github.com/ditojs/objection/issues/161) - `cancelQuery()` with a single row in the `beforeInsert` hook of an `insert()` merges it into the inserted model, like an array of rows, instead of ignoring it [#66](https://github.com/ditojs/objection/issues/66) - `insertGraph()` and `upsertGraph()` reject only after the queries they already started have finished, so a transaction is rolled back after them. Before, rows of these queries could be committed after the rollback on SQLite. [#335](https://github.com/ditojs/objection/pull/335) @@ -91,6 +101,7 @@ ### Other +- `insertGraph()` and `upsertGraph()` are rewritten, with a third less code. Graphs write in far fewer queries, e.g. a 500 row insert in 6 queries instead of 500 on SQLite, and an upsert that changes and removes 100 rows in 7 instead of 80 to 127 - The tests run with vitest instead of mocha - TypeScript 7 for development - sqlite3 6 for development and the examples diff --git a/doc/release-notes/migration.md b/doc/release-notes/migration.md index 3cdb71433..b5be5a6f0 100644 --- a/doc/release-notes/migration.md +++ b/doc/release-notes/migration.md @@ -21,6 +21,8 @@ This section lists the breaking changes in objection 4.0. - [Casts, json values and json methods on MySQL and SQLite](#casts-json-values-and-json-methods-on-mysql-and-sqlite) - [Missing and array join properties throw](#missing-and-array-join-properties-throw) - [upsertGraph() relates moved rows before it deletes](#upsertgraph-relates-moved-rows-before-it-deletes) +- [upsertGraph() upserts below related models in the same graph](#upsertgraph-upserts-below-related-models-in-the-same-graph) +- [insertGraph() and upsertGraph() write in fewer queries](#insertgraph-and-upsertgraph-write-in-fewer-queries) - [typescript: The typings are ES module declarations](#typescript-the-typings-are-es-module-declarations) - [Static hooks get the result in the shape that the query returns](#static-hooks-get-the-result-in-the-shape-that-the-query-returns) - [withGraphFetched() assigns what the relation query returns](#withgraphfetched-assigns-what-the-relation-query-returns) @@ -174,10 +176,22 @@ Fetching relations with `withGraphFetched()`, `$fetchGraph()`, `fetchGraph()` or ## upsertGraph() relates moved rows before it deletes -`upsertGraph()` relates the rows that move to existing owners of a `HasManyRelation` or `HasOneRelation` first, before it deletes or unrelates the rows that the graph removes. 3.x deleted first, so a row that moved out of a deleted parent was deleted with it by `ON DELETE CASCADE`, and failed with a foreign key without `ON DELETE` action. The update hooks of these relates now run before the delete hooks of the same call. +`upsertGraph()` relates the rows that move to existing owners of a `HasManyRelation` or `HasOneRelation` first, before it deletes or unrelates the rows that the graph removes. 3.x deleted first, so a row that moved out of a deleted parent was deleted with it by `ON DELETE CASCADE`, and failed with a foreign key without `ON DELETE` action. The update hooks of these relates now run before the delete hooks of the same call. Rows that wait for the keys or references of models that the graph inserts are moved without them, and patched again once these are known, which runs their update hooks a second time, as the nested `upsertGraph()` calls of 3.x did. Their first write keeps the old values of the props that wait, e.g. a name from a `#ref{}` to an inserted model, so a unique key at the new owner that only the new value satisfies fails with a `UniqueViolationError`. A row related from another owner can no longer take the unique key of a row that the graph removes in its new place, e.g. with `unique(ownerId, name)`: the relate fails with a `UniqueViolationError`. Delete the conflicting row with a separate query in the same transaction first. Replacing such a row by a new one, or renaming a kept row to its key, still works, as do relates into a `HasOneRelation` with a unique foreign key. +## upsertGraph() upserts below related models in the same graph + +3.x upserted the models below a related model with a nested `upsertGraph()` call for each owner and relation, after all other writes of the graph. 4.0 fetches the rows of the related models with their relations together with the rest of the graph, and writes them in the same steps, so their deletes, inserts and patches run with the others of their kind instead of last. The options keep matching the full relation paths. + +A related model with relations whose row doesn't exist throws a `NotFoundError` saying that the model doesn't exist, before anything is written. 3.x wrote the rest of the graph first and then failed in the nested call, or with a foreign key error. + +## insertGraph() and upsertGraph() write in fewer queries + +Rows that the graph removes are deleted or unrelated with one query per relation instead of one per owner, so the static `beforeDelete` and `afterDelete` hooks (or `beforeUpdate` and `afterUpdate` for has-many unrelates) run once with all owners as `items`. Join rows are still removed per owner. + +The changed rows of a model class without `beforeUpdate`, `afterUpdate`, `$beforeUpdate` and `$afterUpdate` hooks are patched with one `UPDATE` per table that picks the values of each row with `CASE`, unless the upsert uses `returning('*')`, the values include json or raw values, or the rows are patched through a many-to-many or belongs-to-one relation. Custom query builders see that one patch query instead of one per row. Models with update hooks are patched one by one, as before. + ## typescript: The typings are ES module declarations The typings no longer use `export =`. They declare the named exports of the ES module they describe. Apart from the default import, ES module and bundler projects need no changes. diff --git a/lib/model/graph/ModelGraph.js b/lib/model/graph/ModelGraph.js deleted file mode 100644 index cf015b835..000000000 --- a/lib/model/graph/ModelGraph.js +++ /dev/null @@ -1,74 +0,0 @@ -import { ModelGraphBuilder } from './ModelGraphBuilder.js'; -const NOT_CALCULATED = {}; - -export class ModelGraph { - constructor(nodes, edges) { - this.nodes = nodes; - this.edges = edges; - - // These are calculated lazily. - this._nodesByObjects = NOT_CALCULATED; - this._nodesByIdPathKeys = NOT_CALCULATED; - } - - static create(rootModelClass, roots) { - const builder = ModelGraphBuilder.buildGraph(rootModelClass, roots); - return new ModelGraph(builder.nodes, builder.edges); - } - - static createEmpty() { - return new ModelGraph([], []); - } - - get rootObjects() { - return this.nodes.filter((node) => !node.parentEdge).map((node) => node.obj); - } - - nodeForObject(obj) { - if (!obj) { - return null; - } - - if (this._nodesByObjects === NOT_CALCULATED) { - this._nodesByObjects = createNodesByObjectsMap(this.nodes); - } - - return this._nodesByObjects.get(obj) || null; - } - - nodeForNode(node) { - if (!node) { - return null; - } - - if (this._nodesByIdPathKeys === NOT_CALCULATED) { - this._nodesByIdPathKeys = createNodesByIdPathKeysMap(this.nodes); - } - - return this._nodesByIdPathKeys.get(node.idPathKey) || null; - } -} - -function createNodesByObjectsMap(nodes) { - const nodesByObjects = new Map(); - - for (const node of nodes) { - nodesByObjects.set(node.obj, node); - } - - return nodesByObjects; -} - -function createNodesByIdPathKeysMap(nodes) { - const nodesByIdPathKeys = new Map(); - - for (const node of nodes) { - const idPathKey = node.idPathKey; - - if (idPathKey !== null) { - nodesByIdPathKeys.set(idPathKey, node); - } - } - - return nodesByIdPathKeys; -} diff --git a/lib/model/graph/ModelGraphBuilder.js b/lib/model/graph/ModelGraphBuilder.js deleted file mode 100644 index f3c9a8552..000000000 --- a/lib/model/graph/ModelGraphBuilder.js +++ /dev/null @@ -1,233 +0,0 @@ -import { isObject, isString, asArray, asSingle } from '../../utils/objectUtils.js'; -import { ValidationErrorType } from '../../model/ValidationError.js'; -import { ModelGraphNode } from './ModelGraphNode.js'; -import { ModelGraphEdge } from './ModelGraphEdge.js'; - -export class ModelGraphBuilder { - constructor() { - this.nodes = []; - this.edges = []; - } - - static buildGraph(rootModelClass, roots) { - const builder = new this(); - builder._buildGraph(rootModelClass, roots); - - return builder; - } - - _buildGraph(rootModelClass, roots) { - if (roots) { - if (Array.isArray(roots)) { - this._buildNodes(rootModelClass, roots); - } else if (isObject(roots)) { - this._buildNode(rootModelClass, roots); - } else { - throw createNotModelError(rootModelClass, roots); - } - } - - this._buildReferences(); - } - - _buildNodes(modelClass, objs, parentNode = null, relation = null) { - objs = asArray(objs); - - objs.forEach((obj, index) => { - this._buildNode(modelClass, obj, parentNode, relation, index); - }); - } - - _buildNode(modelClass, obj, parentNode = null, relation = null, index = null) { - obj = asSingle(obj); - - if (!isObject(obj) || !obj.$isObjectionModel) { - throw createNotModelError(modelClass, obj); - } - - const node = new ModelGraphNode(modelClass, obj); - this.nodes.push(node); - - if (parentNode) { - const edge = new ModelGraphEdge( - ModelGraphEdge.Type.Relation, - parentNode, - node, - relation, - index, - ); - - node.parentEdge = edge; - this._addEdge(parentNode, node, edge); - } - - this._buildRelationNodes(node); - } - - _buildRelationNodes(node) { - for (const relationName of node.modelClass.getRelationNames()) { - const relatedObjects = node.obj[relationName]; - - if (!relatedObjects) { - continue; - } - - const relation = node.modelClass.getRelation(relationName); - - if (relation.isOneToOne()) { - this._buildNode(relation.relatedModelClass, relatedObjects, node, relation); - } else { - this._buildNodes(relation.relatedModelClass, relatedObjects, node, relation); - } - } - } - - _buildReferences() { - const nodesByUid = this._nodesByUid(); - - this._buildObjectReferences(nodesByUid); - this._buildPropertyReferences(nodesByUid); - } - - _nodesByUid() { - const nodesByUid = new Map(); - - for (const node of this.nodes) { - const uid = node.uid; - - if (uid === undefined) { - continue; - } - - nodesByUid.set(uid, node); - } - - return nodesByUid; - } - - _buildObjectReferences(nodesByUid) { - for (const node of this.nodes) { - const ref = node.reference; - - if (ref === undefined) { - continue; - } - - const refNode = nodesByUid.get(ref); - - if (!refNode) { - throw createReferenceNotFoundError(ref); - } - - const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode); - edge.refType = ModelGraphEdge.ReferenceType.Object; - - this._addEdge(node, refNode, edge); - } - } - - _buildPropertyReferences(nodesByUid) { - for (const node of this.nodes) { - forEachPropertyReference(node.obj, ({ path, refMatch, ref, refPath }) => { - const refNode = nodesByUid.get(ref); - - if (!refNode) { - throw createReferenceNotFoundError(ref); - } - - const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode); - - edge.refType = ModelGraphEdge.ReferenceType.Property; - edge.refMatch = refMatch; - edge.refOwnerDataPath = path.slice(); - edge.refRelatedDataPath = refPath; - - this._addEdge(node, refNode, edge); - }); - } - } - - _addEdge(ownerNode, relatedNode, edge) { - this.edges.push(edge); - - ownerNode.edges.push(edge); - relatedNode.edges.push(edge); - - if (edge.type === ModelGraphEdge.Type.Reference) { - ownerNode.refEdges.push(edge); - relatedNode.refEdges.push(edge); - } - } -} - -export function forEachPropertyReference(obj, callback) { - const modelClass = obj.constructor; - const relationNames = modelClass.getRelationNames(); - - for (const prop of Object.keys(obj)) { - if (relationNames.includes(prop)) { - continue; - } - - visitStrings(obj[prop], [prop], (str, path) => { - forEachMatch(modelClass.propRefRegex, str, (match) => { - const [refMatch, ref, refPath] = match; - callback({ path, refMatch, ref, refPath: refPath.trim().split('.') }); - }); - }); - } -} - -function visitStrings(value, path, visit) { - if (Array.isArray(value)) { - visitStringsInArray(value, path, visit); - } else if (isObject(value) && !Buffer.isBuffer(value)) { - visitStringsInObject(value, path, visit); - } else if (isString(value)) { - visit(value, path); - } -} - -function visitStringsInArray(value, path, visit) { - for (let i = 0; i < value.length; ++i) { - path.push(i); - visitStrings(value[i], path, visit); - path.pop(); - } -} - -function visitStringsInObject(value, path, visit) { - for (const prop of Object.keys(value)) { - path.push(prop); - visitStrings(value[prop], path, visit); - path.pop(); - } -} - -function forEachMatch(regex, str, cb) { - let matchResult = regex.exec(str); - - while (matchResult) { - cb(matchResult); - matchResult = regex.exec(str); - } -} - -function createReferenceNotFoundError(ref) { - return new Error( - [ - `could not resolve reference ${ref} in a graph.`, - `If you are sure the #id exist in the same graph,`, - `this may be due to a limitation that a subgraph under a related item`, - `cannot reference an item in a subgraph under another related item.`, - `If you run into this limitation, please open an issue in objection github.`, - ].join(' '), - ); -} - -export function createNotModelError(modelClass, value) { - throw modelClass.createValidationError({ - type: ValidationErrorType.InvalidGraph, - message: `expected value "${value}" to be an instance of ${modelClass.name}`, - }); -} diff --git a/lib/model/graph/ModelGraphEdge.js b/lib/model/graph/ModelGraphEdge.js deleted file mode 100644 index 46a3bb7c8..000000000 --- a/lib/model/graph/ModelGraphEdge.js +++ /dev/null @@ -1,46 +0,0 @@ -const Type = { - Relation: 'Relation', - Reference: 'Reference', -}; - -const ReferenceType = { - Object: 'Object', - Property: 'Property', -}; - -export class ModelGraphEdge { - constructor(type, ownerNode, relatedNode, relation = null, relationIndex = null) { - this.type = type; - - this.ownerNode = ownerNode; - this.relatedNode = relatedNode; - this.relation = relation; - this.relationIndex = relationIndex; - - this.refType = null; - this.refMatch = null; - - this.refOwnerDataPath = null; - this.refRelatedDataPath = null; - } - - static get Type() { - return Type; - } - - static get ReferenceType() { - return ReferenceType; - } - - getOtherNode(node) { - return this.isOwnerNode(node) ? this.relatedNode : this.ownerNode; - } - - isOwnerNode(node) { - return node === this.ownerNode; - } - - isRelatedNode(node) { - return node === this.relatedNode; - } -} diff --git a/lib/model/graph/ModelGraphNode.js b/lib/model/graph/ModelGraphNode.js deleted file mode 100644 index 892191f45..000000000 --- a/lib/model/graph/ModelGraphNode.js +++ /dev/null @@ -1,231 +0,0 @@ -import { ModelGraphEdge } from './ModelGraphEdge.js'; -import { isNumber } from '../../utils/objectUtils.js'; - -const NOT_CALCULATED = {}; - -export class ModelGraphNode { - constructor(modelClass, obj) { - this.modelClass = modelClass; - this.obj = obj; - this.edges = []; - this.hasId = obj.$hasId(); - this.uid = obj[modelClass.uidProp]; - - // These are also included in `edges`. These are simply - // shortcuts for commonly used edges. - this.refEdges = []; - this.parentEdge = null; - - // These are calculated lazily. - this._relationPath = NOT_CALCULATED; - this._relationPathKey = NOT_CALCULATED; - - this._dataPath = NOT_CALCULATED; - this._dataPathKey = NOT_CALCULATED; - - this._idPath = NOT_CALCULATED; - this._idPathKey = NOT_CALCULATED; - } - - get isReference() { - return this.reference !== undefined; - } - - get isDbReference() { - return this.dbReference !== undefined; - } - - get reference() { - return this.obj[this.modelClass.uidRefProp]; - } - - get dbReference() { - return this.obj[this.modelClass.dbRefProp]; - } - - get toBeUnrelated() { - return this.obj[this.modelClass.graphUnrelateProp] === true; - } - - get toBeDeleted() { - return this.obj[this.modelClass.graphDeleteProp] === true; - } - - get toBeUnrelatedOrDeleted() { - return this.toBeUnrelated || this.toBeDeleted; - } - - get parentNode() { - if (this.parentEdge) { - return this.parentEdge.ownerNode; - } else { - return null; - } - } - - get indexInRelation() { - if (this.parentEdge) { - return this.parentEdge.relationIndex; - } else { - return null; - } - } - - get relationName() { - if (this.parentEdge) { - return this.parentEdge.relation.name; - } else { - return null; - } - } - - get relationPath() { - if (this._relationPath === NOT_CALCULATED) { - this._relationPath = this._createRelationPath(); - } - - return this._relationPath; - } - - get relationPathKey() { - if (this._relationPathKey === NOT_CALCULATED) { - this._relationPathKey = this._createRelationPathKey(); - } - - return this._relationPathKey; - } - - get dataPath() { - if (this._dataPath === NOT_CALCULATED) { - this._dataPath = this._createDataPath(); - } - - return this._dataPath; - } - - get dataPathKey() { - if (this._dataPathKey === NOT_CALCULATED) { - this._dataPathKey = this._createDataPathKey(); - } - - return this._dataPathKey; - } - - get idPath() { - if (this._idPath === NOT_CALCULATED) { - this._idPath = this._createIdPath(); - } - - return this._idPath; - } - - get idPathKey() { - if (this._idPathKey === NOT_CALCULATED) { - this._idPathKey = this._createIdPathKey(); - } - - return this._idPathKey; - } - - /** - * If this node is a reference, returns the referred node. - */ - get referencedNode() { - for (const edge of this.refEdges) { - if (edge.refType === ModelGraphEdge.ReferenceType.Object && edge.isOwnerNode(this)) { - return edge.relatedNode; - } - } - - return null; - } - - get descendantRelationNodes() { - return this._collectDescendantRelationNodes([]); - } - - removeEdge(edge) { - // Don't allow removing parent edges for now. It would - // cause all kinds of cache invalidation. - if (edge === this.parentEdge) { - throw new Error('cannot remove parent edge'); - } - - this.edges = this.edges.filter((it) => it !== edge); - this.refEdges = this.refEdges.filter((it) => it !== edge); - } - - _collectDescendantRelationNodes(nodes) { - for (const edge of this.edges) { - if (edge.type === ModelGraphEdge.Type.Relation && edge.isOwnerNode(this)) { - nodes.push(edge.relatedNode); - edge.relatedNode._collectDescendantRelationNodes(nodes); - } - } - - return nodes; - } - - _createRelationPath() { - if (this.parentNode === null) { - return []; - } else { - return [...this.parentNode.relationPath, this.relationName]; - } - } - - _createRelationPathKey() { - return this.relationPath.join('.'); - } - - _createDataPath() { - if (this.parentEdge === null) { - return []; - } else if (this.parentEdge.relation.isOneToOne()) { - return [...this.parentNode.dataPath, this.relationName]; - } else { - return [...this.parentNode.dataPath, this.relationName, this.indexInRelation]; - } - } - - _createDataPathKey() { - const dataPathKey = this.dataPath.reduce((key, it) => { - if (isNumber(it)) { - return `${key}[${it}]`; - } else { - return key ? `${key}.${it}` : it; - } - }, ''); - - return dataPathKey ? '.' + dataPathKey : dataPathKey; - } - - _createIdPath() { - if (!this.obj.$hasId()) { - return null; - } - - if (this.parentEdge === null) { - return [this.obj.$idKey()]; - } else { - const path = this.parentNode.idPath; - - if (path === null) { - return null; - } - - return [...path, this.relationName, this.obj.$idKey()]; - } - } - - _createIdPathKey() { - const idPath = this.idPath; - - if (idPath) { - // JSON keeps the ids and relation names apart, whatever they contain. - return JSON.stringify(idPath); - } else { - return null; - } - } -} diff --git a/lib/queryBuilder/graph/GraphAction.js b/lib/queryBuilder/graph/GraphAction.js deleted file mode 100644 index 621d2da70..000000000 --- a/lib/queryBuilder/graph/GraphAction.js +++ /dev/null @@ -1,124 +0,0 @@ -import { isPostgres } from '../../utils/knexUtils.js'; -import { jsonEquals } from '../../utils/objectUtils.js'; - -const POSTGRES_MAX_INSERT_BATCH_SIZE = 100; -const MAX_CONCURRENCY = 100; - -export class GraphAction { - constructor(graphData) { - this.graphData = graphData; - } - - static get ReturningAllSelector() { - return (op) => { - // Only select `returning('*')` operation. - return op.name === 'returning' && op.args.includes('*'); - }; - } - - static getConcurrency(builder, nodes) { - return nodes.reduce((minConcurrency, node) => { - return Math.min(minConcurrency, node.modelClass.getConcurrency(builder.unsafeKnex())); - }, MAX_CONCURRENCY); - } - - get graph() { - return this.graphData.graph; - } - - get currentGraph() { - return this.graphData.currentGraph; - } - - get graphOptions() { - return this.graphData.graphOptions; - } - - run(builder) { - return null; - } - - _getConcurrency(builder, nodes) { - return GraphAction.getConcurrency(builder, nodes); - } - - _getBatchSize(builder) { - return isPostgres(builder.unsafeKnex()) ? POSTGRES_MAX_INSERT_BATCH_SIZE : 1; - } - - // Change the arrays of keys of the BelongsToManyRelations that the graph - // holds for `node`: Remove the keys of the current related models that it - // deletes or unrelates, and add the keys of the related models that it - // inserts, relates or patches. Other keys stay, e.g. of rows that the - // relation's filter leaves out. Return the props that change. - _setBelongsToManyKeys(node) { - const { actions } = this.graphData; - const currentNode = this.currentGraph.nodeForNode(node); - const changedProps = []; - - const getRelatedNodes = (ownerNode, relation) => - ownerNode - ? ownerNode.edges - .filter((edge) => edge.relation === relation && edge.isOwnerNode(ownerNode)) - .map((edge) => edge.relatedNode) - : []; - - // Relations that the graph leaves out keep their arrays, and aren't loaded - // (#1202): - const relations = node.modelClass - .getRelationNames() - .filter((name) => node.obj[name] !== undefined) - .map((name) => node.modelClass.getRelation(name)) - .filter((relation) => relation.isObjectionBelongsToManyRelation); - - for (const relation of relations) { - const { ownerProp, relatedProp } = relation; - const getKey = (related) => relatedProp.getProp(related.obj, 0); - const keys = relation.getKeys(currentNode ? currentNode.obj : node.obj); - - const removed = getRelatedNodes(currentNode, relation).filter((related) => - ['delete', 'unrelate'].includes(actions.get(related)), - ); - const added = getRelatedNodes(node, relation).filter( - (related) => !related.toBeUnrelatedOrDeleted && actions.get(related) !== 'ignore', - ); - - for (const related of added) { - this._resolveReferences(related); - } - - const newKeys = relation.changeKeys(keys, { - add: added.map(getKey), - remove: removed.map(getKey), - }); - - if (!currentNode || this.graphOptions.shouldUpdate(node)) { - // Inserts and updates write the whole row: - ownerProp.setProp(node.obj, 0, newKeys); - changedProps.push(ownerProp.props[0]); - } else if (!jsonEquals(keys, newKeys)) { - const patch = relation.createKeysPatch(currentNode.obj, newKeys); - Object.assign(node.obj, patch); - changedProps.push(...Object.keys(patch)); - } - } - - return changedProps; - } - - _resolveReferences(node) { - if (node.isReference) { - this._resolveReference(node); - } - } - - _resolveReference(node) { - const refNode = node.referencedNode; - - for (const prop of Object.keys(refNode.obj)) { - if (!node.obj.hasOwnProperty(prop)) { - node.obj[prop] = refNode.obj[prop]; - } - } - } -} diff --git a/lib/queryBuilder/graph/GraphData.js b/lib/queryBuilder/graph/GraphData.js deleted file mode 100644 index 0d2340fbd..000000000 --- a/lib/queryBuilder/graph/GraphData.js +++ /dev/null @@ -1,82 +0,0 @@ -export class GraphData { - constructor({ graph, currentGraph, graphOptions, nodeDbExistence }) { - this.graph = graph; - this.currentGraph = currentGraph; - this.graphOptions = graphOptions; - this.nodeDbExistence = nodeDbExistence; - // Pruning replaces `graph.nodes`, but `keptNodeForRow()` also needs the - // branches below related nodes. - this._unprunedNodes = graph.nodes; - this._keptNodesByRowKey = null; - this.actions = null; - } - - // Decide what to do with each node once, before the first write, since the - // writes change the models that the rules of `GraphOptions` read. Only call - // this after pruning. - planActions() { - const { graph, currentGraph, graphOptions } = this; - this.actions = new Map(); - - for (const node of graph.nodes) { - this.actions.set( - node, - graphOptions.shouldInsert(node, this) - ? 'insert' - : graphOptions.shouldRelate(node, this) - ? 'relate' - : graphOptions.shouldPatchOrUpdateIgnoreDisable(node, this) - ? 'patch' - : 'ignore', - ); - } - - for (const currentNode of currentGraph.nodes) { - this.actions.set( - currentNode, - graphOptions.shouldDelete(currentNode, this) - ? 'delete' - : graphOptions.shouldUnrelate(currentNode, this) - ? 'unrelate' - : 'keep', - ); - } - - return this; - } - - // Find the node that keeps the row of `currentNode` in another place of the - // graph, by relating or patching it there, or by upserting it below a related - // node. Only call this after pruning. - keptNodeForRow(currentNode) { - if (!this._keptNodesByRowKey) { - this._keptNodesByRowKey = new Map(); - const nodes = new Set(this.graph.nodes); - - // Nested upserts write the pruned branches below related nodes, except - // for models marked with `#unrelate` or `#delete` and their descendants: - const isUpsertedBelowRelatedNode = (node) => - !node.toBeUnrelatedOrDeleted && - (nodes.has(node.parentNode) - ? this.graphOptions.shouldRelate(node.parentNode, this) - : isUpsertedBelowRelatedNode(node.parentNode)); - - for (const node of this._unprunedNodes) { - if ( - node.obj.$hasId() && - (nodes.has(node) - ? this.graphOptions.shouldPatchOrUpdateIgnoreDisable(node, this) - : isUpsertedBelowRelatedNode(node)) - ) { - this._keptNodesByRowKey.set(getRowKey(node), node); - } - } - } - - return this._keptNodesByRowKey.get(getRowKey(currentNode)) || null; - } -} - -function getRowKey(node) { - return `${node.modelClass.getTableName()}:${node.obj.$idKey()}`; -} diff --git a/lib/queryBuilder/graph/GraphFetcher.js b/lib/queryBuilder/graph/GraphFetcher.js deleted file mode 100644 index 2a9820f14..000000000 --- a/lib/queryBuilder/graph/GraphFetcher.js +++ /dev/null @@ -1,191 +0,0 @@ -import { asArray, groupBy } from '../../utils/objectUtils.js'; -import { ModelGraph } from '../../model/graph/ModelGraph.js'; -import { FetchStrategy } from './GraphOptions.js'; -import { RelationExpression } from '../RelationExpression.js'; -import { getDeclaredColumns } from '../operations/fieldExpressions.js'; -import { getFieldExpressionColumn } from '../../utils/parseFieldExpression.js'; - -export class GraphFetcher { - /** - * Given a graph and options, fetches the current state of that graph - * from the database and returns it as a ModelGraph instance. - */ - static async fetchCurrentGraph({ builder, graph, graphOptions }) { - const { rootObjects } = graph; - - const rootIds = getRootIds(rootObjects); - const modelClass = builder.modelClass(); - - if (rootIds.length === 0) { - return Promise.resolve(ModelGraph.create(modelClass, [])); - } - - const eagerExpr = RelationExpression.fromModelGraph(rootObjects); - const models = await modelClass - .query() - .childQueryOf(builder, childQueryOptions()) - .modify(propagateMethodCallsFromQuery(builder)) - .modify(buildFetchQuerySelects(graph, graphOptions, eagerExpr)) - .findByIds(rootIds) - .withGraphFetched(eagerExpr) - .internalOptions(fetchQueryInternalOptions()); - - return ModelGraph.create(modelClass, models); - } -} - -function getRootIds(rootObjects) { - return asArray(rootObjects) - .filter((it) => it.$hasId()) - .map((root) => root.$id()); -} - -function propagateMethodCallsFromQuery(builder) { - return (fetchBuilder) => { - // Propagate some method calls from the root query. - for (const method of ['forUpdate', 'forShare', 'forNoKeyUpdate', 'forKeyShare']) { - if (builder.has(method)) { - fetchBuilder[method](); - } - } - }; -} - -function buildFetchQuerySelects(graph, graphOptions, eagerExpr) { - return (builder) => { - const nodesByRelationPath = groupNodesByRelationPath(graph, eagerExpr); - - for (const [relationPath, nodes] of nodesByRelationPath.entries()) { - const selectModifier = createFetchSelectModifier(nodes, graphOptions); - - if (!relationPath) { - builder.modify(selectModifier); - } else { - builder.modifyGraph(relationPath, selectModifier); - } - } - }; -} - -function groupNodesByRelationPath(graph, eagerExpr) { - const nodesByRelationPath = groupBy(graph.nodes, (node) => node.relationPathKey); - - // Not all relation paths have nodes. Relations with nulls or empty arrays - // don't have nodes, but will still need to be fetched. Add these to the - // map as empty arrays. - forEachPath(eagerExpr.node, (relationPath) => { - if (!nodesByRelationPath.has(relationPath)) { - nodesByRelationPath.set(relationPath, []); - } - }); - - return nodesByRelationPath; -} - -function createFetchSelectModifier(nodes, graphOptions) { - if (graphOptions.isFetchStrategy(FetchStrategy.OnlyIdentifiers)) { - return createIdentifierSelector(); - } else if (graphOptions.isFetchStrategy(FetchStrategy.OnlyNeeded)) { - return createInputColumnSelector(nodes); - } else { - return () => {}; - } -} - -// Returns a function that only selects the id column. -function createIdentifierSelector() { - return (builder) => { - builder.select(builder.fullIdColumn()); - }; -} - -// Returns a function that only selects the columns that exist in the input. -function createInputColumnSelector(nodes) { - return (builder) => { - const selects = new Map(); - - for (const node of nodes) { - const databaseJson = node.obj.$toDatabaseJson(builder); - const declaredColumns = getDeclaredColumns(node.modelClass); - - for (const key of Object.keys(databaseJson)) { - // Select the json column of field expressions like `'meta:a'`: - const column = getFieldExpressionColumn(key, declaredColumns) ?? key; - - if (!shouldSelectColumn(column, selects, node)) { - continue; - } - - const selection = - createManyToManyExtraSelectionIfNeeded(builder, column, node) || - createSelection(builder, column, node); - - selects.set(column, selection); - } - } - - const selectArr = Array.from(selects.values()); - const idColumns = asArray(builder.fullIdColumn()); - - for (const idColumn of idColumns) { - if (!selectArr.includes(idColumn)) { - // Always select the identifers. - selectArr.push(idColumn); - } - } - - builder.select(selectArr); - }; -} - -function shouldSelectColumn(column, selects, node) { - const modelClass = node.modelClass; - - return ( - !selects.has(column) && - column !== modelClass.propertyNameToColumnName(modelClass.dbRefProp) && - column !== modelClass.propertyNameToColumnName(modelClass.uidRefProp) && - column !== modelClass.propertyNameToColumnName(modelClass.uidProp) && - column !== modelClass.propertyNameToColumnName(modelClass.graphUnrelateProp) && - column !== modelClass.propertyNameToColumnName(modelClass.graphDeleteProp) - ); -} - -function createManyToManyExtraSelectionIfNeeded(builder, column, node) { - if (node.parentEdge && node.parentEdge.relation.isObjectionManyToManyRelation) { - const relation = node.parentEdge.relation; - const extra = relation.joinTableExtras.find((extra) => extra.aliasCol === column); - - if (extra) { - return `${relation.joinTableAliasFor(builder)}.${extra.joinTableCol} as ${extra.aliasCol}`; - } - } - - return null; -} - -function createSelection(builder, column, node) { - return `${builder.tableRefFor(node.modelClass)}.${column}`; -} - -function childQueryOptions() { - return { - fork: true, - isInternalQuery: true, - }; -} - -function fetchQueryInternalOptions() { - return { - keepImplicitJoinProps: true, - }; -} - -function forEachPath(eagerExprNode, cb, path = []) { - for (const relation of eagerExprNode.$childNames) { - path.push(relation); - cb(path.join('.')); - forEachPath(eagerExprNode[relation], cb, path); - path.pop(); - } -} diff --git a/lib/queryBuilder/graph/GraphLoader.js b/lib/queryBuilder/graph/GraphLoader.js new file mode 100644 index 000000000..cbbdcbd85 --- /dev/null +++ b/lib/queryBuilder/graph/GraphLoader.js @@ -0,0 +1,249 @@ +import { asArray, chunk, groupBy, uniqBy } from '../../utils/objectUtils.js'; +import { getMaxRows } from '../../utils/knexUtils.js'; +import { FetchStrategy } from './GraphOptions.js'; +import { GraphNode } from './GraphNode.js'; +import { RelationExpression } from '../RelationExpression.js'; +import { getDeclaredColumns } from '../operations/fieldExpressions.js'; +import { getFieldExpressionColumn } from '../../utils/parseFieldExpression.js'; + +// Fetch the current state of the graph from the database and pair its rows with +// the models of the graph: First the roots with their relations, then the rows +// of the models that the graph relates from elsewhere, with the relations that +// the graph gives them, level by level, as the rows of related models decide +// which models below them are related too. Models that `insertMissing` inserts +// if they don't exist are looked up too. +export async function loadCurrentGraph(builder, plan) { + const fetchedNodes = new Set(); + + const fetch = async (nodes, { areRelated }) => { + for (const node of nodes) { + fetchedNodes.add(node); + } + + for (const nodesOfClass of groupBy(nodes, (node) => node.modelClass).values()) { + const models = await fetchModels(builder, plan.graphOptions, nodesOfClass); + const modelsByKey = new Map(models.map((model) => [model.$idKey(), model])); + + for (const node of nodesOfClass) { + const model = modelsByKey.get(node.obj.$idKey()); + + if (model) { + const currentNodes = GraphNode.buildGraph(node.modelClass, model, node.relationPath); + const [currentNode] = currentNodes; + plan.currentNodes.push(...currentNodes); + + if (areRelated) { + node.row = currentNode; + currentNode.node = node; + pairChildren(node, currentNode); + } else { + pair(node, currentNode); + } + } else if (areRelated) { + node.isMissing = true; + } + } + } + }; + + // Find the models that the graph relates and gives relations, or inserts + // with `insertMissing` if they don't exist, and that aren't fetched yet. The + // ones below others wait for their rows, and the ones in branches that the + // graph doesn't write are left out. + const findRelatedNodes = () => { + const rowKeys = new Set(plan.currentNodes.map((currentNode) => currentNode.rowKey)); + const nodes = plan.nodes.filter( + (node) => + !fetchedNodes.has(node) && + !node.current && + node.obj.$hasId() && + plan.actionFor(node) === 'relate' && + (hasRelations(node.obj) || + (plan.graphOptions.has(node, 'insertMissing') && !rowKeys.has(node.rowKey))), + ); + const nodeSet = new Set(nodes); + + return nodes.filter((node) => { + for (let parent = node.parent; parent; parent = parent.parent) { + if (nodeSet.has(parent) || !plan.writesBranchOf(parent)) { + return false; + } + } + + return true; + }); + }; + + await fetch( + plan.nodes.filter((node) => node.isRoot && node.hasId), + { areRelated: false }, + ); + + for (let nodes = findRelatedNodes(); nodes.length > 0; nodes = findRelatedNodes()) { + await fetch(nodes, { areRelated: true }); + } +} + +function pair(node, currentNode) { + node.current ??= currentNode; + currentNode.node = node; + pairChildren(node, currentNode); +} + +// Pair the related models with ids with the current rows in the same relations. +// Rows can be related more than once through duplicate join rows, and all of +// them pair with the same model. +function pairChildren(node, currentNode) { + const childrenByRelation = groupBy(currentNode.children, (child) => child.relation.name); + + for (const child of node.children) { + if (child.obj.$hasId()) { + const key = child.obj.$idKey(); + + for (const currentChild of childrenByRelation.get(child.relation.name) || []) { + if (currentChild.obj.$idKey() === key) { + pair(child, currentChild); + } + } + } + } +} + +function hasRelations(obj) { + return obj.constructor.getRelationNames().some((name) => obj[name] !== undefined); +} + +// Fetch the rows of `nodes` with their relations, in as many queries as the +// number of their ids requires. +async function fetchModels(builder, graphOptions, nodes) { + const [{ modelClass }] = nodes; + const uniqueNodes = uniqBy(nodes, (node) => node.obj.$idKey()); + const maxIds = getMaxRows(builder.unsafeKnex(), modelClass.getIdColumnArray().length); + const models = []; + + for (const chunkNodes of chunk(uniqueNodes, maxIds)) { + models.push(...(await fetchModelsOfNodes(builder, graphOptions, chunkNodes))); + } + + return models; +} + +function fetchModelsOfNodes(builder, graphOptions, nodes) { + const [{ modelClass }] = nodes; + const ids = nodes.map((node) => node.obj.$id()); + const expression = RelationExpression.fromModelGraph(nodes.map((node) => node.obj)); + + return modelClass + .query() + .childQueryOf(builder, { fork: true, isInternalQuery: true }) + .modify((fetchBuilder) => { + // Lock the fetched rows like the upsert query asks for. + for (const method of ['forUpdate', 'forShare', 'forNoKeyUpdate', 'forKeyShare']) { + if (builder.has(method)) { + fetchBuilder[method](); + } + } + }) + .modify((fetchBuilder) => { + for (const [relationPath, pathNodes] of getNodesByRelationPath(nodes, expression)) { + const select = createSelect(pathNodes, graphOptions, !relationPath); + + if (!relationPath) { + fetchBuilder.modify(select); + } else { + fetchBuilder.modifyGraph(relationPath, select); + } + } + }) + .findByIds(ids) + .withGraphFetched(expression) + .internalOptions({ keepImplicitJoinProps: true }); +} + +// Group the fetched nodes and their descendants by their relation paths below +// the fetched nodes. Relations with nulls or empty arrays have no nodes, but +// are fetched too. +function getNodesByRelationPath(nodes, expression) { + const nodesByRelationPath = new Map(); + + for (const node of nodes) { + const depth = node.relationPath.length; + + for (const descendant of [node, ...node.descendants]) { + const relationPath = descendant.relationPath.slice(depth).join('.'); + const pathNodes = nodesByRelationPath.get(relationPath); + + if (pathNodes) { + pathNodes.push(descendant); + } else { + nodesByRelationPath.set(relationPath, [descendant]); + } + } + } + + const addPaths = (expressionNode, path) => { + for (const name of expressionNode.$childNames) { + const relationPath = [...path, name]; + const key = relationPath.join('.'); + + if (!nodesByRelationPath.has(key)) { + nodesByRelationPath.set(key, []); + } + + addPaths(expressionNode[name], relationPath); + } + }; + + addPaths(expression.node, []); + return nodesByRelationPath; +} + +function createSelect(nodes, graphOptions, isFetchRoot) { + if (graphOptions.isFetchStrategy(FetchStrategy.OnlyIdentifiers)) { + return (builder) => builder.select(builder.fullIdColumn()); + } else if (graphOptions.isFetchStrategy(FetchStrategy.OnlyNeeded)) { + return (builder) => builder.select(getInputColumnSelections(builder, nodes, isFetchRoot)); + } else { + return () => {}; + } +} + +// Select only the columns that the graph gives, and always the ids. Related +// models are fetched without the join tables of their relations, so their join +// table extras are left out at the root of the fetch. +function getInputColumnSelections(builder, nodes, isFetchRoot) { + const selections = new Map(); + + for (const node of nodes) { + const declaredColumns = getDeclaredColumns(node.modelClass); + + for (const key of Object.keys(node.obj.$toDatabaseJson(builder))) { + // Select the json column of field expressions like `'meta:a'`: + const column = getFieldExpressionColumn(key, declaredColumns) ?? key; + + if (!selections.has(column)) { + const { relation } = node; + const extra = + relation?.keySide === 'join' && + relation.joinTableExtras.find((extra) => extra.aliasCol === column); + + if (!extra) { + selections.set(column, `${builder.tableRefFor(node.modelClass)}.${column}`); + } else if (!isFetchRoot) { + const alias = relation.joinTableAliasFor(builder); + selections.set(column, `${alias}.${extra.joinTableCol} as ${extra.aliasCol}`); + } + } + } + } + + const values = [...selections.values()]; + + for (const idColumn of asArray(builder.fullIdColumn())) { + if (!values.includes(idColumn)) { + values.push(idColumn); + } + } + + return values; +} diff --git a/lib/queryBuilder/graph/GraphNode.js b/lib/queryBuilder/graph/GraphNode.js new file mode 100644 index 000000000..e4196eef8 --- /dev/null +++ b/lib/queryBuilder/graph/GraphNode.js @@ -0,0 +1,232 @@ +import { asArray, asSingle, isNumber, isObject, isString } from '../../utils/objectUtils.js'; +import { ValidationErrorType } from '../../model/ValidationError.js'; + +// A model in a graph that is inserted or upserted, or in the current graph that +// is fetched from the database for it. +export class GraphNode { + constructor(modelClass, obj, { parent = null, relation = null, index = null, relationPath }) { + this.modelClass = modelClass; + this.obj = obj; + this.parent = parent; + this.relation = relation; + // The index in the array of a to-many relation: + this.index = index; + this.children = []; + this.relationPath = parent ? [...parent.relationPath, relation.name] : relationPath; + this.relationPathKey = this.relationPath.join('.'); + // The id state of the input, before the writes complete ids with keys: + this.hasId = obj.$hasId(); + + // Read the markers of the graph, which `removeMarkers()` removes from the + // models once all places of a model are read: + const { uidProp, uidRefProp, dbRefProp, graphUnrelateProp, graphDeleteProp } = modelClass; + this.uid = obj[uidProp]; + this.reference = obj[uidRefProp]; + this.dbReference = obj[dbRefProp]; + this.toBeUnrelated = obj[graphUnrelateProp] === true; + this.toBeDeleted = obj[graphDeleteProp] === true; + + // The node of a `#ref` model, and the `#ref{}` properties of this model: + this.referencedNode = null; + this.propertyReferences = []; + // The nodes whose `#ref` or `#ref{}` point to this node: + this.referencingNodes = []; + this.propertyReferencingNodes = []; + + // How the graph and the current graph pair up, see `GraphLoader`: The + // current node at the place of an input node and the other way around, and + // the current node of a related model's row fetched from elsewhere. + this.current = null; + this.node = null; + this.row = null; + this.isMissing = false; + + // Set by `GraphPlan`: + this.isPruned = false; + } + + get isReference() { + return this.reference !== undefined; + } + + get isDbReference() { + return this.dbReference !== undefined; + } + + get toBeUnrelatedOrDeleted() { + return this.toBeUnrelated || this.toBeDeleted; + } + + get isRoot() { + return !this.parent; + } + + // The current node of the row of this node, at its place or where a relate + // moves it from. + get existing() { + return this.current || this.row; + } + + get dataPath() { + return this.parent + ? [ + ...this.parent.dataPath, + this.relation.name, + ...(this.relation.isOneToOne() ? [] : [this.index]), + ] + : []; + } + + // The data path in the form of validation errors, like `.pets[0].owner`. + get dataPathKey() { + return this.dataPath.map((key) => (isNumber(key) ? `[${key}]` : `.${key}`)).join(''); + } + + get descendants() { + return this.children.flatMap((child) => [child, ...child.descendants]); + } + + get activeChildren() { + return this.children.filter((child) => !child.isPruned); + } + + get rowKey() { + return `${this.modelClass.getTableName()}:${this.obj.$idKey()}`; + } + + // Build the nodes of `roots` and their related models in depth-first order. + // `relationPath` is the place of the roots in the graph that a current graph + // is fetched for. + static buildGraph(modelClass, roots, relationPath = []) { + const nodes = []; + + const addNode = (modelClass, obj, place) => { + if (!isObject(obj) || !obj.$isObjectionModel) { + throw createNotModelError(modelClass, obj); + } + + const node = new GraphNode(modelClass, obj, place); + nodes.push(node); + place.parent?.children.push(node); + + for (const name of modelClass.getRelationNames()) { + const related = obj[name]; + + if (related) { + const relation = modelClass.getRelation(name); + const items = relation.isOneToOne() ? [asSingle(related)] : asArray(related); + + items.forEach((item, index) => { + addNode(relation.relatedModelClass, item, { parent: node, relation, index }); + }); + } + } + }; + + for (const root of asArray(roots)) { + addNode(modelClass, root, { relationPath }); + } + + return nodes; + } + + // Remove the markers from the models, as they're neither written nor + // returned, and return a function that puts them back if the writes fail. + static removeMarkers(nodes) { + const markers = []; + + for (const { modelClass, obj } of nodes) { + for (const prop of [ + modelClass.uidProp, + modelClass.uidRefProp, + modelClass.dbRefProp, + modelClass.graphUnrelateProp, + modelClass.graphDeleteProp, + ]) { + if (Object.hasOwn(obj, prop)) { + markers.push([obj, prop, obj[prop]]); + delete obj[prop]; + } + } + } + + return () => { + for (const [obj, prop, value] of markers) { + obj[prop] = value; + } + }; + } + + // Link the `#ref` and `#ref{}` references of the nodes to the nodes with the + // `#id` that they refer to. + static linkReferences(nodes) { + const nodesByUid = new Map(); + + for (const node of nodes) { + if (node.uid !== undefined) { + nodesByUid.set(node.uid, node); + } + } + + const getNode = (uid) => { + const node = nodesByUid.get(uid); + + if (!node) { + throw new Error(`could not resolve reference ${uid} in a graph.`); + } + + return node; + }; + + for (const node of nodes) { + if (node.isReference) { + node.referencedNode = getNode(node.reference); + node.referencedNode.referencingNodes.push(node); + } + + forEachPropertyReference(node.obj, ({ path, match, uid, refPath, isWholeValue }) => { + const referencedNode = getNode(uid); + node.propertyReferences.push({ path, match, refPath, isWholeValue, referencedNode }); + referencedNode.propertyReferencingNodes.push(node); + }); + } + } +} + +function forEachPropertyReference(obj, callback) { + const modelClass = obj.constructor; + const relationNames = modelClass.getRelationNames(); + + const visit = (value, path) => { + if (Array.isArray(value)) { + value.forEach((item, index) => visit(item, [...path, index])); + } else if (isObject(value) && !Buffer.isBuffer(value)) { + for (const key of Object.keys(value)) { + visit(value[key], [...path, key]); + } + } else if (isString(value)) { + for (const [match, uid, refPath] of value.matchAll(modelClass.propRefRegex)) { + callback({ + path, + match, + uid, + refPath: refPath.trim().split('.'), + isWholeValue: match === value, + }); + } + } + }; + + for (const key of Object.keys(obj)) { + if (!relationNames.includes(key)) { + visit(obj[key], [key]); + } + } +} + +export function createNotModelError(modelClass, value) { + return modelClass.createValidationError({ + type: ValidationErrorType.InvalidGraph, + message: `expected value "${value}" to be an instance of ${modelClass.name}`, + }); +} diff --git a/lib/queryBuilder/graph/GraphNodeDbExistence.js b/lib/queryBuilder/graph/GraphNodeDbExistence.js deleted file mode 100644 index c3272a948..000000000 --- a/lib/queryBuilder/graph/GraphNodeDbExistence.js +++ /dev/null @@ -1,178 +0,0 @@ -import { GraphData } from './GraphData.js'; -import { GraphAction } from './GraphAction.js'; -import * as promiseUtils from '../../utils/promiseUtils/index.js'; - -/** - * This weird little class is responsible for checking (and maintaining information) - * whether certain nodes exist in the database. - * - * Note that this information is only calculated for nodes for which `insertMissing` - * option is true. - */ -export class GraphNodeDbExistence { - static createEveryNodeExistsExistence() { - return new GraphNodeDbExistence(new Map()); - } - - /** - * Goes through the graphs and identifies nodes that may not exist in the db - * (nodes that have an id, but are not found in `currentGraph`) and - * creates an instance of GraphNodeDbExistence that can be used to - * synchronously check if a node exist in the database. - */ - static async create({ builder, graph, graphOptions, currentGraph }) { - if (graphOptions.isInsertOnly()) { - // With insertGraph, we never want to do anything but inserts. - return GraphNodeDbExistence.createEveryNodeExistsExistence(); - } - - const graphData = new GraphData({ - graph, - graphOptions, - currentGraph, - // We don't yet have an instance of GraphNodeDbExistence since we are - // creating one. We can (and should) safely use an instance that - // assumes that all nodes exist in the db for the purposes of this - // method. - nodeDbExistence: GraphNodeDbExistence.createEveryNodeExistsExistence(), - }); - - const { mayNotExist, mayNotExistNodes } = createMayNotExistMap(graphData); - - if (mayNotExist.size == 0) { - // Early exit if we found no items for which we should check their - // existence in the db. - return GraphNodeDbExistence.createEveryNodeExistsExistence(); - } - - const dontExist = await createDontExistMap({ builder, mayNotExist, mayNotExistNodes }); - - return new GraphNodeDbExistence(dontExist); - } - - constructor(dontExist) { - this.dontExist = dontExist; - } - - doesNodeExistInDb(node) { - const idMap = this.dontExist.get(node.modelClass); - - if (!idMap) { - return true; - } - - return !idMap.has(node.obj.$idKey()); - } -} - -function createMayNotExistMap(graphData) { - const { graph, currentGraph, graphOptions } = graphData; - - const mayNotExist = new Map(); - const mayNotExistNodes = []; - - for (const node of graph.nodes) { - if ( - // As an optimization, only consider nodes for which `insertMissing` is true. - // We only need the information for those nodes. - graphOptions.shouldInsertMissing(node) && - // Only consider nodes that will be related. We don't consider nodes that - // would get inserted with an id. Those will still result in a unique - // constraint error. - graphOptions.shouldRelate(node, graphData) && - // Relate nodes may not have an id if they are `#ref` nodes. Only consider - // nodes that have an id so that we can check the existence. - node.hasId && - // We can ignore nodes if they are found anywhere in the graph. `shouldRelate` - // only checks if the node is found in the same relation. - !hasNodeById(currentGraph, node) - ) { - if (!mayNotExist.has(node.modelClass)) { - mayNotExist.set(node.modelClass, new Map()); - } - - mayNotExist.get(node.modelClass).set(node.obj.$idKey(), node.obj.$id()); - mayNotExistNodes.push(node); - } - } - - return { - mayNotExist, - mayNotExistNodes, - }; -} - -function hasNodeById(currentGraph, nodeToFind) { - const { modelClass } = nodeToFind; - - const tableToFind = modelClass.getTableName(); - const idProps = modelClass.getIdPropertyArray(); - - return currentGraph.nodes.some((node) => { - return ( - node.modelClass.getTableName() === tableToFind && - idProps.every((idProp) => node.obj[idProp] === nodeToFind.obj[idProp]) - ); - }); -} - -async function createDontExistMap({ builder, mayNotExist, mayNotExistNodes }) { - const dontExist = cloneExistenceMap(mayNotExist); - const existenceCheckQueries = createExistenceCheckQueries({ builder, mayNotExist }); - - const results = await promiseUtils.map(existenceCheckQueries, (builder) => builder.execute(), { - concurrency: GraphAction.getConcurrency(builder, mayNotExistNodes), - }); - - // Remove all items from the mayNotExist map that we have just proven to exist - // by executing the existenceCheckQueries. - for (const modelResult of results) { - for (const item of modelResult) { - const modelClass = item.constructor; - const idMap = dontExist.get(modelClass); - // Exist, remove from the map. - idMap.delete(item.$idKey()); - } - } - - // Now only items that don't exist in the db are left in this map. - return dontExist; -} - -function createExistenceCheckQueries({ builder, mayNotExist }) { - const builders = []; - - for (const [modelClass, idMap] of mayNotExist.entries()) { - const ids = Array.from(idMap.values()); - - // Create one query per model class (table) to fetch the identifiers - // of the nodes that may not exist. These queries should be super fast - // since they come straight from the index. - builders.push( - modelClass - .query() - .childQueryOf(builder, childQueryOptions()) - .findByIds(ids) - .select(modelClass.getIdColumnArray()), - ); - } - - return builders; -} - -function childQueryOptions() { - return { - fork: true, - isInternalQuery: true, - }; -} - -function cloneExistenceMap(exixtenseMap) { - const clone = new Map(exixtenseMap); - - for (const modelClass of clone.keys()) { - clone.set(modelClass, new Map(clone.get(modelClass))); - } - - return clone; -} diff --git a/lib/queryBuilder/graph/GraphOperation.js b/lib/queryBuilder/graph/GraphOperation.js deleted file mode 100644 index 54fd4817d..000000000 --- a/lib/queryBuilder/graph/GraphOperation.js +++ /dev/null @@ -1,21 +0,0 @@ -export class GraphOperation { - constructor(graphData) { - this.graphData = graphData; - } - - get graph() { - return this.graphData.graph; - } - - get currentGraph() { - return this.graphData.currentGraph; - } - - get graphOptions() { - return this.graphData.graphOptions; - } - - createActions() { - return []; - } -} diff --git a/lib/queryBuilder/graph/GraphOptions.js b/lib/queryBuilder/graph/GraphOptions.js index ccc2a5e59..c41c0caec 100644 --- a/lib/queryBuilder/graph/GraphOptions.js +++ b/lib/queryBuilder/graph/GraphOptions.js @@ -16,7 +16,7 @@ const ALLOW_REFS = 'allowRefs'; const ALLOW_MARKERS = 'allowMarkers'; // The options that are matched against the relation path of each node, see -// `_hasOption()`. +// `has()`. const NODE_OPTION_NAMES = [ NO_RELATE, NO_UNRELATE, @@ -48,23 +48,14 @@ export const FetchStrategy = { }; export class GraphOptions { - // `rootRelationPath` is the relation path of the root node of a recursive - // upsert in the original graph, against which the relation paths and - // expressions of the options match. `insertOnly` is only set by - // `insertGraph`, which never fetches, updates or removes existing rows. - constructor(options, rootRelationPath = [], insertOnly = false) { - if (options instanceof GraphOptions) { - this.options = options.options; - this.rootRelationPath = options.rootRelationPath; - this.insertOnly = options.insertOnly; - } else { - checkForUnknownOptions(options, insertOnly); - this.options = parseRelationExpressions( - insertOnly ? { ...options, ...INSERT_ONLY_OPTIONS } : options, - ); - this.rootRelationPath = rootRelationPath; - this.insertOnly = insertOnly; - } + // `insertOnly` is only set by `insertGraph`, which never fetches, updates or + // removes existing rows. + constructor(options, insertOnly = false) { + checkForUnknownOptions(options, insertOnly); + this.options = parseRelationExpressions( + insertOnly ? { ...options, ...INSERT_ONLY_OPTIONS } : options, + ); + this.insertOnly = insertOnly; } isFetchStrategy(strategy) { @@ -83,125 +74,15 @@ export class GraphOptions { return this.insertOnly; } - // Like `shouldRelate` but ignores settings that explicitly disable relate operations. - shouldRelateIgnoreDisable(node, graphData) { - if (node.toBeUnrelatedOrDeleted) { - return false; - } - - if (node.isReference || node.isDbReference) { - return true; - } - - return ( - this._hasOption(node, RELATE) && - !getCurrentNode(node, graphData) && - !!node.parentEdge && - !!node.parentEdge.relation && - hasRelateProp(node) && - graphData.nodeDbExistence.doesNodeExistInDb(node) - ); - } - - shouldRelate(node, graphData) { - return !this._hasOption(node, NO_RELATE) && this.shouldRelateIgnoreDisable(node, graphData); - } - - // Like `shouldInsert` but ignores settings that explicitly disable insert operations. - shouldInsertIgnoreDisable(node, graphData) { - if (node.toBeUnrelatedOrDeleted) { - return false; - } - - return ( - !getCurrentNode(node, graphData) && - !this.shouldRelateIgnoreDisable(node, graphData) && - (!node.hasId || this.shouldInsertMissing(node)) - ); - } - - shouldInsert(node, graphData) { - return !this._hasOption(node, NO_INSERT) && this.shouldInsertIgnoreDisable(node, graphData); - } - - shouldInsertMissing(node) { - return this._hasOption(node, INSERT_MISSING); - } - - // Like `shouldPatch() || shouldUpdate()` but ignores settings that explicitly disable - // update or patch operations. - shouldPatchOrUpdateIgnoreDisable(node, graphData) { - if (node.toBeUnrelatedOrDeleted) { - return false; - } - - if (this.shouldRelate(node, graphData)) { - // We should update all nodes that are going to be related. Note that - // we don't actually update anything unless there is something to update - // so this is just a preliminary test. - return true; - } - - return !!getCurrentNode(node, graphData); - } - // Whether a node that the graph relates or patches is written with `patch()` // or `update()`. With `noUpdate`, it's neither, and only the foreign keys of // its relates and BelongsToOneRelations are patched. shouldPatch(node) { - return !this._hasOption(node, NO_UPDATE) && !this._hasOption(node, UPDATE); + return !this.has(node, NO_UPDATE) && !this.has(node, UPDATE); } shouldUpdate(node) { - return !this._hasOption(node, NO_UPDATE) && this._hasOption(node, UPDATE); - } - - // Like `shouldUnrelate` but ignores settings that explicitly disable unrelate operations. - shouldUnrelateIgnoreDisable(currentNode) { - return this._hasOption(currentNode, UNRELATE); - } - - shouldUnrelate(currentNode, graphData) { - const node = getNode(currentNode, graphData.graph); - if (node && node.toBeUnrelated) return true; - - return ( - !node && - !hasIgnoredOtherModel(currentNode, graphData) && - !this._hasOption(currentNode, NO_UNRELATE) && - (this.shouldUnrelateIgnoreDisable(currentNode) || - // The unrelate replaces a delete, so `noDelete` disables it too. - (!this._hasOption(currentNode, NO_DELETE) && shouldUnrelateKeptRow(currentNode, graphData))) - ); - } - - shouldDelete(currentNode, graphData) { - const node = getNode(currentNode, graphData.graph); - // `#unrelate` takes precedence over `#delete` if both are set. - if (node && node.toBeDeleted && !node.toBeUnrelated) return true; - - return ( - !node && - !hasIgnoredOtherModel(currentNode, graphData) && - !this._hasOption(currentNode, NO_DELETE) && - !this.shouldUnrelateIgnoreDisable(currentNode) && - // Rows that the graph keeps in another place must not be deleted (#214). - !graphData.keptNodeForRow(currentNode) - ); - } - - shouldInsertOrRelate(node, graphData) { - return this.shouldInsert(node, graphData) || this.shouldRelate(node, graphData); - } - - // The graph ignores models that it neither inserts, relates nor patches, like - // models marked with `#unrelate` or `#delete` and models skipped by `noInsert` - // or `noRelate`. - shouldIgnore(node, graphData) { - return ( - !this.shouldInsertOrRelate(node, graphData) && - !this.shouldPatchOrUpdateIgnoreDisable(node, graphData) - ); + return !this.has(node, NO_UPDATE) && this.has(node, UPDATE); } allowRefs() { @@ -212,33 +93,15 @@ export class GraphOptions { return !!this.options[ALLOW_MARKERS]; } - // The options of recursive upserts keep matching against the relation paths - // of the original graph, which the relation paths of the new root lead to. - rebasedOptions(newRoot) { - return new GraphOptions(this.options, [...this.rootRelationPath, ...newRoot.relationPath]); - } - - _hasOption(node, optionName) { + // Whether the option applies to the relation path of `node`. + has(node, optionName) { const option = this.options[optionName]; - if (isRelationExpression(option)) { - return matchesRelationExpression(option, this._relationPath(node)); - } else if (Array.isArray(option)) { - return option.includes(this._relationPath(node).join('.')); - } else if (typeof option === 'boolean') { - return option; - } else if (option === undefined) { - return false; - } else { - throw new Error( - `expected ${optionName} option value "${option}" to be a boolean, an array of relation paths or a relation expression`, - ); - } - } - - // The relation path of the node in the original graph, see `rebasedOptions()`. - _relationPath(node) { - return [...this.rootRelationPath, ...node.relationPath]; + return isRelationExpression(option) + ? matchesRelationExpression(option, node.relationPath) + : Array.isArray(option) + ? option.includes(node.relationPathKey) + : !!option; } } @@ -260,7 +123,7 @@ function checkForUnknownOptions(options, insertOnly) { } // Parses the string values of the node options into relation expressions, -// once per operation. +// once per operation, and checks the types of the other values. function parseRelationExpressions(options) { let parsedOptions = options; @@ -273,6 +136,10 @@ function parseRelationExpressions(options) { } parsedOptions[optionName] = parseRelationExpression(optionName, option); + } else if (!(option === undefined || typeof option === 'boolean' || Array.isArray(option))) { + throw new Error( + `expected ${optionName} option value "${option}" to be a boolean, an array of relation paths or a relation expression`, + ); } } @@ -351,65 +218,3 @@ function matchesRelationExpression(expression, relationPath) { function isRelationExpression(value) { return !!value && value.isObjectionRelationExpression === true; } - -function getCurrentNode(node, graphData) { - if (!graphData || !node) { - return null; - } - - return graphData.currentGraph.nodeForNode(node); -} - -// For `HasMany` and `HasOne` relations, the relate prop is the id of the -// related model. Use the id state of the input graph, since the parent's -// insert copies the foreign key into the related model, which can complete -// a composite id that contains it and would turn an insert into a relate (#2544). -function hasRelateProp(node) { - const relation = node.parentEdge.relation; - - if (relation.isObjectionHasManyRelation) { - return node.hasId; - } else { - return relation.hasRelateProp(node.obj); - } -} - -// An ignored model that isn't the current model of a to-one relation doesn't -// replace it, so we keep the current one instead of removing it as missing. -function hasIgnoredOtherModel(currentNode, graphData) { - const { relation } = currentNode.parentEdge; - const parentNode = getNode(currentNode.parentNode, graphData.graph); - - return ( - relation.isOneToOne() && - !!parentNode && - parentNode.edges.some( - (edge) => - edge.relation === relation && - edge.isOwnerNode(parentNode) && - graphData.graphOptions.shouldIgnore(edge.relatedNode, graphData), - ) - ); -} - -// A row that the graph keeps in another place is unrelated here instead of -// being deleted, unless it's a has-many row that the graph relates through the -// same relation: That relate moves it by overwriting the foreign key, which -// may not be nullable. -function shouldUnrelateKeptRow(currentNode, graphData) { - const keptNode = graphData.keptNodeForRow(currentNode); - const { relation } = currentNode.parentEdge; - - return ( - !!keptNode && - !(relation.isObjectionHasManyRelation && keptNode.parentEdge?.relation === relation) - ); -} - -function getNode(currentNode, graph) { - if (!graph || !currentNode) { - return null; - } - - return graph.nodeForNode(currentNode); -} diff --git a/lib/queryBuilder/graph/GraphPlan.js b/lib/queryBuilder/graph/GraphPlan.js new file mode 100644 index 000000000..dabf9dc14 --- /dev/null +++ b/lib/queryBuilder/graph/GraphPlan.js @@ -0,0 +1,301 @@ +// What a graph insert or upsert does with each model of the graph and each row +// of the current graph: The rules that decide it, and the pruning of branches +// that nothing is written in. +export class GraphPlan { + constructor({ nodes, graphOptions }) { + this.nodes = nodes; + this.currentNodes = []; + this.graphOptions = graphOptions; + this.actions = null; + // The decisions only change while the current graph is loaded, and are + // cached once it is, see `cacheDecisions()`: + this._intents = null; + this._nodeActions = null; + this._activeNodes = null; + this._activeCurrentNodes = null; + this._keptNodesByRowKey = null; + } + + get activeNodes() { + return (this._activeNodes ??= this.nodes.filter((node) => !node.isPruned)); + } + + get activeCurrentNodes() { + return (this._activeCurrentNodes ??= this.currentNodes.filter( + (currentNode) => !currentNode.isPruned, + )); + } + + cacheDecisions() { + this._intents = new Map(); + this._nodeActions = new Map(); + } + + // What the data asks to do with `node`, before the options that disable it: + // Models marked with `#unrelate` or `#delete` are removed, references and + // models with ids that the options let relate are related, models at their + // place are patched, and new models are inserted. Models with ids that are + // neither are `'missing'`. + intentFor(node) { + let intent = this._intents?.get(node); + + if (!intent) { + intent = this._getIntent(node); + this._intents?.set(node, intent); + } + + return intent; + } + + _getIntent(node) { + const insertMissing = this.graphOptions.has(node, 'insertMissing'); + + return node.toBeUnrelatedOrDeleted + ? 'remove' + : node.isReference + ? 'relate' + : node.isMissing + ? insertMissing + ? 'insert' + : 'missing' + : node.isDbReference || this._canRelate(node) + ? 'relate' + : node.current + ? 'patch' + : !node.hasId || insertMissing + ? 'insert' + : 'missing'; + } + + // Return true if the graph writes the branch below `node`: below models at + // their place, inserted models, related models with fetched rows and `#ref` + // models, but not below removed or ignored ones. + writesBranchOf(node) { + const action = this.actionFor(node); + + return ( + !node.toBeUnrelatedOrDeleted && + (!!node.current || + action === 'insert' || + (action === 'relate' && (!!node.row || node.isReference))) + ); + } + + // What the graph does with `node`: `'insert'`, `'relate'`, `'patch'` or + // `'ignore'`. Once the writes start, read `actions` instead. + actionFor(node) { + let action = this._nodeActions?.get(node); + + if (!action) { + action = this._getAction(node); + this._nodeActions?.set(node, action); + } + + return action; + } + + _getAction(node) { + const { graphOptions } = this; + + switch (this.intentFor(node)) { + case 'insert': + return graphOptions.has(node, 'noInsert') ? 'ignore' : 'insert'; + case 'relate': + return !graphOptions.has(node, 'noRelate') ? 'relate' : node.current ? 'patch' : 'ignore'; + case 'patch': + return 'patch'; + default: + return 'ignore'; + } + } + + // What the graph does with the row of `currentNode`: `'delete'`, + // `'unrelate'` or `'keep'`. + actionForCurrent(currentNode) { + const { node } = currentNode; + const has = (optionName) => this.graphOptions.has(currentNode, optionName); + + if (node) { + // `#unrelate` takes precedence over `#delete` if both are set. + return node.toBeUnrelated ? 'unrelate' : node.toBeDeleted ? 'delete' : 'keep'; + } else if (this._hasIgnoredOtherModel(currentNode)) { + return 'keep'; + } else if (has('unrelate')) { + return has('noUnrelate') ? 'keep' : 'unrelate'; + } else { + // Rows that the graph keeps in another place must not be deleted (#214). + // They're unrelated instead, which replaces the delete, so `noDelete` + // disables it too. + const keptNode = this.keptNodeForRow(currentNode); + + return keptNode + ? isUnrelatedWhenKept(currentNode, keptNode) && !has('noUnrelate') && !has('noDelete') + ? 'unrelate' + : 'keep' + : has('noDelete') + ? 'keep' + : 'delete'; + } + } + + // Leave out the branches that require no writes. We never do anything below + // models that are removed, and we never delete recursively. + prune() { + const isInsertOnly = this.graphOptions.isInsertOnly(); + // The rows that were fetched for pruned nodes are left alone too: + const pruneDescendants = (nodes) => { + for (const node of nodes) { + for (const descendant of node.descendants) { + descendant.isPruned = true; + + for (const row of descendant.row ? [descendant.row, ...descendant.row.descendants] : []) { + row.isPruned = true; + } + } + } + + this._activeNodes = null; + this._activeCurrentNodes = null; + }; + + // Nothing is written below models that are neither inserted nor at their + // place. `upsertGraph()` upserts the branches below related models, whose + // rows were fetched for that, and below `#ref` models of new models, but + // `insertGraph()` leaves them out (#217). + pruneDescendants( + this.activeNodes.filter( + (node) => + !node.current && + this.intentFor(node) !== 'insert' && + !( + this.actionFor(node) === 'relate' && + (node.row || (node.isReference && !this.graphOptions.isInsertOnly())) + ), + ), + ); + + // Nothing is written below models that `noInsert` skips, as their + // descendants would be orphans (#215). Branches that references from + // outside point into are kept, since the references need their models. + pruneDescendants( + this.activeNodes.filter( + (node) => + this.intentFor(node) === 'insert' && + this.actionFor(node) === 'ignore' && + !isReferencedFromOutside(node), + ), + ); + + if (!isInsertOnly) { + // Rows that the graph removes are removed without their descendants. Don't + // remove rows in relations that the graph doesn't mention. + const removedNodes = this.activeCurrentNodes.filter((currentNode) => !currentNode.node); + pruneDescendants(removedNodes); + + for (const currentNode of removedNodes) { + const parentNode = currentNode.parent?.node; + + if (parentNode && parentNode.obj[currentNode.relation.name] === undefined) { + currentNode.isPruned = true; + this._activeCurrentNodes = null; + } + } + + // Models marked with `#unrelate` or `#delete` are unrelated or deleted as + // a whole, in the graph and in the current graph. + const markedNodes = this.activeNodes.filter((node) => node.toBeUnrelatedOrDeleted); + pruneDescendants(markedNodes); + pruneDescendants(markedNodes.map((node) => node.current).filter(Boolean)); + } + + return this; + } + + // Decide what to do with each node once, before the first write, since the + // writes change the models that the rules read. Only call this after + // pruning. + plan() { + const actions = new Map(); + + for (const node of this.activeNodes) { + actions.set(node, this.actionFor(node)); + } + + for (const currentNode of this.activeCurrentNodes) { + actions.set(currentNode, this.actionForCurrent(currentNode)); + } + + this.actions = actions; + return this; + } + + _canRelate(node) { + return ( + this.graphOptions.has(node, 'relate') && + !node.current && + !node.isRoot && + hasRelateProp(node) && + !node.isMissing + ); + } + + // An ignored model that isn't the current model of a to-one relation doesn't + // replace it, so we keep the current one instead of removing it as missing. + _hasIgnoredOtherModel(currentNode) { + const { relation } = currentNode; + const parentNode = currentNode.parent?.node; + + return ( + relation.isOneToOne() && + !!parentNode && + parentNode.activeChildren.some( + (child) => child.relation === relation && this.actionFor(child) === 'ignore', + ) + ); + } + + // Find the node that keeps the row of `currentNode` in another place of the + // graph, by relating or patching it there. + keptNodeForRow(currentNode) { + if (!this._keptNodesByRowKey) { + this._keptNodesByRowKey = new Map(); + + for (const node of this.activeNodes) { + const action = this.actionFor(node); + + if (node.obj.$hasId() && (action === 'relate' || action === 'patch')) { + this._keptNodesByRowKey.set(node.rowKey, node); + } + } + } + + return this._keptNodesByRowKey.get(currentNode.rowKey) || null; + } +} + +function isReferencedFromOutside(node) { + const nodes = new Set(node.descendants.filter((descendant) => !descendant.isPruned)); + + return [...nodes].some((descendant) => + [...descendant.referencingNodes, ...descendant.propertyReferencingNodes].some( + (referencingNode) => !referencingNode.isPruned && !nodes.has(referencingNode), + ), + ); +} + +// For `HasMany` and `HasOne` relations, the relate prop is the id of the +// related model. Use the id state of the input graph, since the parent's +// insert copies the foreign key into the related model, which can complete +// a composite id that contains it and would turn an insert into a relate (#2544). +function hasRelateProp(node) { + const { relation } = node; + return relation.keySide === 'related' ? node.hasId : relation.hasRelateProp(node.obj); +} + +// A row that the graph keeps in another place is unrelated here, unless it's a +// has-many row that the graph relates through the same relation: That relate +// moves it by overwriting the foreign key, which may not be nullable. +function isUnrelatedWhenKept(currentNode, keptNode) { + const { relation } = currentNode; + return !(relation.keySide === 'related' && keptNode.relation === relation); +} diff --git a/lib/queryBuilder/graph/GraphUpsert.js b/lib/queryBuilder/graph/GraphUpsert.js index 8d10a9d31..0e432364b 100644 --- a/lib/queryBuilder/graph/GraphUpsert.js +++ b/lib/queryBuilder/graph/GraphUpsert.js @@ -1,26 +1,24 @@ -import { ModelGraph } from '../../model/graph/ModelGraph.js'; -import { ModelGraphEdge } from '../../model/graph/ModelGraphEdge.js'; -import { createNotModelError } from '../../model/graph/ModelGraphBuilder.js'; -import { GraphFetcher } from '../graph/GraphFetcher.js'; -import { GraphInsert } from './insert/GraphInsert.js'; -import { GraphPatch } from './patch/GraphPatch.js'; -import { GraphRelate } from './patch/GraphRelate.js'; -import { GraphDelete } from './delete/GraphDelete.js'; -import { GraphRecursiveUpsert } from './recursiveUpsert/GraphRecursiveUpsert.js'; +import { GraphNode, createNotModelError } from './GraphNode.js'; import { GraphOptions } from './GraphOptions.js'; +import { GraphPlan } from './GraphPlan.js'; +import { GraphWriter } from './GraphWriter.js'; +import { loadCurrentGraph } from './GraphLoader.js'; import { ValidationErrorType } from '../../model/ValidationError.js'; import { RelationExpression } from '../RelationExpression.js'; -import { GraphNodeDbExistence } from './GraphNodeDbExistence.js'; -import { GraphData } from './GraphData.js'; -import { uniqBy, asArray, isObject } from '../../utils/objectUtils.js'; +import { asArray, isObject } from '../../utils/objectUtils.js'; export class GraphUpsert { constructor({ rootModelClass, objects, upsertOptions, insertOnly = false }) { - checkCanBeConvertedToModels(rootModelClass, objects); + // Wrap `undefined` too, so that it fails the check: + for (const obj of Array.isArray(objects) ? objects : [objects]) { + if (!isObject(obj)) { + throw createNotModelError(rootModelClass, obj); + } + } this.objects = rootModelClass.ensureModelArray(objects, GraphUpsert.modelOptions); this.isArray = Array.isArray(objects); - this.graphOptions = new GraphOptions(upsertOptions, [], insertOnly); + this.graphOptions = new GraphOptions(upsertOptions, insertOnly); } static get modelOptions() { @@ -41,442 +39,156 @@ export class GraphUpsert { } } - run(builder) { - const modelClass = builder.modelClass(); + async run(builder) { const { graphOptions } = this; + const nodes = GraphNode.buildGraph(builder.modelClass(), this.objects); + GraphNode.linkReferences(nodes); + const restoreMarkers = GraphNode.removeMarkers(nodes); - const graph = ModelGraph.create(modelClass, this.objects); - assignDbRefsAsRelateProps(graph); - - return createGraphData(builder, graphOptions, graph) - .then(checkForErrors(builder)) - .then(pruneGraphs()) - .then((graphData) => graphData.planActions()) - .then(checkForJoinTableRelatedTableErrors(builder)) - .then(executeOperations(builder)) - .then(removeReferenceProps(graph.nodes)) - .then(returnResult(this.objects, this.isArray)); - } -} - -function checkCanBeConvertedToModels(modelClass, objects) { - // Wrap `undefined` too, so that it fails the check: - for (const obj of Array.isArray(objects) ? objects : [objects]) { - if (!isObject(obj)) { - throw createNotModelError(modelClass, obj); - } - } -} - -function assignDbRefsAsRelateProps(graph) { - for (const node of graph.nodes) { - if (!node.parentEdge || !node.parentEdge.relation || !node.isDbReference) { - continue; - } - - node.parentEdge.relation.setRelateProp(node.obj, asArray(node.dbReference)); - } -} - -async function createGraphData(builder, graphOptions, graph) { - const currentGraph = await fetchCurrentGraph(builder, graphOptions, graph); - - const nodeDbExistence = await GraphNodeDbExistence.create({ - builder, - graph, - graphOptions, - currentGraph, - }); - - return new GraphData({ graph, currentGraph, graphOptions, nodeDbExistence }); -} - -function fetchCurrentGraph(builder, graphOptions, graph) { - if (graphOptions.isInsertOnly()) { - return Promise.resolve(ModelGraph.createEmpty()); - } else { - return GraphFetcher.fetchCurrentGraph({ builder, graph, graphOptions }); - } -} - -// Remove branches from the graph that require no operations. For example -// we never want to do anything for descendant nodes of a node that is -// deleted or unrelated. We never delete recursively. -function pruneGraphs() { - return (graphData) => { - pruneRelatedBranches(graphData); - pruneSkippedBranches(graphData); - - if (!graphData.graphOptions.isInsertOnly()) { - pruneDeletedBranches(graphData); - pruneUnrelatedOrDeletedBranches(graphData); - } - - return graphData; - }; -} - -function pruneRelatedBranches(graphData) { - const relateNodes = graphData.graph.nodes.filter((node) => { - return ( - !graphData.currentGraph.nodeForNode(node) && - !graphData.graphOptions.shouldInsertIgnoreDisable(node, graphData) - ); - }); - - removeBranchesFromGraph(findRoots(relateNodes), graphData.graph); -} - -// Nothing is written below nodes that `noInsert` skips, as their descendants -// would be orphans (#215). Branches that references from outside point into -// are kept, since the references need their models. -function pruneSkippedBranches(graphData) { - const { graph, graphOptions } = graphData; - - const skippedNodes = graph.nodes.filter((node) => { - return ( - graphOptions.shouldInsertIgnoreDisable(node, graphData) && - !graphOptions.shouldInsert(node, graphData) && - !isReferencedFromOutside(node.descendantRelationNodes) - ); - }); - - removeBranchesFromGraph(findRoots(skippedNodes), graph); -} - -function isReferencedFromOutside(nodes) { - const nodeSet = new Set(nodes); - - return nodes.some((node) => - node.refEdges.some((edge) => edge.isRelatedNode(node) && !nodeSet.has(edge.ownerNode)), - ); -} - -function pruneDeletedBranches(graphData) { - const { graph, currentGraph } = graphData; - - const deleteNodes = currentGraph.nodes.filter((currentNode) => !graph.nodeForNode(currentNode)); - const roots = findRoots(deleteNodes); - - // Don't delete relations the current graph doesn't even mention. - // So if the parent node doesn't even have the relation, it's not - // supposed to be deleted. - const rootsNotInRelation = roots.filter((deleteRoot) => { - if (!deleteRoot.parentNode) { - return false; - } - - const { relation } = deleteRoot.parentEdge; - const parentNode = graph.nodeForNode(deleteRoot.parentNode); - - if (!parentNode) { - return false; - } - - return parentNode.obj[relation.name] === undefined; - }); - - removeBranchesFromGraph(roots, currentGraph); - removeNodesFromGraph(new Set(rootsNotInRelation), currentGraph); -} - -// Models marked with `#unrelate` or `#delete` are unrelated or deleted -// as a whole. Their relations are ignored, both in the upserted graph -// and in the current graph. -function pruneUnrelatedOrDeletedBranches(graphData) { - const { graph, currentGraph } = graphData; - - const roots = findRoots(graph.nodes.filter((node) => node.toBeUnrelatedOrDeleted)); - const currentRoots = roots.map((node) => currentGraph.nodeForNode(node)).filter(Boolean); - - removeBranchesFromGraph(roots, graph); - removeBranchesFromGraph(currentRoots, currentGraph); -} - -function findRoots(nodes) { - const nodeSet = new Set(nodes); - - return uniqBy( - nodes.filter((node) => { - let parentNode = node.parentNode; - - while (parentNode) { - if (nodeSet.has(parentNode)) { - return false; - } - - parentNode = parentNode.parentNode; - } - - return true; - }), - ); -} - -function removeBranchesFromGraph(branchRoots, graph) { - const nodesToRemove = new Set( - branchRoots.reduce( - (nodesToRemove, node) => [...nodesToRemove, ...node.descendantRelationNodes], - [], - ), - ); - - removeNodesFromGraph(nodesToRemove, graph); -} - -function removeNodesFromGraph(nodesToRemove, graph) { - const edgesToRemove = new Set(); - - for (const node of nodesToRemove) { - for (const edge of node.edges) { - const otherNode = edge.getOtherNode(node); - - if (!nodesToRemove.has(otherNode)) { - otherNode.removeEdge(edge); - edgesToRemove.add(edge); + for (const node of nodes) { + const { referencedNode } = node; + + if (node.parent && node.isDbReference) { + node.relation.setRelateProp(node.obj, asArray(node.dbReference)); + } else if (referencedNode?.obj.$hasId() && !node.obj.$hasId()) { + // A `#ref` to an existing model is that model, also when its relations + // are loaded and upserted: + node.obj.$id(referencedNode.obj.$id()); } } - } - - graph.nodes = graph.nodes.filter((node) => !nodesToRemove.has(node)); - graph.edges = graph.edges.filter((edge) => !edgesToRemove.has(edge)); - - return graph; -} - -function checkForErrors(builder) { - return (graphData) => { - checkForNotFoundErrors(graphData, builder); - checkForUnallowedRelationErrors(graphData, builder); - checkForUnallowedReferenceErrors(graphData, builder); - // Root models can't be marked even with `allowMarkers`, so report that - // first instead of pointing to the option. - checkForRootUnrelateOrDeleteErrors(graphData, builder); - checkForUnallowedMarkerErrors(graphData, builder); - - if (graphData.graphOptions.isInsertOnly()) { - checkForHasManyRelateErrors(graphData); - checkForInsertBelowRelateErrors(graphData, builder); - } - - return graphData; - }; -} - -// Inserting or relating a node of a many-to-many relation inserts a join row, -// which would be a new related row if the join table is also the related -// table (#133). Check this before this graph is written. It runs after -// pruning, as the branches below related nodes are upserted and checked by -// nested `upsertGraph` calls against their own current graphs. -function checkForJoinTableRelatedTableErrors(builder) { - return (graphData) => { - const { currentGraph, graph, actions } = graphData; - for (const node of graph.nodes) { - const relation = node.parentEdge && node.parentEdge.relation; - const action = actions.get(node); + const plan = new GraphPlan({ nodes, graphOptions }); - if ( - relation && - relation.isObjectionManyToManyRelation && - !currentGraph.nodeForNode(node) && - (action === 'insert' || action === 'relate') - ) { - relation.assertCanModifyJoinTable(builder, action); + try { + if (!graphOptions.isInsertOnly()) { + await loadCurrentGraph(builder, plan); } - } - - return graphData; - }; -} - -function checkForNotFoundErrors(graphData, builder) { - const { graphOptions, currentGraph, graph } = graphData; - // The branches below related nodes are upserted and checked by nested - // `upsertGraph` calls, against the current graphs of the related nodes: - const nestedNodes = new Set( - graph.nodes - .filter((node) => graphOptions.shouldRelate(node, graphData)) - .flatMap((node) => node.descendantRelationNodes), - ); - for (const node of graph.nodes) { - if ( - node.obj.$hasId() && - !nestedNodes.has(node) && - !node.toBeUnrelatedOrDeleted && - !graphOptions.shouldInsertIgnoreDisable(node, graphData) && - !graphOptions.shouldRelateIgnoreDisable(node, graphData) && - !currentGraph.nodeForNode(node) - ) { - if (!node.parentNode) { - throw node.modelClass.createNotFoundError(builder.context(), { - message: `root model (id=${node.obj.$id()}) does not exist. If you want to insert it with an id, use the insertMissing option`, - dataPath: node.dataPath, - }); - } else { - throw node.modelClass.createNotFoundError(builder.context(), { - message: `model (id=${node.obj.$id()}) is not a child of model (id=${node.parentNode.obj.$id()}). If you want to relate it, use the relate option. If you want to insert it with an id, use the insertMissing option`, - dataPath: node.dataPath, - }); - } + plan.cacheDecisions(); + checkGraph(builder, plan); + plan.prune().plan(); + checkJoinTables(builder, plan); + await new GraphWriter(plan).write(builder); + } catch (err) { + // The input can be retried as it was: + restoreMarkers(); + throw err; } + + return this.isArray ? this.objects : this.objects[0]; } } -function checkForUnallowedRelationErrors(graphData, builder) { - const { graph } = graphData; - const allowedExpression = builder.allowedGraphExpression(); +function checkGraph(builder, plan) { + const { nodes, graphOptions } = plan; + const modelClass = builder.modelClass(); + const createInvalidGraphError = (message) => + modelClass.createValidationError({ type: ValidationErrorType.InvalidGraph, message }); - if (allowedExpression) { - const rootsObjs = graph.nodes.filter((node) => !node.parentEdge).map((node) => node.obj); - const expression = RelationExpression.fromModelGraph(rootsObjs); + for (const node of nodes) { + if (plan.intentFor(node) === 'missing') { + const id = node.obj.$id(); - if (!allowedExpression.isSubExpression(expression)) { - throw builder.modelClass().createValidationError({ - type: ValidationErrorType.UnallowedRelation, - message: 'trying to upsert an unallowed relation', + throw node.modelClass.createNotFoundError(builder.context(), { + message: node.isMissing + ? `model (id=${id}) does not exist. If you want to insert it with an id, use the insertMissing option` + : node.isRoot + ? `root model (id=${id}) does not exist. If you want to insert it with an id, use the insertMissing option` + : `model (id=${id}) is not a child of model (id=${node.parent.obj.$id()}). If you want to relate it, use the relate option. If you want to insert it with an id, use the insertMissing option`, + dataPath: node.dataPath, }); } } -} - -function checkForUnallowedReferenceErrors(graphData, builder) { - const { graph, graphOptions } = graphData; - - if (graphOptions.allowRefs()) { - return; - } - if (graph.edges.some((edge) => edge.type === ModelGraphEdge.Type.Reference)) { - throw builder.modelClass().createValidationError({ - type: ValidationErrorType.InvalidGraph, - message: - '#ref references are not allowed in a graph by default. see the allowRefs insert/upsert graph option', + const allowedExpression = builder.allowedGraphExpression(); + const rootObjs = nodes.filter((node) => node.isRoot).map((node) => node.obj); + + if ( + allowedExpression && + !allowedExpression.isSubExpression(RelationExpression.fromModelGraph(rootObjs)) + ) { + throw modelClass.createValidationError({ + type: ValidationErrorType.UnallowedRelation, + message: 'trying to upsert an unallowed relation', }); } -} -// `#unrelate` and `#delete` markers come from the graph data, which may be -// untrusted input. Like `#ref`, they have to be allowed explicitly (#2410). -// An insert has nothing to unrelate or delete, so `insertGraph` rejects them. -function checkForUnallowedMarkerErrors(graphData, builder) { - const { graph, graphOptions } = graphData; - - if (graphOptions.allowMarkers()) { - return; + if ( + !graphOptions.allowRefs() && + nodes.some((node) => node.isReference || node.propertyReferences.length > 0) + ) { + throw createInvalidGraphError( + '#ref references are not allowed in a graph by default. see the allowRefs insert/upsert graph option', + ); } - const node = graph.nodes.find((node) => node.toBeUnrelatedOrDeleted); + // `#unrelate` and `#delete` markers come from the graph data, which may be + // untrusted input. Like `#ref`, they have to be allowed explicitly (#2410). + // Root models can't be marked even with `allowMarkers`, so report that first. + // An insert has nothing to unrelate or delete, so `insertGraph` rejects them. + const markedNode = nodes.find((node) => node.toBeUnrelatedOrDeleted); - if (node) { - const { graphUnrelateProp, graphDeleteProp } = node.modelClass; - - throw builder.modelClass().createValidationError({ - type: ValidationErrorType.InvalidGraph, - message: graphOptions.isInsertOnly() - ? `${graphUnrelateProp} and ${graphDeleteProp} can only be used in upsertGraph, not in insertGraph` - : `${graphUnrelateProp} and ${graphDeleteProp} markers are not allowed in a graph by default. see the allowMarkers upsertGraph option`, - }); - } -} + if (markedNode) { + const { graphUnrelateProp, graphDeleteProp } = markedNode.modelClass; + const markers = `${graphUnrelateProp} and ${graphDeleteProp}`; -function checkForRootUnrelateOrDeleteErrors(graphData, builder) { - for (const node of graphData.graph.nodes) { - if (!node.parentEdge && node.toBeUnrelatedOrDeleted) { - const { graphUnrelateProp, graphDeleteProp } = node.modelClass; - - throw builder.modelClass().createValidationError({ - type: ValidationErrorType.InvalidGraph, - message: `${graphUnrelateProp} and ${graphDeleteProp} can only be used for related models, not for root models`, - }); + if (nodes.some((node) => node.isRoot && node.toBeUnrelatedOrDeleted)) { + throw createInvalidGraphError( + `${markers} can only be used for related models, not for root models`, + ); + } else if (!graphOptions.allowMarkers()) { + throw createInvalidGraphError( + graphOptions.isInsertOnly() + ? `${markers} can only be used in upsertGraph, not in insertGraph` + : `${markers} markers are not allowed in a graph by default. see the allowMarkers upsertGraph option`, + ); } } -} -function checkForHasManyRelateErrors(graphData) { - const { graph, graphOptions } = graphData; + if (graphOptions.isInsertOnly()) { + const relatedNodes = nodes.filter((node) => plan.actionFor(node) === 'relate'); - for (const node of graph.nodes) { - if ( - graphOptions.shouldRelate(node, graphData) && - node.parentEdge.relation.isObjectionHasManyRelation - ) { + if (relatedNodes.some((node) => node.relation.keySide === 'related')) { throw new Error( 'You cannot relate HasManyRelation or HasOneRelation using insertGraph, because those require update operations. Consider using upsertGraph instead.', ); } - } -} - -// `upsertGraph` upserts the branches below related models recursively, but -// `insertGraph` prunes them, which would drop the new models there (#217). -// Models with ids below related models are taken as fetched models with -// loaded relations, and are ignored. -function checkForInsertBelowRelateErrors(graphData, builder) { - const { graph, graphOptions } = graphData; - for (const node of graph.nodes) { - if (graphOptions.shouldRelate(node, graphData)) { - const newNode = node.descendantRelationNodes.find( - (descendantNode) => - !descendantNode.hasId && graphOptions.shouldInsert(descendantNode, graphData), + // `upsertGraph()` upserts the branches below related models, but + // `insertGraph()` leaves them out, which would drop the new models there + // (#217). Models with ids below related models are taken as fetched models + // with loaded relations, and are ignored. + for (const node of relatedNodes) { + const newNode = node.descendants.find( + (descendant) => !descendant.hasId && plan.actionFor(descendant) === 'insert', ); if (newNode) { - throw builder.modelClass().createValidationError({ - type: ValidationErrorType.InvalidGraph, - message: `insertGraph cannot insert the new model at '${newNode.dataPathKey.substring(1)}' below a related model. Insert it with a separate query or use upsertGraph instead`, - }); + throw createInvalidGraphError( + `insertGraph cannot insert the new model at '${newNode.dataPathKey.substring(1)}' below a related model. Insert it with a separate query or use upsertGraph instead`, + ); } } } } -function executeOperations(builder) { - return (graphData) => { - const operations = graphData.graphOptions.isInsertOnly() - ? [new GraphInsert(graphData)] - : [ - new GraphRelate(graphData), - new GraphDelete(graphData, { belongsToOne: false }), - new GraphInsert(graphData), - new GraphPatch(graphData), - new GraphDelete(graphData, { belongsToOne: true }), - new GraphRecursiveUpsert(graphData), - ]; +// Inserting or relating a model of a many-to-many relation inserts a join row, +// which would be a new related row if the join table is also the related table +// (#133). +function checkJoinTables(builder, plan) { + for (const node of plan.activeNodes) { + const action = plan.actions.get(node); - return operations.reduce((promise, operation) => { - const actions = operation.createActions(); - - return promise.then(() => executeActions(builder, actions)); - }, Promise.resolve()); - }; -} - -function executeActions(builder, actions) { - return actions.reduce( - (promise, action) => promise.then(() => action.run(builder)), - Promise.resolve(), - ); -} - -// The reference markers only describe the input graph, don't return them. -// `nodes` are the nodes before pruning, since the results of pruned branches -// contain markers too. -function removeReferenceProps(nodes) { - return () => { - for (const node of nodes) { - delete node.obj[node.modelClass.uidRefProp]; - delete node.obj[node.modelClass.dbRefProp]; + if ( + node.relation?.keySide === 'join' && + !node.current && + (action === 'insert' || action === 'relate') + ) { + node.relation.assertCanModifyJoinTable(builder, action); } - }; -} - -function isOnConflictOperation(op) { - return op.name === 'onConflict' || op.name === 'ignore' || op.name === 'merge'; + } } -function returnResult(objects, isArray) { - return () => (isArray ? objects : objects[0]); +function isOnConflictOperation(operation) { + return ['onConflict', 'ignore', 'merge'].includes(operation.name); } diff --git a/lib/queryBuilder/graph/GraphWriter.js b/lib/queryBuilder/graph/GraphWriter.js new file mode 100644 index 000000000..667e5b1b4 --- /dev/null +++ b/lib/queryBuilder/graph/GraphWriter.js @@ -0,0 +1,1194 @@ +import { getModel } from '../../model/getModel.js'; +import { raw } from '../RawBuilder.js'; +import { getDeclaredColumns } from '../operations/fieldExpressions.js'; +import { isFieldExpressionKey } from '../../utils/parseFieldExpression.js'; +import { getMaxRows, hasBatchInsert, hasBatchInsertDefaults } from '../../utils/knexUtils.js'; +import { isInternalProp } from '../../utils/internalPropUtils.js'; +import { + asArray, + chunk, + difference, + get, + groupBy, + isObject, + isPlainObject, + jsonEquals, + set, +} from '../../utils/objectUtils.js'; +import { map as promiseMap } from '../../utils/promiseUtils/index.js'; + +const MAX_INSERT_BATCH_SIZE = 100; +const MAX_BULK_PATCH_SIZE = 500; +const MAX_CONCURRENCY = 100; + +// Write a planned graph in steps, in an order that keeps keys and constraints +// intact (#214): +// 1. Move the rows that the graph relates to existing owners of HasMany and +// HasOne relations, as they may still sit below rows that the next step +// deletes. Rows that wait for inserts are patched again in step 4. +// 2. Remove rows that reference their owners, before the inserts that may need +// their unique keys. +// 3. Insert the new rows in the order of their keys and references, then the +// keys deferred to break cycles and the join rows. +// 4. Patch the changed rows, the relates, and the keys of owners whose +// BelongsToOne rows were inserted, related or removed. +// 5. Remove the BelongsToOne rows that the owners no longer reference. +// `insertGraph()` only inserts. +export class GraphWriter { + constructor(plan) { + this.plan = plan; + // The nodes that each node waits for, through its keys or its references: + this.dependencies = createDependencies(plan); + // Dependencies that are written after the inserts, to break cycles: + this.deferredDependencies = new Set(); + // Relates whose rows are moved to their new owners before the removals, and + // of those the rows that wait for inserts, which are patched again after: + this.movedNodes = new Set( + plan.graphOptions.isInsertOnly() + ? [] + : plan.activeNodes.filter((node) => this._isMovedFirst(node)), + ); + this.waitingMovedNodes = new Set( + [...this.movedNodes].filter((node) => this._waitsForInserts(node)), + ); + this.steps = this._planSteps(); + } + + async write(builder) { + // Fill in the keys that existing models leave out, before new children read them: + for (const node of this.plan.activeNodes) { + copyKeysFromExisting(node); + } + + for (const step of this.steps) { + await this._writeStep(builder, step); + } + + // `#ref` models return the props of the models they refer to: + for (const node of this.plan.activeNodes) { + resolveReference(node); + } + } + + _planSteps() { + const { activeNodes, activeCurrentNodes, actions, graphOptions } = this.plan; + const insertSteps = [ + ...this._planInsertLevels().map((level) => ({ kind: 'insert', ...level })), + { kind: 'deferredKeys', dependencies: [...this.deferredDependencies] }, + { + kind: 'joinRows', + nodes: activeNodes.filter( + (node) => + node.relation?.keySide === 'join' && + !node.current && + ['insert', 'relate'].includes(actions.get(node)), + ), + }, + ]; + + if (graphOptions.isInsertOnly()) { + return insertSteps; + } else { + const removedNodes = activeCurrentNodes.filter((node) => actions.get(node) !== 'keep'); + const isReferencedByOwner = (currentNode) => currentNode.relation.keySide === 'owner'; + + return [ + ...this._planSlotRounds([...this.movedNodes]).map((nodes) => ({ kind: 'move', nodes })), + { + kind: 'remove', + nodes: removedNodes.filter((node) => !isReferencedByOwner(node)), + areReferenced: true, + }, + ...insertSteps, + ...this._planSlotRounds( + activeNodes.filter((node) => { + const action = actions.get(node); + return ( + action === 'patch' || + (action === 'relate' && + (!this.movedNodes.has(node) || this.waitingMovedNodes.has(node))) + ); + }), + ).map((nodes) => ({ kind: 'patch', nodes })), + { + kind: 'remove', + // Owners unrelate their BelongsToOne rows by patching their keys: + nodes: removedNodes.filter( + (node) => isReferencedByOwner(node) && actions.get(node) === 'delete', + ), + areReferenced: false, + }, + ]; + } + } + + _writeStep(builder, step) { + switch (step.kind) { + case 'move': + return this._move(builder, step.nodes); + case 'remove': + return this._remove(builder, step.nodes, step.areReferenced); + case 'insert': + return this._insert(builder, step.nodes, step.references); + case 'deferredKeys': + return this._writeDeferredKeys(builder, step.dependencies); + case 'joinRows': + return this._insertJoinRows(builder, step.nodes); + case 'patch': + return this._patch(builder, step.nodes); + } + } + + // Group the nodes into levels that can be inserted once the levels before + // them are: Nodes wait for the nodes whose keys they store and the nodes + // that they reference. All nodes take part, so that the keys of the nodes + // that aren't inserted are resolved in order too. + _planInsertLevels() { + const nodes = this.plan.activeNodes; + const handledNodes = new Set(); + const levels = []; + + const getPendingDependencies = (node) => + (this.dependencies.get(node) || []).filter( + (dependency) => + !this.deferredDependencies.has(dependency) && + !handledNodes.has(dependency.node) && + !dependency.node.current, + ); + + // Return true if `node` waits for `blockingNode`, through other nodes too. + const isBlockedBy = (node, blockingNode, visitedNodes = new Set()) => { + if (node === blockingNode) { + return true; + } else if (visitedNodes.has(node)) { + return false; + } else { + visitedNodes.add(node); + return getPendingDependencies(node).some((dependency) => + isBlockedBy(dependency.node, blockingNode, visitedNodes), + ); + } + }; + + // Break a cycle by deferring one BelongsToOne key that points to a `#ref` + // model: Its owner is inserted without the key, which is patched in once + // the referenced model is inserted. Without `#ref`, the graph is a tree. + const breakCycle = () => { + for (const node of nodes) { + if (!handledNodes.has(node)) { + for (const dependency of getPendingDependencies(node)) { + if (this._canDefer(dependency, node) && isBlockedBy(dependency.node, node)) { + this.deferredDependencies.add(dependency); + return true; + } + } + } + } + + return false; + }; + + const isReady = (node) => !handledNodes.has(node) && getPendingDependencies(node).length === 0; + const isReadyReference = (node) => node.isReference && isReady(node); + + while (handledNodes.size < nodes.length) { + const levelNodes = nodes.filter(isReady); + + if (levelNodes.length > 0) { + const level = { nodes: levelNodes, references: [] }; + levels.push(level); + + for (const node of levelNodes) { + handledNodes.add(node); + } + + // `#ref` models aren't inserted. They're resolved right after the + // inserts of the level that completes their dependencies, so that the + // nodes that wait for them can be inserted in the next level (#311). + for ( + let references = nodes.filter(isReadyReference); + references.length > 0; + references = nodes.filter(isReadyReference) + ) { + for (const node of references) { + handledNodes.add(node); + level.references.push(node); + } + } + } else if (!breakCycle()) { + throw new Error('the object graph contains cyclic references'); + } + } + + return levels; + } + + _canDefer(dependency, node) { + const { actions } = this.plan; + const { relation } = dependency; + + return ( + relation && + hasOwnerKey(relation) && + dependency.node.isReference && + // The owner is patched by its id, which models without one don't have (#2305). + node.modelClass.getIdColumnArray().length > 0 && + // Both ends need to be inserted for the key to be patched in later. + actions.get(node) === 'insert' && + actions.get(dependency.node.referencedNode) === 'insert' && + // A `#ref{}` that reads the key might be resolved before it's patched in. + !node.propertyReferencingNodes.some((referencingNode) => + referencingNode.propertyReferences.some( + (reference) => + reference.referencedNode === node && + relation.ownerProp.props.includes(reference.refPath[0]), + ), + ) + ); + } + + // Return true if the HasMany or HasOne relate of `node` moves its row to an + // existing owner before the removals, see the steps above. `#ref` models may + // get their ids from the inserts, and a relate into the slot of a HasOne row + // waits for the removal of that row, or its move, as its key may be unique. + _isMovedFirst(node, visitedNodes = new Set()) { + const { actions } = this.plan; + visitedNodes.add(node); + + if ( + actions.get(node) === 'relate' && + !node.isReference && + node.relation.keySide === 'related' && + node.parent.existing + ) { + const slotNode = getSlotNode(node); + const moverNode = this._getSlotMover(node, slotNode); + return ( + !(slotNode && actions.get(slotNode) !== 'keep') && + // Swapped rows are related together: + (!moverNode || visitedNodes.has(moverNode) || this._isMovedFirst(moverNode, visitedNodes)) + ); + } else { + return false; + } + } + + // Find the relate that moves the current row in the slot of the + // HasOneRelation that `node` relates into to another owner. + _getSlotMover(node, slotNode = this.plan.actions.get(node) === 'relate' && getSlotNode(node)) { + const { actions } = this.plan; + const keptNode = slotNode && this.plan.keptNodeForRow(slotNode); + return keptNode && actions.get(keptNode) === 'relate' ? keptNode : null; + } + + // Split the nodes into rounds, so that a relate into the slot of a + // HasOneRelation runs after the relate that moves the row in it to another + // owner, as the foreign key may be unique. + _planSlotRounds(nodes) { + const moverNodes = new Map(nodes.map((node) => [node, this._getSlotMover(node)])); + const pendingNodes = new Set(nodes); + const rounds = []; + + while (pendingNodes.size > 0) { + const readyNodes = [...pendingNodes].filter( + (node) => !pendingNodes.has(moverNodes.get(node)), + ); + // Swapped rows can't be ordered, so we relate them together: + const round = readyNodes.length > 0 ? readyNodes : [...pendingNodes]; + rounds.push(round); + + for (const node of round) { + pendingNodes.delete(node); + } + } + + return rounds; + } + + // Return true if `node` waits for the keys or references of inserted models. + _waitsForInserts(node) { + const { actions } = this.plan; + return (this.dependencies.get(node) || []).some( + (dependency) => actions.get(dependency.node.referencedNode || dependency.node) === 'insert', + ); + } + + async _move(builder, nodes) { + const [waitingNodes, otherNodes] = partition(nodes, (node) => this.waitingMovedNodes.has(node)); + + for (const node of otherNodes) { + this._resolveDependencies(node); + } + + await this._patch(builder, otherNodes); + await promiseMap(waitingNodes, (node) => this._moveWaiting(builder, node), { + concurrency: getConcurrency(builder, waitingNodes), + }); + } + + // Move a row that waits for inserts with what is known of it before them, + // through a model of its own, as `node.obj` is patched once they're written. + async _moveWaiting(builder, node) { + const { graphOptions } = this.plan; + const { modelClass, relation } = node; + const idProps = modelClass.getIdPropertyArray(); + // Leave out the keys that its related models give it, and all references: + const waitingProps = (this.dependencies.get(node) || []).flatMap((dependency) => + dependency.reference + ? [dependency.reference.path[0]] + : dependency.relation && dependency.related !== node + ? dependency.relation.ownerProp.props + : [], + ); + resolveKeys(node); + const props = + graphOptions.shouldPatch(node) || graphOptions.shouldUpdate(node) + ? difference(this._findChanges(node).changedProps, [...idProps, ...waitingProps]) + : relation.relatedProp.props; + // Give the hooks the ids, but don't write them, and don't parse the values + // again with `fromJson()`: + const model = Object.assign(new modelClass(), pick(node.obj, [...idProps, ...props])); + model.$omitFromDatabaseJson(idProps); + const count = await modelClass + .query() + .childQueryOf(builder) + .patch(model) + .findById(node.obj.$id()); + + if (!count) { + await throwForMissingRows(builder, modelClass, [node]); + } + } + + _remove(builder, nodes, areReferenced) { + const { actions } = this.plan; + const queries = []; + + for (const [relation, relationNodes] of groupBy(nodes, (node) => node.relation)) { + // Join rows are removed per owner, as one query would remove the rows of + // all owners with all related rows. + const groups = + relation.keySide === 'join' + ? [...groupBy(relationNodes, (node) => node.parent).values()] + : [relationNodes]; + + for (const groupNodes of groups) { + // Each row binds its id, and the owner keys of its owner at most. + const maxRows = getMaxRows( + builder.unsafeKnex(), + relation.relatedModelClass.getIdColumnArray().length + relation.ownerProp.size, + ); + const actionGroups = [...groupBy(groupNodes, (node) => actions.get(node))].flatMap( + ([action, actionNodes]) => chunk(actionNodes, maxRows).map((nodes) => [action, nodes]), + ); + + for (const [action, actionNodes] of actionGroups) { + // Rows that reference their owners are found through the relation. + // BelongsToOne rows aren't referenced anymore once their owners are + // patched. + const owners = [...new Set(actionNodes.map((node) => node.parent.obj))]; + const query = ( + areReferenced + ? relation.ownerModelClass.relatedQuery(relation.name).for(owners) + : relation.relatedModelClass.query() + ) + .childQueryOf(builder) + .findByIds(actionNodes.map((node) => node.obj.$id())); + + queries.push(action === 'unrelate' ? query.unrelate() : query.delete()); + } + } + } + + return promiseMap(queries, (query) => query.execute(), { + concurrency: getConcurrency(builder, nodes), + }); + } + + async _insert(builder, nodes, references) { + const batchSize = getBatchSize(builder); + const batches = [...groupBy(nodes, (node) => node.modelClass).values()].flatMap((nodes) => + chunk(nodes, batchSize), + ); + + await promiseMap(batches, (batch) => this._insertBatch(builder, batch), { + concurrency: getConcurrency(builder, nodes), + }); + + for (const node of references) { + this._resolveDependencies(node); + } + } + + // Copy the keys and `#ref{}` values that `node` waits for into its model. + _resolveDependencies(node) { + for (const dependency of this.dependencies.get(node) || []) { + if (!this.deferredDependencies.has(dependency)) { + if (dependency.relation) { + resolveKeys(dependency.related); + } else if (dependency.reference) { + resolvePropertyReference(dependency.dependent, dependency.reference); + } + } + } + } + + async _insertBatch(builder, nodes) { + const { actions } = this.plan; + + for (const node of nodes) { + this._resolveDependencies(node); + + // Join table extras are written to the join rows. + if (node.relation?.keySide === 'join') { + node.relation.omitExtraProps([node.obj]); + } + + if (actions.get(node) === 'insert') { + this._setKeyArrays(node); + } + } + + const insertNodes = nodes.filter((node) => actions.get(node) === 'insert'); + + if (insertNodes.length > 0) { + const [{ modelClass }] = insertNodes; + + for (const node of insertNodes) { + node.obj.$validate(null, { dataPath: node.dataPathKey }); + } + + await Promise.all( + insertNodes.map( + (node) => + node.parent && node.relation.beforeInsert(node.obj, builder.context(), node.parent.obj), + ), + ); + + for (const batch of createBatches( + builder, + insertNodes.map((node) => node.obj), + )) { + await modelClass + .query() + .insert(batch) + .childQueryOf(builder) + .copyFrom(builder, isReturningAll) + .execute(); + } + + await fetchGeneratedKeys(builder, modelClass, insertNodes); + } + } + + _writeDeferredKeys(builder, dependencies) { + const owners = groupBy(dependencies, (dependency) => dependency.dependent); + + return promiseMap( + [...owners], + ([owner, dependencies]) => { + const { modelClass } = owner; + const idProp = modelClass.getIdRelationProperty(); + const patch = {}; + + for (const { relation, related } of dependencies) { + resolveKeys(related); + + relation.ownerProp.forEach((i) => { + relation.ownerProp.patch(patch, i, relation.ownerProp.getProp(owner.obj, i)); + }); + } + + return modelClass + .query() + .childQueryOf(builder) + .patch(patch) + .whereInComposite(idProp.refs(builder), [idProp.getProps(owner.obj)]); + }, + { concurrency: getConcurrency(builder, [...owners.keys()]) }, + ); + } + + async _insertJoinRows(builder, nodes) { + const knex = builder.unsafeKnex(); + const inserts = []; + + for (const [joinModelClass, joinNodes] of groupBy(nodes, (node) => + node.relation.getJoinModelClass(knex), + )) { + const objs = await Promise.all( + joinNodes.map(async (node) => { + const { relation, parent } = node; + resolveReference(node); + + const obj = joinModelClass.fromJson( + relation.createJoinModel(relation.ownerProp.getProps(parent.obj), node.obj), + ); + + await relation.joinTableBeforeInsert(obj, builder.context()); + return obj; + }), + ); + + // Join rows that have their ids can be inserted in batches everywhere. + const batchSize = objs.every((obj) => obj.$hasId()) + ? MAX_INSERT_BATCH_SIZE + : getBatchSize(builder); + + for (const objsOfBatch of chunk(objs, batchSize)) { + for (const batch of createBatches(builder, objsOfBatch)) { + inserts.push(() => joinModelClass.query().childQueryOf(builder).insert(batch)); + } + } + } + + return promiseMap(inserts, (insert) => insert(), { + concurrency: getConcurrency(builder, nodes), + }); + } + + async _patch(builder, nodes) { + const patches = nodes.map((node) => this._preparePatch(builder, node)).filter(Boolean); + const rowKeyCounts = groupBy(patches, ({ node }) => node.rowKey); + const [bulkPatches, rowPatches] = partition( + patches, + (patch) => rowKeyCounts.get(patch.node.rowKey).length === 1 && isBulkPatch(builder, patch), + ); + + await promiseMap(rowPatches, (patch) => this._writePatch(builder, patch), { + concurrency: getConcurrency(builder, nodes), + }); + await this._writeBulkPatches(builder, bulkPatches); + } + + // Find out what to write for `node`, validate it, and return the patch, or + // nothing if there is nothing to write. + _preparePatch(builder, node) { + const { graphOptions } = this.plan; + const shouldPatch = graphOptions.shouldPatch(node); + const shouldUpdate = graphOptions.shouldUpdate(node); + // Inserts, relates and removals of BelongsToOne rows change the keys of + // their owners, and relates into other relations the keys of the related + // rows, which are written even with `noUpdate` (#53): + const keyProps = [ + ...this._setKeysOfRelatedRows(node), + ...this._clearKeysOfRemovedRows(node), + ...this._setKeyArrays(node), + ...(this.plan.actions.get(node) === 'relate' && node.relation.keySide === 'related' + ? node.relation.relatedProp.props + : []), + ]; + // Changing the id never makes sense with `upsertGraph()`: + const idProps = node.modelClass.getIdPropertyArray(); + + // Leave out the props that don't change, and return true if any do. + const handleUpdate = () => { + const { changedProps, unchangedProps } = this._findChanges(node); + const propsToUpdate = difference( + shouldPatch || shouldUpdate ? changedProps : keyProps, + idProps, + ); + + if (propsToUpdate.length > 0) { + node.obj.$omitFromDatabaseJson( + difference([...changedProps, ...unchangedProps], propsToUpdate), + ); + return true; + } else { + return false; + } + }; + + // Without a `beforeUpdate` or `$beforeUpdate` hook, we know right away if + // there is anything to update. Otherwise, the hook may still change the + // model, and we check after it runs, but only if the model has changes + // that aren't ids, as relates are no updates (#2233, #2605). + const hasBeforeUpdate = hasHook(node.obj, 'beforeUpdate'); + + if ( + hasBeforeUpdate + ? !this._findChanges(node).changedProps.some((prop) => !idProps.includes(prop)) + : !handleUpdate() + ) { + return null; + } + + node.obj.$validate(null, { + dataPath: node.dataPathKey, + patch: shouldPatch || !shouldUpdate, + }); + + // Rows that their owners relate by their keys can't be found through the + // relation until the owners are patched, so they're patched directly. + const { relation } = node; + const isPatchedThroughRelation = + !!node.parent && + (relation.keySide === 'join' || + (relation.keySide === 'owner' && this.plan.actions.get(node) !== 'relate')); + + return { node, shouldPatch, hasBeforeUpdate, handleUpdate, isPatchedThroughRelation }; + } + + _writePatch( + builder, + { node, shouldPatch, hasBeforeUpdate, handleUpdate, isPatchedThroughRelation }, + ) { + const query = ( + isPatchedThroughRelation + ? // Find the row through the keys of the owner's current row, as the + // graph may leave them out of the owner: + (node.parent.existing || node.parent).obj + .$relatedQuery(node.relation.name) + .findById(node.obj.$id()) + : (node.current?.obj || node.obj).$query() + ) + .childQueryOf(builder, { fork: true, isInternalQuery: true }) + // The current graph was fetched through the relations, so the patched + // models are known to be related to their parents. + .internalOptions({ skipRelationFilter: true }) + .copyFrom(builder, isReturningAll); + + if (hasBeforeUpdate) { + query.internalContext().runBefore.push((result, query) => { + if (!handleUpdate()) { + query.internalOptions().returnImmediatelyValue = null; + } + + return result; + }); + } + + return (shouldPatch ? query.patch(node.obj) : query.update(node.obj)) + .execute() + .then(async (result) => { + if (isObject(result) && result.$isObjectionModel) { + // Handle `returning('*')`. + node.obj.$set(result); + } else if (result === 0 || result === undefined) { + // A hook that leaves nothing to update returns null instead. Rows + // patched through relations are gone from there if not found. + if (isPatchedThroughRelation) { + throw createPatchNotFoundError(builder, node); + } else { + await throwForMissingRows(builder, node.modelClass, [node]); + } + } + + return result; + }); + } + + // Write the patches of the rows of each table in one query, with a `CASE` + // per column that picks the value of each row by its id: + // `SET col = CASE WHEN id = 1 THEN 'a' WHEN id = 2 THEN 'b' ELSE col END`. + // The column in `ELSE` also gives Postgres the type of the values. + _writeBulkPatches(builder, patches) { + const batches = [...groupBy(patches, ({ node }) => node.modelClass)].flatMap( + ([modelClass, tablePatches]) => + chunkByParameters(builder, tablePatches, modelClass.getIdColumnArray()).map((batch) => ({ + modelClass, + batch, + })), + ); + + return promiseMap( + batches, + async ({ modelClass, batch }) => { + const idColumns = modelClass.getIdColumnArray(); + const columns = [...new Set(batch.flatMap(({ json }) => Object.keys(json)))]; + const condition = idColumns.map(() => '?? = ?').join(' AND '); + const update = {}; + + for (const column of columns) { + const rows = batch.filter(({ json }) => column in json); + const sql = `CASE ${rows.map(() => `WHEN ${condition} THEN ?`).join(' ')} ELSE ?? END`; + const bindings = rows.flatMap(({ node, json }) => [ + ...idColumns.flatMap((idColumn, i) => [idColumn, asArray(node.obj.$id())[i]]), + json[column], + ]); + + update[modelClass.columnNameToPropertyName(column)] = raw(sql, [...bindings, column]); + } + + // The rows were validated one by one already. + const nodes = batch.map(({ node }) => node); + const count = await modelClass + .query() + .childQueryOf(builder, { fork: true, isInternalQuery: true }) + .patch(modelClass.fromJson(update, { patch: true, skipValidation: true })) + .findByIds(nodes.map((node) => node.obj.$id())); + + if (count !== nodes.length) { + await throwForMissingRows(builder, modelClass, nodes); + } + }, + { + concurrency: getConcurrency( + builder, + patches.map(({ node }) => node), + ), + }, + ); + } + + // Change the arrays of keys that `node` holds for the relations that the graph + // gives it: Remove the keys of the current related rows that it deletes or + // unrelates, and add the keys of the related models that it writes. Other + // keys stay, e.g. of rows that the relation's filter leaves out. Relations + // that the graph leaves out keep their arrays (#1202). Return the props that + // change. + _setKeyArrays(node) { + const { actions, graphOptions } = this.plan; + const { existing } = node; + const props = []; + + for (const name of node.modelClass.getRelationNames()) { + const relation = node.obj[name] !== undefined && node.modelClass.getRelation(name); + + if (relation && hasOwnerKeyArray(relation)) { + const { ownerProp, relatedProp } = relation; + const getKey = (related) => relatedProp.getProp(related.obj, 0); + const keys = relation.getKeys(existing ? existing.obj : node.obj); + const removed = (existing?.activeChildren || []).filter( + (child) => + child.relation.name === name && ['delete', 'unrelate'].includes(actions.get(child)), + ); + const added = node.activeChildren.filter( + (child) => + child.relation.name === name && + !child.toBeUnrelatedOrDeleted && + actions.get(child) !== 'ignore', + ); + + for (const related of added) { + resolveReference(related); + } + + const newKeys = relation.changeKeys(keys, { + add: added.map(getKey), + remove: removed.map(getKey), + }); + + if (!existing || graphOptions.shouldUpdate(node)) { + // Inserts and updates write the whole row: + ownerProp.setProp(node.obj, 0, newKeys); + props.push(ownerProp.props[0]); + } else if (!jsonEquals(keys, newKeys)) { + const patch = relation.createKeysPatch(existing.obj, newKeys); + Object.assign(node.obj, patch); + props.push(...Object.keys(patch)); + } + } + } + + return props; + } + + // Point the keys of `node` to its BelongsToOne rows that are inserted or + // related, and return the props that change. + _setKeysOfRelatedRows(node) { + const { actions } = this.plan; + const props = []; + + for (const child of node.activeChildren) { + const { relation } = child; + + if ( + hasOwnerKey(relation) && + relation.relatedProp.hasProps(child.obj) && + // Ignored models don't replace the current one, and models marked with + // `#unrelate` or `#delete` are handled below. + actions.get(child) !== 'ignore' + ) { + relation.ownerProp.forEach((i) => { + const currentValue = node.existing && relation.ownerProp.getProp(node.existing.obj, i); + const value = relation.relatedProp.getProp(child.obj, i); + + if (currentValue != value) { + relation.ownerProp.setProp(node.obj, i, value); + props.push(relation.ownerProp.props[i]); + } + }); + } + } + + return props; + } + + // Clear the keys of `node` that point to BelongsToOne rows that are removed, + // and return the props that change. + _clearKeysOfRemovedRows(node) { + const { actions } = this.plan; + const props = []; + + for (const currentChild of node.existing?.activeChildren || []) { + const { relation } = currentChild; + + if ( + hasOwnerKey(relation) && + (node.obj[relation.name] === null || currentChild.node?.toBeUnrelatedOrDeleted) && + actions.get(currentChild) !== 'keep' + ) { + relation.ownerProp.forEach((i) => { + if (relation.ownerProp.getProp(node.existing.obj, i) != null) { + relation.ownerProp.setProp(node.obj, i, null); + props.push(relation.ownerProp.props[i]); + } + }); + } + } + + return props; + } + + _findChanges(node) { + const { obj, relation } = node; + const currentObj = node.current?.obj || {}; + const relationNames = node.modelClass.getRelationNames(); + // The extras of a new join row are written with it, and patching them + // would overwrite other join rows of the same models (Vincit#1268): + const extraProps = + relation?.keySide === 'join' && !node.current + ? relation.joinTableExtras.map((extra) => extra.aliasProp) + : []; + const changedProps = []; + const unchangedProps = []; + + for (const prop of Object.keys(obj)) { + if (!isInternalProp(prop) && !relationNames.includes(prop)) { + const value = obj[prop]; + const currentValue = currentObj[prop]; + + // We can't know if a prop that the current model doesn't have changes, + // so we assume it does (#732). + if ( + !extraProps.includes(prop) && + (currentValue === undefined || !nonStrictEquals(currentValue, value)) + ) { + changedProps.push(prop); + } else { + unchangedProps.push(prop); + } + } + } + + // Query properties may change the values too. + if (obj.$$queryProps) { + changedProps.push(...Object.keys(obj.$$queryProps)); + } + + return { changedProps, unchangedProps }; + } +} + +// Collect what each node waits for: The node on the other side of its relation +// if the node stores its keys, and the nodes it references with `#ref` and +// `#ref{}`. +function createDependencies(plan) { + const { actions } = plan; + const dependencies = new Map(); + + const addDependency = (dependent, dependency) => { + const nodeDependencies = dependencies.get(dependent); + const entry = { dependent, ...dependency }; + + if (nodeDependencies) { + nodeDependencies.push(entry); + } else { + dependencies.set(dependent, [entry]); + } + }; + + for (const node of plan.activeNodes) { + const { parent, relation } = node; + + // Models marked with `#unrelate` or `#delete` are never inserted or related, + // so their keys aren't copied. Ignored models don't replace the current one, + // so their keys aren't copied to their owners either. + if (parent && !node.toBeUnrelatedOrDeleted) { + if (relation.keySide === 'related') { + addDependency(node, { node: parent, relation, related: node }); + } else if (relation.keySide === 'owner' && actions.get(node) !== 'ignore') { + addDependency(parent, { node, relation, related: node }); + } + } + } + + for (const node of plan.activeNodes) { + if (node.referencedNode && !node.referencedNode.isPruned) { + addDependency(node, { node: node.referencedNode }); + } + + for (const reference of node.propertyReferences) { + if (!reference.referencedNode.isPruned) { + addDependency(node, { node: reference.referencedNode, reference }); + } + } + } + + return dependencies; +} + +// Fetch the keys that the relations of the inserted nodes need, but that the +// database generated and the insert didn't return, like a uuid that isn't the +// id (Vincit#853). +async function fetchGeneratedKeys(builder, modelClass, nodes) { + const keyProps = (node) => [ + ...(node.relation && node.relation.keySide !== 'related' ? [node.relation.relatedProp] : []), + ...node.activeChildren + .filter((child) => child.relation.keySide !== 'owner') + .map((child) => child.relation.ownerProp), + ]; + const missingNodes = nodes.filter((node) => + keyProps(node).some((prop) => prop.getProps(node.obj).some((value) => value === undefined)), + ); + + if (missingNodes.length > 0 && missingNodes.every((node) => node.obj.$hasId())) { + const columns = [ + ...new Set(missingNodes.flatMap((node) => keyProps(node)).flatMap((prop) => prop.cols)), + ]; + const rows = await modelClass + .query() + .childQueryOf(builder, { fork: true, isInternalQuery: true }) + .findByIds(missingNodes.map((node) => node.obj.$id())) + .select([...modelClass.getIdColumnArray(), ...columns]); + const rowsByKey = new Map(rows.map((row) => [row.$idKey(), row])); + + for (const node of missingNodes) { + const row = rowsByKey.get(node.obj.$idKey()); + + for (const prop of keyProps(node)) { + prop.forEach((i) => { + if (prop.getProp(node.obj, i) === undefined) { + prop.setProp(node.obj, i, prop.getProp(row, i)); + } + }); + } + } + } +} + +// Copy the keys of the relation of `node` to the side that stores them, +// whichever node was inserted first (#311). +function resolveKeys(node) { + const { relation, parent } = node; + + // The arrays of keys of owners are written by `_setKeyArrays()`. + if (relation.keySide === 'related' || hasOwnerKey(relation)) { + const { ownerProp, relatedProp } = relation; + const [source, sourceProp, target, targetProp] = + relation.keySide === 'owner' + ? [node, relatedProp, parent, ownerProp] + : [parent, ownerProp, node, relatedProp]; + + resolveReference(source); + resolveReference(target); + + targetProp.forEach((i) => { + targetProp.setProp(target.obj, i, sourceProp.getProp(source.obj, i)); + }); + } +} + +// Give a `#ref` model the props of the model it refers to. +function resolveReference(node) { + const { referencedNode } = node; + + if (referencedNode) { + for (const prop of Object.keys(referencedNode.obj)) { + if (!Object.hasOwn(node.obj, prop)) { + node.obj[prop] = referencedNode.obj[prop]; + } + } + } +} + +// Replace a `#ref{}` in a prop of `node` with the value it refers to. A whole +// value is replaced even if the relation set the prop already (Vincit#1378). +function resolvePropertyReference(node, { path, match, refPath, isWholeValue, referencedNode }) { + const value = get(referencedNode.obj, refPath); + set(node.obj, path, isWholeValue ? value : get(node.obj, path).replace(match, value)); +} + +// Fill in the keys of `node` that the graph leaves out from its row: the keys +// of its relations from wherever the row was found, and the key of its parent +// relation only from its row in the same place. +function copyKeysFromExisting(node) { + const { existing, current } = node; + + if (existing) { + const copies = [ + ...(node.parent && current ? [[node.relation.relatedProp, current]] : []), + ...node.activeChildren + .filter((child) => !hasOwnerKeyArray(child.relation)) + .map((child) => [child.relation.ownerProp, existing]), + ]; + + for (const [prop, source] of copies) { + prop.forEach((i) => { + if (prop.getProp(node.obj, i) === undefined) { + prop.setProp(node.obj, i, prop.getProp(source.obj, i)); + } + }); + } + } +} + +// Patches are written in bulk if nothing needs the single model: no update +// hooks, no `returning('*')`, no relation to patch through, and plain values. +function isBulkPatch(builder, patch) { + const { node, shouldPatch, hasBeforeUpdate, isPatchedThroughRelation } = patch; + + if ( + shouldPatch && + !hasBeforeUpdate && + !hasHook(node.obj, 'afterUpdate') && + !isPatchedThroughRelation && + !node.obj.$$queryProps && + !builder.has(isReturningAll) + ) { + const declaredColumns = getDeclaredColumns(node.modelClass); + patch.json = node.obj.$toDatabaseJson(builder); + + return Object.entries(patch.json).every( + ([key, value]) => + isPlainValue(value) && + // Field expressions like `'meta:a'` write into json columns. + !isFieldExpressionKey(key, declaredColumns), + ); + } else { + return false; + } +} + +function hasHook(obj, name) { + const Model = getModel(); + return obj.constructor[name] !== Model[name] || obj[`$${name}`] !== Model.prototype[`$${name}`]; +} + +function isPlainValue(value) { + return ( + value === null || + value instanceof Date || + ['string', 'number', 'boolean', 'bigint'].includes(typeof value) + ); +} + +// Each row binds its ids and value per column, its ids in the `WHERE` clause, +// and each column its name in `ELSE`. +function chunkByParameters(builder, patches, idColumns) { + const maxColumns = patches.reduce((max, { json }) => Math.max(max, Object.keys(json).length), 0); + const parametersPerRow = (idColumns.length + 1) * maxColumns + idColumns.length + maxColumns; + return chunk( + patches, + Math.min(MAX_BULK_PATCH_SIZE, getMaxRows(builder.unsafeKnex(), parametersPerRow)), + ); +} + +// Throw for the first row of the patched nodes that doesn't exist anymore, e.g. +// deleted through `ON DELETE CASCADE` by an earlier write of the graph. Rows that +// still exist count as patched, even if a trigger skipped them, or MySQL didn't +// count them as changed. +async function throwForMissingRows(builder, modelClass, nodes) { + const existingRows = await modelClass + .query() + .childQueryOf(builder, { fork: true, isInternalQuery: true }) + .findByIds(nodes.map((node) => node.obj.$id())) + .select(modelClass.getIdColumnArray()); + const existingKeys = new Set(existingRows.map((row) => row.$idKey())); + const node = nodes.find((node) => !existingKeys.has(node.obj.$idKey())); + + if (node) { + throw createPatchNotFoundError(builder, node); + } +} + +function createPatchNotFoundError(builder, node) { + return node.modelClass.createNotFoundError(builder.context(), { + message: `model (id=${node.obj.$id()}) was not found when updating it. An earlier write of the graph or a concurrent query may have deleted it`, + dataPath: node.dataPath, + }); +} + +// Find the current row in the slot of the HasOneRelation that `node` relates +// into, or null if it relates into another relation or an empty slot. +function getSlotNode(node) { + const { relation } = node; + return relation.keySide === 'related' && relation.isOneToOne() + ? (node.parent.existing?.activeChildren.find( + (child) => child.relation.name === relation.name, + ) ?? null) + : null; +} + +function hasOwnerKey(relation) { + return relation.keySide === 'owner' && relation.isOneToOne(); +} + +function hasOwnerKeyArray(relation) { + return relation.keySide === 'owner' && !relation.isOneToOne(); +} + +// Split the models into the batches of rows that can be inserted with one query +// each: Rows without columns can't be inserted in batches (#1694), the rows of +// a batch need the same columns where the database doesn't fill in defaults, +// and the batches need to stay under the number of values a query can bind. +function createBatches(builder, objs) { + const knex = builder.unsafeKnex(); + const columnsByObj = new Map(objs.map((obj) => [obj, Object.keys(obj.$toDatabaseJson(builder))])); + // `beforeInsert` hooks may still change the columns, so their rows can only + // be grouped where the database fills in defaults: + const getColumnsKey = (obj) => + hasBatchInsertDefaults(knex) + ? '' + : hasHook(obj, 'beforeInsert') + ? obj + : [...columnsByObj.get(obj)].sort().join(); + const batches = []; + + for (const group of groupBy(objs, getColumnsKey).values()) { + const [rows, emptyRows] = partition(group, (obj) => columnsByObj.get(obj).length > 0); + const columnCount = Math.max(0, ...rows.map((obj) => columnsByObj.get(obj).length)); + batches.push(...chunk(rows, getMaxRows(knex, columnCount)), ...emptyRows.map((obj) => [obj])); + } + + return batches; +} + +function getBatchSize(builder) { + return hasBatchInsert(builder.unsafeKnex()) ? MAX_INSERT_BATCH_SIZE : 1; +} + +function getConcurrency(builder, nodes) { + const knex = builder.unsafeKnex(); + return nodes.reduce( + (concurrency, node) => Math.min(concurrency, node.modelClass.getConcurrency(knex)), + MAX_CONCURRENCY, + ); +} + +function isReturningAll(operation) { + return operation.name === 'returning' && operation.args.includes('*'); +} + +function pick(obj, props) { + return Object.fromEntries( + props.filter((prop) => Object.hasOwn(obj, prop)).map((prop) => [prop, obj[prop]]), + ); +} + +function partition(items, predicate) { + return [items.filter(predicate), items.filter((item) => !predicate(item))]; +} + +function nonStrictEquals(value1, value2) { + // Only use loose equality for non-JSON values, so that for example `false` + // and `[]` are not considered equal (#1874). + return ( + (!isJsonValue(value1) && !isJsonValue(value2) && value1 == value2) || jsonEquals(value1, value2) + ); +} + +function isJsonValue(value) { + return Array.isArray(value) || isPlainObject(value); +} diff --git a/lib/queryBuilder/graph/delete/GraphDelete.js b/lib/queryBuilder/graph/delete/GraphDelete.js deleted file mode 100644 index ab9c67b22..000000000 --- a/lib/queryBuilder/graph/delete/GraphDelete.js +++ /dev/null @@ -1,26 +0,0 @@ -import { GraphOperation } from '../GraphOperation.js'; -import { GraphDeleteAction } from './GraphDeleteAction.js'; - -export class GraphDelete extends GraphOperation { - // The rows of BelongsToOneRelations are referenced by their owners, and are - // deleted after `GraphPatch` points the owners to new rows or to null, like - // `unrelate()`. All other rows reference their owners, and are deleted - // before the inserts, which may need their unique keys. - constructor(graphData, { belongsToOne }) { - super(graphData); - this.belongsToOne = belongsToOne; - } - - createActions() { - return [ - new GraphDeleteAction(this.graphData, { - nodes: this.currentGraph.nodes.filter( - (currentNode) => - this.graphData.actions.get(currentNode) !== 'keep' && - !!currentNode.parentEdge.relation.isObjectionBelongsToOneRelation === this.belongsToOne, - ), - areReferenced: !this.belongsToOne, - }), - ]; - } -} diff --git a/lib/queryBuilder/graph/delete/GraphDeleteAction.js b/lib/queryBuilder/graph/delete/GraphDeleteAction.js deleted file mode 100644 index e233ccd63..000000000 --- a/lib/queryBuilder/graph/delete/GraphDeleteAction.js +++ /dev/null @@ -1,76 +0,0 @@ -import { GraphAction } from '../GraphAction.js'; -import { groupBy } from '../../../utils/objectUtils.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -export class GraphDeleteAction extends GraphAction { - constructor(graphData, { nodes, areReferenced = true }) { - super(graphData); - // Nodes to delete. - this.nodes = nodes; - // Whether the parents still reference the nodes. Rows of - // BelongsToOneRelations can be deleted after their owners were patched to - // reference new rows or null, so they can't be found through the relation. - this.areReferenced = areReferenced; - } - - run(builder) { - const nodesTodelete = this._filterOutBelongsToOneRelationUnrelates(this.nodes); - const builders = this._createDeleteBuilders(builder, nodesTodelete); - - return promiseUtils.map(builders, (builder) => builder.execute(), { - concurrency: this._getConcurrency(builder, nodesTodelete), - }); - } - - _filterOutBelongsToOneRelationUnrelates(nodes) { - // `BelongsToOneRelation` unrelate is handled by `GraphPatch` because - // unrelating a `BelongsToOneRelation` is just a matter of updating - // one field of the parent node. - return nodes.filter((node) => { - return !( - this.graphData.actions.get(node) === 'unrelate' && - node.parentEdge.relation.isObjectionBelongsToOneRelation - ); - }); - } - - _createDeleteBuilders(parentBuilder, nodesTodelete) { - const nodesByRelation = groupBy(nodesTodelete, getRelation); - const builders = []; - - nodesByRelation.forEach((nodes, relation) => { - const nodesByParent = groupBy(nodes, getParent); - - nodesByParent.forEach((nodes, parentNode) => { - // Nodes of the same relation and parent can still require different - // operations when they are marked with `#unrelate` or `#delete`. - const nodesByAction = groupBy(nodes, (node) => this.graphData.actions.get(node)); - - nodesByAction.forEach((nodes, action) => { - const builder = ( - this.areReferenced - ? parentNode.obj.$relatedQuery(relation.name) - : relation.relatedModelClass.query() - ).childQueryOf(parentBuilder); - - if (!this.areReferenced || !relation.isObjectionBelongsToOneRelation) { - // This is useless in case of a referenced BelongsToOneRelation. - builder.findByIds(nodes.map((node) => node.obj.$id())); - } - - builders.push(action === 'unrelate' ? builder.unrelate() : builder.delete()); - }); - }); - }); - - return builders; - } -} - -function getRelation(node) { - return node.parentEdge.relation; -} - -function getParent(node) { - return node.parentNode; -} diff --git a/lib/queryBuilder/graph/insert/DeferredRelationGraphInsertAction.js b/lib/queryBuilder/graph/insert/DeferredRelationGraphInsertAction.js deleted file mode 100644 index 20c8b9b39..000000000 --- a/lib/queryBuilder/graph/insert/DeferredRelationGraphInsertAction.js +++ /dev/null @@ -1,52 +0,0 @@ -import { GraphAction } from '../GraphAction.js'; -import { groupBy } from '../../../utils/objectUtils.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -/** - * Patches in the foreign keys of `BelongsToOneRelation` edges that `GraphInsert` - * had to defer to break a dependency cycle. The owner nodes of these edges were - * inserted without the foreign keys, which are copied from the related nodes - * and patched in with one query per owner. - */ -export class DeferredRelationGraphInsertAction extends GraphAction { - constructor(graphData, { edges }) { - super(graphData); - - // `BelongsToOneRelation` edges whose owner node needs to be patched. - this.edges = edges; - } - - run(builder) { - const edgesByOwner = groupBy(this.edges, (edge) => edge.ownerNode); - const owners = [...edgesByOwner.keys()]; - const concurrency = this._getConcurrency(builder, owners); - - return promiseUtils.map( - owners, - (owner) => this._patchOwner(builder, owner, edgesByOwner.get(owner)), - { concurrency }, - ); - } - - _patchOwner(builder, ownerNode, edges) { - const { modelClass } = ownerNode; - const ownerIdProp = modelClass.getIdRelationProperty(); - const patch = {}; - - for (const { relation, relatedNode } of edges) { - const { ownerProp, relatedProp } = relation; - - ownerProp.forEach((i) => { - const value = relatedProp.getProp(relatedNode.obj, i); - ownerProp.setProp(ownerNode.obj, i, value); - ownerProp.patch(patch, i, value); - }); - } - - return modelClass - .query() - .childQueryOf(builder) - .patch(patch) - .whereInComposite(ownerIdProp.refs(builder), [ownerIdProp.getProps(ownerNode.obj)]); - } -} diff --git a/lib/queryBuilder/graph/insert/GraphInsert.js b/lib/queryBuilder/graph/insert/GraphInsert.js deleted file mode 100644 index 06f5fd8af..000000000 --- a/lib/queryBuilder/graph/insert/GraphInsert.js +++ /dev/null @@ -1,249 +0,0 @@ -import { JoinRowGraphInsertAction } from './JoinRowGraphInsertAction.js'; -import { DeferredRelationGraphInsertAction } from './DeferredRelationGraphInsertAction.js'; -import { GraphInsertAction } from './GraphInsertAction.js'; -import { GraphOperation } from '../GraphOperation.js'; -import { ModelGraphEdge } from '../../../model/graph/ModelGraphEdge.js'; - -export class GraphInsert extends GraphOperation { - constructor(...args) { - super(...args); - this.dependencies = this._createDependencyMap(); - this.deferredEdges = new Set(); - } - - createActions() { - return [ - ...this._createNormalActions(), - ...this._createDeferredRelationActions(), - ...this._createJoinRowActions(), - ]; - } - - _createDependencyMap() { - const dependencies = new Map(); - - for (const edge of this.graph.edges) { - if (edge.type == ModelGraphEdge.Type.Relation) { - this._createRelationDependency(edge, dependencies); - } else { - this._createReferenceDependency(edge, dependencies); - } - } - - return dependencies; - } - - _createRelationDependency(edge, dependencies) { - if (edge.relatedNode.toBeUnrelatedOrDeleted) { - // Models marked with `#unrelate` or `#delete` are never inserted or - // related, so their keys must not be copied to the owner (or vice versa). - return; - } - - if (edge.relation.isObjectionHasManyRelation) { - // In case of HasManyRelation the related node depends on the owner node - // because the related node has the foreign key. - this._addDependency(edge.relatedNode, edge, dependencies); - } else if ( - (edge.relation.isObjectionBelongsToOneRelation || - edge.relation.isObjectionBelongsToManyRelation) && - // Ignored models don't replace the current one, so their keys must not - // be copied to the owner either. - this.graphData.actions.get(edge.relatedNode) !== 'ignore' - ) { - // In case of BelongsToOneRelation and BelongsToManyRelation the owner - // node depends on the related node because the owner node has the keys. - this._addDependency(edge.ownerNode, edge, dependencies); - } - } - - _createReferenceDependency(edge, dependencies) { - this._addDependency(edge.ownerNode, edge, dependencies); - } - - _addDependency(node, edge, dependencies) { - let edges = dependencies.get(node); - - if (!edges) { - edges = []; - dependencies.set(node, edges); - } - - edges.push(edge); - } - - _createNormalActions() { - const handledNodes = new Set(); - const actions = []; - - while (true) { - // At this point, don't care if the nodes have already been inserted before - // given to this class. `GraphInsertAction` will test that and only insert - // new ones. We need to pass all nodes to `GraphInsertActions` so that we - // can resolve all dependencies. - const nodesToInsert = this.graph.nodes.filter((node) => { - return ( - !node.isReference && !handledNodes.has(node) && !this._hasDependencies(node, handledNodes) - ); - }); - - // `#ref` nodes aren't inserted, so they can be resolved in the same - // action as the nodes they depend on, once these are inserted. - const insertedNodes = new Set([...handledNodes, ...nodesToInsert]); - const refNodes = this.graph.nodes.filter((node) => { - return ( - node.isReference && !handledNodes.has(node) && !this._hasDependencies(node, insertedNodes) - ); - }); - - const nodes = [...nodesToInsert, ...refNodes]; - - if (nodes.length === 0) { - if (handledNodes.size !== this.graph.nodes.length && this._breakCycle(handledNodes)) { - continue; - } - - break; - } - - actions.push( - new GraphInsertAction(this.graphData, { - nodes: nodesToInsert, - refNodes, - dependencies: this.dependencies, - deferredEdges: this.deferredEdges, - }), - ); - - for (const node of nodes) { - handledNodes.add(node); - } - } - - if (handledNodes.size !== this.graph.nodes.length) { - throw new Error('the object graph contains cyclic references'); - } - - return actions; - } - - // Tries to break a dependency cycle by deferring one `BelongsToOneRelation` - // edge that points to a `#ref` node. The owner node is then inserted without - // the foreign key, which is patched in by `DeferredRelationGraphInsertAction` - // once the referenced node has been inserted. Cycles can only exist through - // `#ref` nodes, since the graph is a tree otherwise. - _breakCycle(handledNodes) { - for (const node of this.graph.nodes) { - if (handledNodes.has(node)) { - continue; - } - - for (const edge of this._pendingDependencyEdges(node, handledNodes)) { - if ( - this._canDeferEdge(edge, node) && - this._isBlockedBy(edge.relatedNode, node, handledNodes, new Set()) - ) { - this.deferredEdges.add(edge); - return true; - } - } - } - - return false; - } - - _canDeferEdge(edge, node) { - return ( - edge.type === ModelGraphEdge.Type.Relation && - edge.relation.isObjectionBelongsToOneRelation && - edge.isOwnerNode(node) && - edge.relatedNode.isReference && - // The owner is patched by its id, which models without one don't have (#2305). - node.modelClass.getIdColumnArray().length > 0 && - // Both ends need to be inserted for the foreign key to be patched in later. - this.graphData.actions.get(node) === 'insert' && - this.graphData.actions.get(edge.relatedNode.referencedNode) === 'insert' && - !this._isReadByPropertyReference(node, edge.relation.ownerProp) - ); - } - - // Returns true if a `#ref{}` property reference reads one of the `props` of - // `node`. Such references may be resolved before a deferred prop is set, - // which would resolve them to `undefined`. - _isReadByPropertyReference(node, props) { - return node.refEdges.some((edge) => { - return ( - edge.refType === ModelGraphEdge.ReferenceType.Property && - edge.isRelatedNode(node) && - props.props.includes(edge.refRelatedDataPath[0]) - ); - }); - } - - // Returns true if `node` (transitively) waits for `blockingNode` to be handled. - _isBlockedBy(node, blockingNode, handledNodes, visitedNodes) { - if (node === blockingNode) { - return true; - } - - if (visitedNodes.has(node)) { - return false; - } - - visitedNodes.add(node); - - for (const edge of this._pendingDependencyEdges(node, handledNodes)) { - if (this._isBlockedBy(edge.getOtherNode(node), blockingNode, handledNodes, visitedNodes)) { - return true; - } - } - - return false; - } - - _createDeferredRelationActions() { - if (this.deferredEdges.size === 0) { - return []; - } - - return [ - new DeferredRelationGraphInsertAction(this.graphData, { - edges: [...this.deferredEdges], - }), - ]; - } - - _hasDependencies(node, handledNodes) { - return this._pendingDependencyEdges(node, handledNodes).length > 0; - } - - // Returns the dependency edges of `node` whose other node still needs to be - // inserted first. Deferred edges are ignored. - _pendingDependencyEdges(node, handledNodes) { - const edges = this.dependencies.get(node) || []; - - return edges.filter((edge) => { - const dependencyNode = edge.getOtherNode(node); - - return ( - !this.deferredEdges.has(edge) && - !handledNodes.has(dependencyNode) && - !this.currentGraph.nodeForNode(dependencyNode) - ); - }); - } - - _createJoinRowActions() { - return [ - new JoinRowGraphInsertAction(this.graphData, { - nodes: this.graph.nodes.filter((node) => { - return ( - this.currentGraph.nodeForNode(node) === null && - node.parentEdge && - node.parentEdge.relation.isObjectionManyToManyRelation - ); - }), - }), - ]; - } -} diff --git a/lib/queryBuilder/graph/insert/GraphInsertAction.js b/lib/queryBuilder/graph/insert/GraphInsertAction.js deleted file mode 100644 index 89a1979d6..000000000 --- a/lib/queryBuilder/graph/insert/GraphInsertAction.js +++ /dev/null @@ -1,236 +0,0 @@ -import { GraphAction } from '../GraphAction.js'; -import { groupBy, chunk, get, set } from '../../../utils/objectUtils.js'; -import { ModelGraphEdge } from '../../../model/graph/ModelGraphEdge.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -/** - * Inserts a batch of nodes for a GraphInsert. - * - * One of these is created for each batch of nodes that can be inserted at once. - * However, the nodes can have a different table and not all databases support - * batch inserts, so this class splits the inserts into further sub batches - * when needed. - */ -export class GraphInsertAction extends GraphAction { - constructor(graphData, { nodes, refNodes, dependencies, deferredEdges }) { - super(graphData); - - // Nodes to insert, and `#ref` nodes to resolve once they are inserted. - this.nodes = nodes; - this.refNodes = refNodes; - this.dependencies = dependencies; - // Dependency edges that are resolved once the dependency node has been - // inserted, and are patched in by `DeferredRelationGraphInsertAction`. - this.deferredEdges = deferredEdges; - } - - async run(builder) { - const batches = this._createInsertBatches(builder); - const concurrency = this._getConcurrency(builder, this.nodes); - - await promiseUtils.map(batches, (batch) => this._insertBatch(builder, batch), { concurrency }); - this._resolveDependencies(this.refNodes); - } - - _createInsertBatches(builder) { - const batches = []; - const batchSize = this._getBatchSize(builder); - const nodesByModelClass = groupBy(this.nodes, getModelClass); - - for (const nodes of nodesByModelClass.values()) { - for (const nodeBatch of chunk(nodes, batchSize)) { - batches.push(nodeBatch); - } - } - - return batches; - } - - async _insertBatch(parentBuilder, nodes) { - await this._beforeInsert(nodes); - await this._insert(parentBuilder, nodes); - } - - _beforeInsert(nodes) { - this._resolveDependencies(nodes); - - for (const node of nodes) { - if (this.graphData.actions.get(node) === 'insert') { - this._setBelongsToManyKeys(node); - } - } - - this._omitManyToManyExtraProps(nodes); - this._copyRelationPropsFromCurrentIfNeeded(nodes); - - return Promise.resolve(); - } - - _resolveDependencies(nodes) { - for (const node of nodes) { - const edges = this.dependencies.get(node); - - if (edges) { - for (const edge of edges) { - if (this.deferredEdges.has(edge)) { - continue; - } - - // `node` needs `dependencyNode` to have been inserted (and it has been). - const dependencyNode = edge.getOtherNode(node); - - this._resolveDependency(dependencyNode, edge); - } - } - } - } - - _resolveDependency(dependencyNode, edge) { - if (edge.type === ModelGraphEdge.Type.Relation) { - // The keys of join tables and of BelongsToManyRelations are set separately. - if (!edge.relation.joinTable && !edge.relation.isObjectionBelongsToManyRelation) { - this._resolveRelationDependency(dependencyNode, edge); - } - } else if (edge.refType === ModelGraphEdge.ReferenceType.Object) { - this._resolveReferences(edge.getOtherNode(dependencyNode)); - } else if (edge.refType === ModelGraphEdge.ReferenceType.Property) { - this._resolvePropertyReferenceNode(dependencyNode, edge); - } - } - - _resolveRelationDependency(dependencyNode, edge) { - const dependentNode = edge.getOtherNode(dependencyNode); - - let sourceProp; - let targetProp; - - if (edge.isOwnerNode(dependencyNode)) { - sourceProp = edge.relation.ownerProp; - targetProp = edge.relation.relatedProp; - } else { - targetProp = edge.relation.ownerProp; - sourceProp = edge.relation.relatedProp; - } - - this._resolveReferences(dependencyNode); - - targetProp.forEach((i) => { - targetProp.setProp(dependentNode.obj, i, sourceProp.getProp(dependencyNode.obj, i)); - }); - } - - _resolvePropertyReferenceNode(dependencyNode, edge) { - const dependentNode = edge.getOtherNode(dependencyNode); - - let sourcePath; - let targetPath; - - if (edge.isOwnerNode(dependencyNode)) { - sourcePath = edge.refOwnerDataPath; - targetPath = edge.refRelatedDataPath; - } else { - targetPath = edge.refOwnerDataPath; - sourcePath = edge.refRelatedDataPath; - } - - const sourceValue = get(dependencyNode.obj, sourcePath); - const targetValue = get(dependentNode.obj, targetPath); - - if (targetValue === edge.refMatch) { - set(dependentNode.obj, targetPath, sourceValue); - } else { - set(dependentNode.obj, targetPath, targetValue.replace(edge.refMatch, sourceValue)); - } - } - - _omitManyToManyExtraProps(nodes) { - for (const node of nodes) { - if ( - node.parentEdge && - node.parentEdge.type === ModelGraphEdge.Type.Relation && - node.parentEdge.relation.joinTableExtras.length > 0 - ) { - node.parentEdge.relation.omitExtraProps([node.obj]); - } - } - } - - _copyRelationPropsFromCurrentIfNeeded(nodes) { - for (const node of nodes) { - const currentNode = this.currentGraph.nodeForNode(node); - - if (!currentNode) { - continue; - } - - for (const edge of node.edges) { - if (edge.type !== ModelGraphEdge.Type.Relation) { - continue; - } - - // The arrays of BelongsToManyRelations are set by `_setBelongsToManyKeys()`: - if (edge.relation.isObjectionBelongsToManyRelation && edge.isOwnerNode(node)) { - continue; - } - - const prop = edge.isOwnerNode(node) ? edge.relation.ownerProp : edge.relation.relatedProp; - - prop.forEach((i) => { - const value = prop.getProp(node.obj, i); - - if (value !== undefined) { - return; - } - - prop.setProp(node.obj, i, prop.getProp(currentNode.obj, i)); - }); - } - } - } - - _insert(parentBuilder, nodes) { - const [{ modelClass }] = nodes; - - nodes = nodes.filter((node) => this.graphData.actions.get(node) === 'insert'); - - for (const node of nodes) { - delete node.obj[modelClass.uidProp]; - delete node.obj[modelClass.graphUnrelateProp]; - delete node.obj[modelClass.graphDeleteProp]; - node.obj.$validate(null, { dataPath: node.dataPathKey }); - } - - if (nodes.length === 0) { - return; - } - - return this._runRelationBeforeInsertMethods(parentBuilder, nodes).then(() => { - return modelClass - .query() - .insert(nodes.map((node) => node.obj)) - .childQueryOf(parentBuilder) - .copyFrom(parentBuilder, GraphAction.ReturningAllSelector) - .execute(); - }); - } - - _runRelationBeforeInsertMethods(parentBuilder, nodes) { - return Promise.all( - nodes.map((node) => { - if (node.parentEdge) { - return node.parentEdge.relation.beforeInsert( - node.obj, - parentBuilder.context(), - node.parentNode.obj, - ); - } else { - return null; - } - }), - ); - } -} - -function getModelClass(node) { - return node.modelClass; -} diff --git a/lib/queryBuilder/graph/insert/JoinRowGraphInsertAction.js b/lib/queryBuilder/graph/insert/JoinRowGraphInsertAction.js deleted file mode 100644 index d0b189bca..000000000 --- a/lib/queryBuilder/graph/insert/JoinRowGraphInsertAction.js +++ /dev/null @@ -1,87 +0,0 @@ -import { GraphAction } from '../GraphAction.js'; -import { groupBy, chunk } from '../../../utils/objectUtils.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -export class JoinRowGraphInsertAction extends GraphAction { - constructor(graphData, { nodes }) { - super(graphData); - this.nodes = nodes; - } - - run(builder) { - const batches = this._createInsertBatches(builder); - const concurrency = this._getConcurrency(builder, this.nodes); - - return promiseUtils.map(batches, (batch) => this._insertBatch(builder, batch), { concurrency }); - } - - _createInsertBatches(builder) { - const batches = []; - const batchSize = this._getBatchSize(builder); - const nodesByModel = groupBy(this.nodes, (node) => getJoinTableModel(builder, node)); - - for (const [joinTableModelClass, nodes] of nodesByModel.entries()) { - for (const nodeBatch of chunk(nodes, batchSize)) { - batches.push(this._createBatch(joinTableModelClass, nodeBatch)); - } - } - - return batches; - } - - _createBatch(joinTableModelClass, nodes) { - return nodes - .filter((node) => { - const action = this.graphData.actions.get(node); - return action === 'insert' || action === 'relate'; - }) - .map((node) => ({ - node, - joinTableModelClass, - joinTableObj: this._createJoinTableObj(joinTableModelClass, node), - })); - } - - _createJoinTableObj(joinTableModelClass, node) { - const { parentEdge, parentNode } = node; - const { relation } = parentEdge; - - this._resolveReferences(node); - - return joinTableModelClass.fromJson( - relation.createJoinModel(relation.ownerProp.getProps(parentNode.obj), node.obj), - ); - } - - _insertBatch(parentBuilder, batch) { - return this._beforeInsert(parentBuilder, batch).then(() => this._insert(parentBuilder, batch)); - } - - _beforeInsert(parentBuilder, batch) { - return Promise.all( - batch.map(({ node, joinTableObj }) => { - if (node.parentEdge) { - return node.parentEdge.relation.joinTableBeforeInsert( - joinTableObj, - parentBuilder.context(), - ); - } else { - return null; - } - }), - ); - } - - _insert(parentBuilder, batch) { - if (batch.length > 0) { - return batch[0].joinTableModelClass - .query() - .childQueryOf(parentBuilder) - .insert(batch.map((it) => it.joinTableObj)); - } - } -} - -function getJoinTableModel(builder, node) { - return node.parentEdge.relation.getJoinModelClass(builder.unsafeKnex()); -} diff --git a/lib/queryBuilder/graph/patch/GraphPatch.js b/lib/queryBuilder/graph/patch/GraphPatch.js deleted file mode 100644 index a83490bc3..000000000 --- a/lib/queryBuilder/graph/patch/GraphPatch.js +++ /dev/null @@ -1,14 +0,0 @@ -import { GraphOperation } from '../GraphOperation.js'; -import { createPatchActions, isRelatedFirst } from './GraphRelate.js'; - -export class GraphPatch extends GraphOperation { - createActions() { - return createPatchActions( - this.graphData, - this.graph.nodes.filter((node) => { - const action = this.graphData.actions.get(node); - return (action === 'relate' && !isRelatedFirst(node, this.graphData)) || action === 'patch'; - }), - ); - } -} diff --git a/lib/queryBuilder/graph/patch/GraphPatchAction.js b/lib/queryBuilder/graph/patch/GraphPatchAction.js deleted file mode 100644 index 7569d71aa..000000000 --- a/lib/queryBuilder/graph/patch/GraphPatchAction.js +++ /dev/null @@ -1,305 +0,0 @@ -import { getModel } from '../../../model/getModel.js'; -import { GraphAction } from '../GraphAction.js'; -import { UpdateOperation } from '../../operations/UpdateOperation.js'; -import { isInternalProp } from '../../../utils/internalPropUtils.js'; -import { difference, isObject, isPlainObject, jsonEquals } from '../../../utils/objectUtils.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -export class GraphPatchAction extends GraphAction { - constructor(graphData, { nodes }) { - super(graphData); - // Nodes to patch. - this.nodes = nodes; - } - - run(builder) { - return promiseUtils.map(this.nodes, (node) => this._runForNode(builder, node), { - concurrency: this._getConcurrency(builder, this.nodes), - }); - } - - _runForNode(builder, node) { - const shouldPatch = this.graphOptions.shouldPatch(node); - const shouldUpdate = this.graphOptions.shouldUpdate(node); - - // BelongsToOneRelation inserts and relates change the parent object's - // properties. That's why we handle them here. - const changedPropsBecauseOfBelongsToOneInsert = this._handleBelongsToOneInserts(node); - - // BelongsToOneRelation deletes and unrelates change the parent object's - // properties. That's why we handle them here. - const changePropsBecauseOfBelongsToOneDelete = this._handleBelongsToOneDeletes(node); - - // BelongsToManyRelation changes are written to the parent object's arrays - // of keys. Related nodes have their relations written by the nested upsert - // of `GraphRecursiveUpsert`. - const changedPropsBecauseOfBelongsToMany = - this.graphData.actions.get(node) === 'patch' ? this._setBelongsToManyKeys(node) : []; - - // HasManyRelation and HasOneRelation relates write the related object's - // foreign keys, also with `noUpdate`. - const changedPropsBecauseOfRelate = - this.graphData.actions.get(node) === 'relate' && - node.parentEdge.relation.isObjectionHasManyRelation - ? node.parentEdge.relation.relatedProp.props - : []; - - const handleUpdate = () => { - const { changedProps, unchangedProps } = this._findChanges(node); - const allProps = [...changedProps, ...unchangedProps]; - - const propsToUpdate = difference( - shouldPatch || shouldUpdate - ? changedProps - : [ - ...changedPropsBecauseOfBelongsToOneInsert, - ...changePropsBecauseOfBelongsToOneDelete, - ...changedPropsBecauseOfBelongsToMany, - ...changedPropsBecauseOfRelate, - ], - - // Remove id properties from the props to update. With upsertGraph - // it never makes sense to change the id. - node.modelClass.getIdPropertyArray(), - ); - - const update = propsToUpdate.length > 0; - if (update) { - // Don't update the fields that we know not to change. - node.obj.$omitFromDatabaseJson(difference(allProps, propsToUpdate)); - } - - return update; - }; - - const Model = getModel(); - // See if the model defines a beforeUpdate or $beforeUpdate hook. If it does - // not, we can check for updated properties now and drop out immediately if - // there is nothing to update. Otherwise, we need to wait for the hook to be - // called before calling handleUpdate, but only if the node contains changes - // that aren't id properties (relates). See issues #2233, #2605. - const hasBeforeUpdate = - node.obj.constructor.beforeUpdate !== Model.beforeUpdate || - node.obj.$beforeUpdate !== Model.prototype.$beforeUpdate; - - if ( - (hasBeforeUpdate && !this._hasNonIdPropertyChanges(node)) || - (!hasBeforeUpdate && !handleUpdate()) - ) { - return null; - } - - delete node.obj[node.modelClass.uidProp]; - delete node.obj[node.modelClass.uidRefProp]; - delete node.obj[node.modelClass.dbRefProp]; - delete node.obj[node.modelClass.graphUnrelateProp]; - delete node.obj[node.modelClass.graphDeleteProp]; - - node.obj.$validate(null, { - dataPath: node.dataPathKey, - patch: shouldPatch || (!shouldPatch && !shouldUpdate), - }); - - const updateBuilder = this._createBuilder(node) - .childQueryOf(builder, childQueryOptions()) - // The current graph was fetched through the relations, so the patched - // models are known to be related to their parents. - .internalOptions({ skipRelationFilter: true }) - .copyFrom(builder, GraphAction.ReturningAllSelector); - - if (hasBeforeUpdate) { - updateBuilder.internalContext().runBefore.push((result, builder) => { - // Call handleUpdate in the runBefore hook which runs after the - // $beforeUpdate hook, allowing it to modify the object before the - // updated properties are determined. See issue #2233. - if (hasBeforeUpdate && !handleUpdate()) { - builder.internalOptions().returnImmediatelyValue = null; - } - return result; - }); - } - - if (shouldPatch) { - updateBuilder.patch(node.obj); - } else { - updateBuilder.update(node.obj); - } - - return updateBuilder - .runAfter((result, updateBuilder) => { - if (isObject(result) && result.$isObjectionModel) { - // Handle returning('*'). - node.obj.$set(result); - } else if ( - (result === 0 || result === undefined) && - // Not if a hook resolved the query, e.g. with `cancelQuery()`, or - // there was no column to write: - !updateBuilder.findOperation(UpdateOperation).isResolvedWithoutWrite(updateBuilder) - ) { - // The row is gone, e.g. deleted through `ON DELETE CASCADE` by an - // earlier write of the graph. A `beforeUpdate` hook that leaves - // nothing to update returns null instead. - throw node.modelClass.createNotFoundError(builder.context(), { - message: `model (id=${node.obj.$id()}) was not found when updating it. An earlier write of the graph or a concurrent query may have deleted it`, - dataPath: node.dataPath, - }); - } - - return result; - }) - .execute(); - } - - _handleBelongsToOneInserts(node) { - const currentNode = this.currentGraph.nodeForNode(node); - const updatedProps = []; - - for (const edge of node.edges) { - if ( - edge.isOwnerNode(node) && - edge.relation && - edge.relation.isObjectionBelongsToOneRelation && - edge.relation.relatedProp.hasProps(edge.relatedNode.obj) && - // Ignored models don't replace the current one, and models marked with - // `#unrelate` or `#delete` are handled below. - this.graphData.actions.get(edge.relatedNode) !== 'ignore' - ) { - const { relation } = edge; - - relation.ownerProp.forEach((i) => { - const currentValue = currentNode && relation.ownerProp.getProp(currentNode.obj, i); - const relatedValue = relation.relatedProp.getProp(edge.relatedNode.obj, i); - - if (currentValue != relatedValue) { - relation.ownerProp.setProp(node.obj, i, relatedValue); - updatedProps.push(relation.ownerProp.props[i]); - } - }); - } - } - - return updatedProps; - } - - _handleBelongsToOneDeletes(node) { - const currentNode = this.currentGraph.nodeForNode(node); - const updatedProps = []; - - if (!currentNode) { - return updatedProps; - } - - for (const edge of currentNode.edges) { - if ( - edge.isOwnerNode(currentNode) && - edge.relation.isObjectionBelongsToOneRelation && - (node.obj[edge.relation.name] === null || - isToBeUnrelatedOrDeleted(this.graph.nodeForNode(edge.relatedNode))) && - this.graphData.actions.get(edge.relatedNode) !== 'keep' - ) { - const { relation } = edge; - - relation.ownerProp.forEach((i) => { - const currentValue = relation.ownerProp.getProp(currentNode.obj, i); - - if (currentValue != null) { - relation.ownerProp.setProp(node.obj, i, null); - updatedProps.push(relation.ownerProp.props[i]); - } - }); - } - } - - return updatedProps; - } - - _findChanges(node) { - const obj = node.obj; - const currentNode = this.currentGraph.nodeForNode(node); - const currentObj = (currentNode && currentNode.obj) || {}; - const relationNames = node.modelClass.getRelationNames(); - - const unchangedProps = []; - const changedProps = []; - - for (const prop of Object.keys(obj)) { - if (isInternalProp(prop) || relationNames.includes(prop)) { - continue; - } - - const value = obj[prop]; - const currentValue = currentObj[prop]; - - // If the current object doesn't have the property, we have to assume - // it changes (we cannot know if it will). If the object does have the - // property, we test non-strict equality. See issue #732. - if (currentValue === undefined || !nonStrictEquals(currentValue, value)) { - changedProps.push(prop); - } else { - unchangedProps.push(prop); - } - } - - // We cannot know if the query properties cause changes to the values. - // We must assume that they do. - if (obj.$$queryProps) { - changedProps.push(...Object.keys(obj.$$queryProps)); - } - - return { - changedProps, - unchangedProps, - }; - } - - _hasNonIdPropertyChanges(node) { - const idProps = node.modelClass.getIdPropertyArray(); - return this._findChanges(node).changedProps.some((prop) => !idProps.includes(prop)); - } - - _createBuilder(node) { - if (node.parentEdge && !node.parentEdge.relation.isObjectionHasManyRelation) { - return this._createRelatedBuilder(node); - } else { - return this._createRootBuilder(node); - } - } - - _createRelatedBuilder(node) { - return node.parentNode.obj - .$relatedQuery(node.parentEdge.relation.name) - .findById(node.obj.$id()); - } - - _createRootBuilder(node) { - const currentNode = this.currentGraph.nodeForNode(node); - const currentObj = currentNode && currentNode.obj; - - return currentObj ? currentObj.$query() : node.obj.$query(); - } -} - -function childQueryOptions() { - return { - fork: true, - isInternalQuery: true, - }; -} - -function nonStrictEquals(val1, val2) { - // Only use loose equality for non-JSON values, so that for example `false` - // and `[]` are not considered equal. See issue #1874. - if (!isJsonValue(val1) && !isJsonValue(val2) && val1 == val2) { - return true; - } - - return jsonEquals(val1, val2); -} - -function isJsonValue(value) { - return Array.isArray(value) || isPlainObject(value); -} - -function isToBeUnrelatedOrDeleted(node) { - return !!node && node.toBeUnrelatedOrDeleted; -} diff --git a/lib/queryBuilder/graph/patch/GraphRelate.js b/lib/queryBuilder/graph/patch/GraphRelate.js deleted file mode 100644 index 4690f316e..000000000 --- a/lib/queryBuilder/graph/patch/GraphRelate.js +++ /dev/null @@ -1,94 +0,0 @@ -import { GraphOperation } from '../GraphOperation.js'; -import { GraphPatchAction } from './GraphPatchAction.js'; - -// Relate the rows that move to existing owners of HasManyRelations and -// HasOneRelations before `GraphDelete` runs, as they may still sit below rows -// that it deletes, e.g. with `ON DELETE CASCADE` (#214). -export class GraphRelate extends GraphOperation { - createActions() { - const nodes = this.graph.nodes.filter((node) => isRelatedFirst(node, this.graphData)); - - for (const node of nodes) { - const { relation } = node.parentEdge; - const { ownerProp, relatedProp } = relation; - const currentParentNode = this.currentGraph.nodeForNode(node.parentNode); - - // The graph may leave out the owner's key if it isn't the id: - relatedProp.forEach((i) => { - relatedProp.setProp( - node.obj, - i, - ownerProp.getProp(node.parentNode.obj, i) ?? ownerProp.getProp(currentParentNode.obj, i), - ); - }); - } - - return createPatchActions(this.graphData, nodes); - } -} - -// A relate into the slot of a HasOneRelation runs after the relate that moves -// the row in it to another owner, as the foreign key may be unique. Order the -// patches of `nodes` in as many actions as the chains of such moves need. -export function createPatchActions(graphData, nodes) { - const actions = []; - let pending = nodes; - - while (pending.length > 0) { - const ready = pending.filter((node) => !pending.includes(getSlotMoverNode(node, graphData))); - // Swapped rows can't be ordered, so we relate them together: - const batch = ready.length > 0 ? ready : pending; - actions.push(new GraphPatchAction(graphData, { nodes: batch })); - pending = pending.filter((node) => !batch.includes(node)); - } - - return actions; -} - -// `#ref` nodes wait for the inserts, as their ids may come from them. A relate -// into the slot of a HasOneRelation waits for `GraphDelete` to remove the row -// in it, as the foreign key may be unique, and for the relate that moves it -// to another owner if that one waits. -export function isRelatedFirst(node, graphData, visitedNodes = new Set()) { - const { currentGraph, actions } = graphData; - const relation = node.parentEdge?.relation; - const slotNode = getCurrentSlotNode(node, graphData); - const moverNode = getSlotMoverNode(node, graphData); - - visitedNodes.add(node); - - return ( - actions.get(node) === 'relate' && - !node.isReference && - !!relation?.isObjectionHasManyRelation && - !!currentGraph.nodeForNode(node.parentNode) && - !(slotNode && actions.get(slotNode) !== 'keep') && - // Swapped rows are related together: - (!moverNode || - visitedNodes.has(moverNode) || - isRelatedFirst(moverNode, graphData, visitedNodes)) - ); -} - -// Find the relate that moves the current row in the slot of the HasOneRelation -// that `node` relates into to another owner: -function getSlotMoverNode(node, graphData) { - const slotNode = graphData.actions.get(node) === 'relate' && getCurrentSlotNode(node, graphData); - const keptNode = slotNode && graphData.keptNodeForRow(slotNode); - - return keptNode && graphData.actions.get(keptNode) === 'relate' ? keptNode : null; -} - -// Find the current node in the slot of the HasOneRelation that `node` relates -// into, or null if it relates into a HasManyRelation or an empty slot: -function getCurrentSlotNode(node, graphData) { - const relation = node.parentEdge?.relation; - const currentParentNode = graphData.currentGraph.nodeForNode(node.parentNode); - const edge = - relation?.isOneToOne() && - currentParentNode?.edges.find( - (edge) => edge.relation === relation && edge.isOwnerNode(currentParentNode), - ); - - return edge?.relatedNode ?? null; -} diff --git a/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsert.js b/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsert.js deleted file mode 100644 index e4e07cd50..000000000 --- a/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsert.js +++ /dev/null @@ -1,24 +0,0 @@ -import { GraphOperation } from '../GraphOperation.js'; -import { GraphRecursiveUpsertAction } from './GraphRecursiveUpsertAction.js'; - -export class GraphRecursiveUpsert extends GraphOperation { - createActions() { - return [ - new GraphRecursiveUpsertAction(this.graphData, { - nodes: this.graph.nodes.filter( - (node) => this.graphData.actions.get(node) === 'relate' && hasRelations(node.obj), - ), - }), - ]; - } -} - -function hasRelations(obj) { - for (const relationName of obj.constructor.getRelationNames()) { - if (obj[relationName] !== undefined) { - return true; - } - } - - return false; -} diff --git a/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsertAction.js b/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsertAction.js deleted file mode 100644 index 7d26d4393..000000000 --- a/lib/queryBuilder/graph/recursiveUpsert/GraphRecursiveUpsertAction.js +++ /dev/null @@ -1,119 +0,0 @@ -import { GraphAction } from '../GraphAction.js'; -import { groupBy, get, set } from '../../../utils/objectUtils.js'; -import { forEachPropertyReference } from '../../../model/graph/ModelGraphBuilder.js'; -import * as promiseUtils from '../../../utils/promiseUtils/index.js'; - -export class GraphRecursiveUpsertAction extends GraphAction { - constructor(graphData, { nodes }) { - super(graphData); - // Nodes to upsert. - this.nodes = nodes; - } - - run(builder) { - const builders = this._createUpsertBuilders(builder, this.nodes); - - return promiseUtils.map(builders, (builder) => builder.execute(), { - concurrency: this._getConcurrency(builder, this.nodes), - }); - } - - _createUpsertBuilders(parentBuilder, nodesToUpsert) { - const nodesByRelation = groupBy(nodesToUpsert, getRelation); - const builders = []; - - nodesByRelation.forEach((nodes) => { - const nodesByParent = groupBy(nodes, getParent); - - nodesByParent.forEach((nodes) => { - for (const node of nodes) { - this._resolveReferences(node); - } - - builders.push( - nodes[0].modelClass - .query() - .childQueryOf(parentBuilder) - .copyFrom(parentBuilder, GraphAction.ReturningAllSelector) - .upsertGraph( - nodes.map((node) => node.obj), - this.graphOptions.rebasedOptions(nodes[0]), - ), - ); - }); - }); - - return builders; - } - - /** - * The nodes inside the subgraph we are about to recursively upsert may - * have references outside that graph that won't be available during the - * recursive upsertGraph call. This method resolves the references. - * - * TODO: This doesn't work if a recursively upserted node refers to - * a node inside another recursively upsertable graph. - */ - _resolveReferences(node) { - node.obj.$traverse((obj) => this._resolveReferencesForObject(obj)); - } - - _resolveReferencesForObject(obj) { - this._resolveObjectReference(obj); - this._resolvePropertyReferences(obj); - } - - _resolveObjectReference(obj) { - const modelClass = obj.constructor; - const ref = obj[modelClass.uidRefProp]; - - if (!ref) { - return; - } - - const referencedNode = this.graph.nodes.find((it) => it.uid === ref); - - if (!referencedNode) { - return; - } - - const relationNames = referencedNode.modelClass.getRelationNames(); - - for (const prop of Object.keys(referencedNode.obj)) { - if (relationNames.includes(prop)) { - continue; - } - - obj[prop] = referencedNode.obj[prop]; - } - - delete obj[modelClass.uidRefProp]; - } - - _resolvePropertyReferences(obj) { - forEachPropertyReference(obj, ({ path, refMatch, ref, refPath }) => { - const referencedNode = this.graph.nodes.find((it) => it.uid === ref); - - if (!referencedNode) { - return; - } - - const referencedValue = get(referencedNode.obj, refPath); - const value = get(obj, path); - - if (value === refMatch) { - set(obj, path, referencedValue); - } else { - set(obj, path, value.replace(refMatch, referencedValue)); - } - }); - } -} - -function getRelation(node) { - return node.parentEdge.relation; -} - -function getParent(node) { - return node.parentNode; -} diff --git a/lib/relations/Relation.js b/lib/relations/Relation.js index f1e098ff9..c095a5fe9 100644 --- a/lib/relations/Relation.js +++ b/lib/relations/Relation.js @@ -82,6 +82,13 @@ export class Relation { return false; } + // Where the keys that connect owners and related models are stored: in the + // `'owner'` rows, the `'related'` rows or the rows of a `'join'` table. Owners + // of relations that aren't one-to-one hold arrays of keys. + get keySide() { + throw new Error('not implemented'); + } + clone() { const relation = new this.constructor(this.name, this.ownerModelClass); diff --git a/lib/relations/belongsToMany/BelongsToManyRelation.js b/lib/relations/belongsToMany/BelongsToManyRelation.js index 64bea6d1e..e1baf4b71 100644 --- a/lib/relations/belongsToMany/BelongsToManyRelation.js +++ b/lib/relations/belongsToMany/BelongsToManyRelation.js @@ -20,6 +20,11 @@ const ownersAlias = 'objectiontmpowners'; const keysAlias = 'objectiontmpkeys'; export class BelongsToManyRelation extends Relation { + // The owner holds the keys, in an array. + get keySide() { + return 'owner'; + } + setMapping(mapping) { const retVal = super.setMapping(mapping); diff --git a/lib/relations/belongsToOne/BelongsToOneRelation.js b/lib/relations/belongsToOne/BelongsToOneRelation.js index 47d9caf8d..883bfef19 100644 --- a/lib/relations/belongsToOne/BelongsToOneRelation.js +++ b/lib/relations/belongsToOne/BelongsToOneRelation.js @@ -9,6 +9,10 @@ export class BelongsToOneRelation extends Relation { return true; } + get keySide() { + return 'owner'; + } + insert(_, owner) { return new BelongsToOneInsertOperation('insert', { relation: this, diff --git a/lib/relations/hasMany/HasManyRelation.js b/lib/relations/hasMany/HasManyRelation.js index 3b974e66d..ce142b0dd 100644 --- a/lib/relations/hasMany/HasManyRelation.js +++ b/lib/relations/hasMany/HasManyRelation.js @@ -4,6 +4,10 @@ import { HasManyRelateOperation } from './HasManyRelateOperation.js'; import { HasManyUnrelateOperation } from './HasManyUnrelateOperation.js'; export class HasManyRelation extends Relation { + get keySide() { + return 'related'; + } + insert(_, owner) { return new HasManyInsertOperation('insert', { relation: this, diff --git a/lib/relations/manyToMany/ManyToManyRelation.js b/lib/relations/manyToMany/ManyToManyRelation.js index ed64e3f8e..a940fc995 100644 --- a/lib/relations/manyToMany/ManyToManyRelation.js +++ b/lib/relations/manyToMany/ManyToManyRelation.js @@ -34,6 +34,10 @@ const nestedJoinTypes = { }; export class ManyToManyRelation extends Relation { + get keySide() { + return 'join'; + } + setMapping(mapping) { const retVal = super.setMapping(mapping); diff --git a/lib/utils/knexUtils.js b/lib/utils/knexUtils.js index 4cfa85290..3b3bc7b52 100644 --- a/lib/utils/knexUtils.js +++ b/lib/utils/knexUtils.js @@ -40,6 +40,27 @@ export function hasBatchInsert(knex) { return isPostgres(knex) || isMsSql(knex) || isSqlite(knex); } +// knex inserts the rows of a SQLite batch with `select ... union all`, which +// fills the columns that a row leaves out with NULL, not with their defaults. +export function hasBatchInsertDefaults(knex) { + return !isSqlite(knex); +} + +// The number of rows that fit into one query if each binds `parameters` values. +// SQL Server binds 2100 values, Postgres 65535 and SQLite 999 (32766 since 3.32, +// but builds may be older), and we leave a margin for the values that aren't +// rows. The MySQL drivers inline the values in the query. +export function getMaxRows(knex, parameters) { + const maxParameters = isMsSql(knex) + ? 2000 + : isPostgres(knex) + ? 65000 + : isSqlite(knex) + ? 900 + : Infinity; + return Math.max(1, Math.floor(maxParameters / parameters)); +} + // The `pg` drivers return `json` and `jsonb` columns parsed, and `text` columns // as text. export function isJsonParsedByDriver(knex) { diff --git a/tests/integration/graph/GraphInsert.js b/tests/integration/graph/GraphInsert.js index 6980eeda2..47bc254de 100644 --- a/tests/integration/graph/GraphInsert.js +++ b/tests/integration/graph/GraphInsert.js @@ -2,13 +2,6 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import mockKnexFactory from '../../../testUtils/mockKnex.js'; import { Model, raw } from 'objection'; -import { ModelGraph } from '../../../lib/model/graph/ModelGraph.js'; -import { GraphInsert } from '../../../lib/queryBuilder/graph/insert/GraphInsert.js'; -import { GraphOptions } from '../../../lib/queryBuilder/graph/GraphOptions.js'; -import { GraphNodeDbExistence } from '../../../lib/queryBuilder/graph/GraphNodeDbExistence.js'; -import { GraphFetcher } from '../../../lib/queryBuilder/graph/GraphFetcher.js'; -import { GraphData } from '../../../lib/queryBuilder/graph/GraphData.js'; -import { asArray } from '../../../lib/utils/objectUtils.js'; export default (session) => { const ID_NOT_IN_DB = 1000000; @@ -285,6 +278,10 @@ export default (session) => { it("should only insert rows that don't exists in db", () => { return test({ modelClass: Person, + // `insertGraph()` doesn't fetch the existing rows, so leave them + // untouched with `upsertGraph()`: + upsert: true, + graphOptions: { noUpdate: true, noDelete: true }, graphIn: { id: matti.id, @@ -338,7 +335,8 @@ export default (session) => { ], }, - postgresNumQueries: 3, + // 3 queries to fetch the current graph and 3 to insert the new rows: + postgresNumQueries: 6, }); }); }); @@ -1026,39 +1024,19 @@ export default (session) => { function test({ modelClass, graphIn, - graphOptions: rawGraphOptions = {}, + graphOptions = {}, + upsert = false, queryOut, graphOut, postgresNumQueries = null, check, }) { - const builder = modelClass.query(); - const models = modelClass.ensureModelArray(graphIn, { skipValidation: true }); - rawGraphOptions = Object.assign({}, rawGraphOptions, { insertMissing: true }); - const graphOptions = new GraphOptions(rawGraphOptions); - const graph = assignDbRefsAsRelateProps(ModelGraph.create(modelClass, models)); - const nodeDbExistence = GraphNodeDbExistence.createEveryNodeExistsExistence(); - - return GraphFetcher.fetchCurrentGraph({ builder, graph, graphOptions }) - .then((currentGraph) => { - numExecutedQueries = 0; - - const graphData = new GraphData({ - graph, - currentGraph, - graphOptions, - nodeDbExistence, - }); - const graphInsert = new GraphInsert(graphData.planActions()); - const actions = graphInsert.createActions(); - let promise = Promise.resolve(); + numExecutedQueries = 0; - actions.forEach((action) => { - promise = promise.then(() => action.run(builder)); - }); + const query = modelClass.query(); + const options = { allowRefs: true, ...graphOptions }; - return promise; - }) + return (upsert ? query.upsertGraph(graphIn, options) : query.insertGraph(graphIn, options)) .then(() => { if (session.isPostgres() && postgresNumQueries) { expect(numExecutedQueries).toBe(postgresNumQueries); @@ -1074,18 +1052,6 @@ export default (session) => { }); } - function assignDbRefsAsRelateProps(graph) { - for (const node of graph.nodes) { - if (!node.parentEdge || !node.parentEdge.relation || !node.isDbReference) { - continue; - } - - node.parentEdge.relation.setRelateProp(node.obj, asArray(node.dbReference)); - } - - return graph; - } - async function createSchema() { await session.knex.schema .dropTableIfExists('relatives') diff --git a/tests/integration/insertGraph.js b/tests/integration/insertGraph.js index d1b07a2e5..54b325bbf 100644 --- a/tests/integration/insertGraph.js +++ b/tests/integration/insertGraph.js @@ -444,6 +444,27 @@ export default (session) => { }); }); + it('should resolve #ref{} in a foreign key that the relation also sets (Vincit#1378)', async () => { + const inserted = await Model1.query().insertGraph( + { + '#id': 'root', + model1Prop1: 'root', + model1Relation2: [{ model2Prop1: 'child', model1Id: '#ref{root.id}' }], + }, + { allowRefs: true }, + ); + + const child = await Model2.query().findOne('model2_prop1', 'child'); + expect(child.model1Id).toBe(inserted.id); + }); + + it('should insert several models without properties (Vincit#1694)', async () => { + const inserted = await Model1.query().insertGraph([{}, {}]); + + expect(inserted.map((model) => typeof model.id)).toEqual(['number', 'number']); + expect(await Model1.query().findByIds(inserted.map((model) => model.id))).toHaveLength(2); + }); + it('relate: true option should cause models with id to be related instead of inserted', () => { return Model2.query() .insertGraph( diff --git a/tests/integration/misc/issue853.js b/tests/integration/misc/issue853.js new file mode 100644 index 000000000..08890e6ad --- /dev/null +++ b/tests/integration/misc/issue853.js @@ -0,0 +1,140 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { Model } from 'objection'; + +export default (session) => { + describe('insertGraph() with relations through a key that the database generates #853', () => { + const { knex } = session; + + class Person extends Model { + static get tableName() { + return 'issue853_person'; + } + + static get relationMappings() { + return { + friends: { + relation: Model.ManyToManyRelation, + modelClass: Person, + join: { + from: 'issue853_person.uid', + through: { from: 'issue853_friend.aUid', to: 'issue853_friend.bUid' }, + to: 'issue853_person.uid', + }, + }, + }; + } + } + + const generateUid = () => + session.isPostgres() + ? knex.raw('gen_random_uuid()::text') + : session.isMySql() + ? knex.raw('(uuid())') + : session.isMsSql() + ? knex.raw('newid()') + : knex.raw('(lower(hex(randomblob(16))))'); + + const dropTables = () => + knex.schema.dropTableIfExists('issue853_friend').dropTableIfExists('issue853_person'); + + beforeAll(async () => { + await dropTables(); + await knex.schema + .createTable('issue853_person', (table) => { + table.increments('id'); + table.string('name'); + table.string('uid', 36).defaultTo(generateUid()); + }) + .createTable('issue853_friend', (table) => { + table.string('aUid', 36); + table.string('bUid', 36); + }); + }); + + afterAll(dropTables); + + it('should relate the inserted models by the generated key', async () => { + const person = await Person.query(knex).insertGraph({ + name: 'a', + friends: [{ name: 'b' }, { name: 'c' }], + }); + + const fetched = await Person.query(knex) + .findById(person.id) + .withGraphFetched('friends(orderByName)') + .modifiers({ orderByName: (query) => query.orderBy('name') }); + + expect(fetched.friends.map((friend) => friend.name)).toEqual(['b', 'c']); + expect(person.uid).toBe(fetched.uid); + }); + }); + + describe('upsertGraph() with relations that share a join table #853', () => { + const { knex } = session; + + const createLink = (linkType) => ({ + relation: Model.ManyToManyRelation, + modelClass: Node, + join: { + from: 'issue853_node.id', + through: { + from: 'issue853_link.sourceId', + to: 'issue853_link.targetId', + extra: ['linkType'], + beforeInsert: (link) => { + link.linkType = linkType; + }, + }, + to: 'issue853_node.id', + }, + filter: (query) => query.where('issue853_link.linkType', linkType), + }); + + class Node extends Model { + static get tableName() { + return 'issue853_node'; + } + + static get relationMappings() { + return { outgoing: createLink('outgoing'), incoming: createLink('incoming') }; + } + } + + const dropTables = () => + knex.schema.dropTableIfExists('issue853_link').dropTableIfExists('issue853_node'); + + beforeAll(async () => { + await dropTables(); + await knex.schema + .createTable('issue853_node', (table) => { + table.integer('id').primary(); + }) + .createTable('issue853_link', (table) => { + table.integer('sourceId'); + table.integer('targetId'); + table.string('linkType'); + table.primary(['sourceId', 'targetId', 'linkType']); + }); + + await knex('issue853_node').insert([{ id: 2 }, { id: 3 }]); + await knex('issue853_link').insert([ + { sourceId: 2, targetId: 3, linkType: 'outgoing' }, + { sourceId: 3, targetId: 2, linkType: 'incoming' }, + ]); + }); + + afterAll(dropTables); + + it('should keep the join rows of each relation', async () => { + await Node.query(knex).upsertGraph( + { id: 2, '#id': 'two', outgoing: [{ id: 3, incoming: [{ '#ref': 'two' }] }] }, + { allowRefs: true }, + ); + + expect(await knex('issue853_link').orderBy('sourceId')).toEqual([ + { sourceId: 2, targetId: 3, linkType: 'outgoing' }, + { sourceId: 3, targetId: 2, linkType: 'incoming' }, + ]); + }); + }); +}; diff --git a/tests/integration/misc/upsertGraphEdgeCases.js b/tests/integration/misc/upsertGraphEdgeCases.js new file mode 100644 index 000000000..b5f6b4298 --- /dev/null +++ b/tests/integration/misc/upsertGraphEdgeCases.js @@ -0,0 +1,686 @@ +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import { Model } from 'objection'; + +export default (session) => { + const collectQueries = async (callback) => { + const queries = []; + const onQuery = (query) => queries.push(query); + session.knex.on('query', onQuery); + + try { + await callback(); + } finally { + session.knex.removeListener('query', onQuery); + } + + return queries; + }; + + describe('upsertGraph() edge cases of loading, pruning and writing', () => { + const { knex } = session; + + class Toy extends Model { + static get tableName() { + return 'graph_edge_toy'; + } + } + + class Pet extends Model { + static get tableName() { + return 'graph_edge_pet'; + } + + static get relationMappings() { + return { + favoriteToy: { + relation: Model.BelongsToOneRelation, + modelClass: Toy, + join: { from: 'graph_edge_pet.favoriteToyId', to: 'graph_edge_toy.id' }, + }, + owner: { + relation: Model.BelongsToOneRelation, + modelClass: Person, + join: { from: 'graph_edge_pet.ownerId', to: 'graph_edge_person.id' }, + }, + toys: { + relation: Model.HasManyRelation, + modelClass: Toy, + join: { from: 'graph_edge_pet.id', to: 'graph_edge_toy.petId' }, + }, + }; + } + } + + class Person extends Model { + static get tableName() { + return 'graph_edge_person'; + } + + static get jsonAttributes() { + return ['idolIds']; + } + + static get relationMappings() { + return { + children: { + relation: Model.HasManyRelation, + modelClass: Person, + join: { from: 'graph_edge_person.id', to: 'graph_edge_person.parentId' }, + }, + boss: { + relation: Model.BelongsToOneRelation, + modelClass: Person, + join: { from: 'graph_edge_person.bossId', to: 'graph_edge_person.id' }, + }, + friends: { + relation: Model.ManyToManyRelation, + modelClass: Person, + join: { + from: 'graph_edge_person.id', + through: { from: 'graph_edge_friend.aId', to: 'graph_edge_friend.bId' }, + to: 'graph_edge_person.id', + }, + }, + idols: { + relation: Model.BelongsToManyRelation, + modelClass: Person, + join: { from: 'graph_edge_person.idolIds', to: 'graph_edge_person.id' }, + }, + pets: { + relation: Model.HasManyRelation, + modelClass: Pet, + join: { from: 'graph_edge_person.id', to: 'graph_edge_pet.ownerId' }, + }, + petsByCode: { + relation: Model.HasManyRelation, + modelClass: Pet, + join: { from: 'graph_edge_person.code', to: 'graph_edge_pet.ownerCode' }, + }, + }; + } + } + + const tables = ['graph_edge_friend', 'graph_edge_toy', 'graph_edge_pet', 'graph_edge_person']; + + const dropTables = async () => { + for (const table of tables) { + await knex.schema.dropTableIfExists(table); + } + }; + + beforeAll(async () => { + await dropTables(); + await knex.schema + .createTable('graph_edge_person', (table) => { + table.increments('id'); + table.string('name'); + table.integer('parentId'); + table.integer('bossId'); + table.string('code'); + // The pg drivers only parse JSON columns. + if (session.isPostgres()) { + table.jsonb('idolIds'); + } else { + table.text('idolIds'); + } + }) + .createTable('graph_edge_pet', (table) => { + table.increments('id'); + table.string('name'); + table.string('kind').notNullable().defaultTo('plain'); + table.integer('ownerId'); + table.string('ownerCode'); + table.integer('favoriteToyId'); + table.unique(['ownerId', 'name']); + }) + .createTable('graph_edge_toy', (table) => { + table.increments('id'); + table.string('name'); + table.integer('petId'); + }) + .createTable('graph_edge_friend', (table) => { + table.integer('aId'); + table.integer('bId'); + }); + }); + + // Truncating resets the ids, which the rows get in the order of insertion: + // persons 1 to 3, pets 1 and 2 of person 2 and pet 3 of person 3, toys 1 and + // 2 of pet 1 and toys 3 and 4 of pet 3. + beforeEach(async () => { + for (const table of tables) { + await knex(table).truncate(); + } + + for (const name of ['root', 'friend', 'other']) { + await knex('graph_edge_person').insert({ name, code: name }); + } + + for (const [name, ownerId] of [ + ['p1', 2], + ['p2', 2], + ['p3', 3], + ]) { + await knex('graph_edge_pet').insert({ name, ownerId }); + } + + for (const [name, petId] of [ + ['t1', 1], + ['t2', 1], + ['t3', 3], + ['t4', 3], + ]) { + await knex('graph_edge_toy').insert({ name, petId }); + } + }); + + afterAll(dropTables); + + const toyIds = async () => (await knex('graph_edge_toy').orderBy('id')).map((toy) => toy.id); + + describe('rows below models that are pruned', () => { + it('should not remove the rows below a related model that noRelate skips', async () => { + await Person.query(knex).upsertGraph( + { id: 1, friends: [{ id: 2, pets: [{ id: 1, toys: [] }] }] }, + { relate: true, noRelate: ['friends'] }, + ); + + expect(await toyIds()).toEqual([1, 2, 3, 4]); + }); + + it('should not remove the rows below a new model that noInsert skips', async () => { + await Person.query(knex).upsertGraph( + { id: 1, children: [{ name: 'new', pets: [{ id: 1, toys: [] }] }] }, + { relate: true, noInsert: ['children'] }, + ); + + expect(await toyIds()).toEqual([1, 2, 3, 4]); + }); + + it('should not remove the rows below a model marked with #unrelate', async () => { + await Person.query(knex).upsertGraph( + { id: 1, friends: [{ id: 2, '#unrelate': true, pets: [{ id: 1, toys: [] }] }] }, + { allowMarkers: true, relate: true }, + ); + + expect(await toyIds()).toEqual([1, 2, 3, 4]); + }); + }); + + it('should not load the models below a model that noRelate skips', async () => { + await Person.query(knex).upsertGraph( + { id: 1, friends: [{ id: 2, children: [{ id: 999, children: [] }] }] }, + { relate: true, noRelate: ['friends'] }, + ); + + expect(await knex('graph_edge_friend')).toEqual([]); + }); + + it('should throw before writing for a #dbRef to a missing model with relations', async () => { + const err = await Person.query(knex) + .upsertGraph({ id: 1, friends: [{ '#dbRef': 999, children: [{ name: 'nc' }] }] }) + .catch((err) => err); + + expect(err).toBeInstanceOf(Person.NotFoundError); + expect(await knex('graph_edge_friend')).toEqual([]); + expect(await knex('graph_edge_person').where('name', 'nc')).toEqual([]); + }); + + it('should keep the markers of the input when the upsert fails', async () => { + const input = Person.fromJson({ + id: 1, + friends: [{ id: 2, '#unrelate': true }], + children: [{ id: 999 }], + }); + + await expect( + Person.query(knex).upsertGraph(input, { allowMarkers: true }), + ).rejects.toBeInstanceOf(Person.NotFoundError); + expect(input.friends[0]['#unrelate']).toBe(true); + }); + + describe('#ref models with relations', () => { + it('should insert the models below a #ref to a new model', async () => { + const root = await Person.query(knex).upsertGraph( + { + id: 1, + children: [{ '#id': 'child', name: 'new' }], + friends: [{ '#ref': 'child', pets: [{ name: 'refpet' }] }], + }, + { relate: true, allowRefs: true }, + ); + + const [child] = root.children; + expect(await knex('graph_edge_pet').where('name', 'refpet')).toEqual([ + { + id: expect.any(Number), + name: 'refpet', + kind: 'plain', + ownerId: child.id, + ownerCode: null, + favoriteToyId: null, + }, + ]); + }); + + it('should upsert the models below a #ref to an existing model', async () => { + await Person.query(knex).upsertGraph( + { + id: 1, + children: [{ '#id': 'child', id: 2 }], + friends: [{ '#ref': 'child', pets: [{ id: 2, name: 'renamed' }] }], + }, + { relate: true, allowRefs: true }, + ); + + expect(await knex('graph_edge_pet').where('ownerId', 2).orderBy('id')).toEqual([ + { + id: 2, + name: 'renamed', + kind: 'plain', + ownerId: 2, + ownerCode: null, + favoriteToyId: null, + }, + ]); + }); + + it('should return the props of the model that an already related #ref refers to', async () => { + await knex('graph_edge_friend').insert({ aId: 1, bId: 2 }); + + const root = await Person.query(knex).upsertGraph( + { + id: 1, + children: [{ '#id': 'child', id: 2, name: 'x2' }], + friends: [{ '#ref': 'child' }], + }, + { relate: true, allowRefs: true }, + ); + + expect(root.friends[0]).toMatchObject({ id: 2, name: 'x2' }); + expect(await knex('graph_edge_friend')).toEqual([{ aId: 1, bId: 2 }]); + }); + + it('should give a #ref related into a HasManyRelation the key of a parent inserted later (#311)', async () => { + const [, b] = await Person.query(knex).upsertGraph( + [ + { name: 'A', children: [{ '#id': 'n', name: 'N' }] }, + { + name: 'B', + boss: { name: 'Q', boss: { name: 'Q2' } }, + children: [{ '#ref': 'n' }], + }, + ], + { allowRefs: true, relate: true }, + ); + + expect((await knex('graph_edge_person').where('name', 'N').first()).parentId).toBe(b.id); + }); + + it('should break a cycle through a BelongsToOne key instead of an array of keys', async () => { + const [a, b] = await Person.query(knex).upsertGraph( + [ + { '#id': 'a', name: 'A', idols: [{ '#ref': 'b' }] }, + { '#id': 'b', name: 'B', boss: { '#ref': 'a' } }, + ], + { allowRefs: true }, + ); + + expect((await Person.query(knex).findById(a.id)).idolIds).toEqual([b.id]); + expect((await Person.query(knex).findById(b.id)).bossId).toBe(a.id); + }); + }); + + it('should load and remove the rows below a related model once', async () => { + const queries = await collectQueries(() => + Person.query(knex).upsertGraph( + { id: 1, friends: [{ id: 3, pets: [{ id: 3, toys: [{ id: 3 }] }] }] }, + { relate: true }, + ), + ); + + const deletes = queries.filter(({ sql }) => /^delete/i.test(sql)); + expect(deletes).toHaveLength(1); + expect(deletes[0].bindings.filter((value) => value === 4)).toHaveLength(1); + expect(await toyIds()).toEqual([1, 2, 3]); + }); + + it('should read the markers of a model in every place it appears', async () => { + const friend = Person.fromJson({ '#dbRef': 2 }); + + await Person.query(knex).upsertGraph({ id: 1, friends: [friend], children: [friend] }); + + expect((await knex('graph_edge_person')).map((person) => person.id).sort()).toEqual([ + 1, 2, 3, + ]); + expect((await knex('graph_edge_person').where('id', 2).first()).parentId).toBe(1); + }); + + it('should insert the column defaults of rows that leave out columns', async () => { + await Person.query(knex).insertGraph({ + name: 'new', + pets: [{ name: 'a', kind: 'cat' }, { name: 'b' }], + }); + + expect( + await knex('graph_edge_pet').whereIn('name', ['a', 'b']).orderBy('name').select('kind'), + ).toEqual([{ kind: 'cat' }, { kind: 'plain' }]); + }); + + it('should insert the column defaults of rows that a beforeInsert hook leaves out', async () => { + class HookedPet extends Pet { + $beforeInsert() { + if (this.name === 'a') { + this.kind = 'cat'; + } + } + } + + await HookedPet.query(knex).insertGraph([{ name: 'a' }, { name: 'b' }]); + + expect( + await knex('graph_edge_pet').whereIn('name', ['a', 'b']).orderBy('name').select('kind'), + ).toEqual([{ kind: 'cat' }, { kind: 'plain' }]); + }); + + it('should move rows with their other changes, which unique keys may need', async () => { + await knex('graph_edge_pet').where('id', 3).update({ name: 'p1' }); + + await Person.query(knex).upsertGraph( + { id: 3, pets: [{ id: 3 }, { id: 1, name: 'z' }] }, + { relate: true }, + ); + + expect(await knex('graph_edge_pet').where('id', 1).first()).toMatchObject({ + name: 'z', + ownerId: 3, + }); + }); + + it('should move rows that wait for inserts with their other changes', async () => { + await knex('graph_edge_pet').where('id', 3).update({ name: 'p1' }); + + await Person.query(knex).upsertGraph( + { id: 3, pets: [{ id: 3 }, { id: 1, name: 'z', favoriteToy: { name: 'new toy' } }] }, + { relate: true }, + ); + + const toy = await knex('graph_edge_toy').where('name', 'new toy').first(); + expect(await knex('graph_edge_pet').where('id', 1).first()).toMatchObject({ + name: 'z', + ownerId: 3, + favoriteToyId: toy.id, + }); + }); + + it('should give the hooks of moved rows that wait for inserts their ids', async () => { + const parsedNames = []; + const updatedIds = []; + + class HookedPet extends Pet { + $parseJson(json, options) { + parsedNames.push(json.name); + return super.$parseJson(json, options); + } + + $beforeUpdate() { + updatedIds.push(this.id); + } + } + + class HookedPerson extends Person { + static get relationMappings() { + const { pets } = super.relationMappings; + return { pets: { ...pets, modelClass: HookedPet } }; + } + } + + await HookedPerson.query(knex).upsertGraph( + { id: 3, pets: [{ id: 3 }, { id: 1, name: 'z', favoriteToy: { name: 'new toy' } }] }, + { relate: true }, + ); + + expect(updatedIds).toEqual([1, 1]); + // The input is parsed once: + expect(parsedNames).toEqual([undefined, 'z']); + }); + + it('should resolve the references of moved rows to models that are not inserted', async () => { + await Person.query(knex).upsertGraph( + { + id: 3, + pets: [ + { '#id': 'p', id: 3, name: 'p3' }, + { id: 1, name: 'next to #ref{p.name}' }, + ], + }, + { relate: true, allowRefs: true }, + ); + + expect((await knex('graph_edge_pet').where('id', 1).first()).name).toBe('next to p3'); + }); + + it('should run the update hooks of moved rows once', async () => { + const calls = []; + + class HookedPet extends Pet { + $beforeUpdate() { + calls.push(this.ownerId); + } + } + + class HookedPerson extends Person { + static get relationMappings() { + const { pets } = super.relationMappings; + return { pets: { ...pets, modelClass: HookedPet } }; + } + } + + await HookedPerson.query(knex).upsertGraph( + { id: 3, pets: [{ id: 3 }, { id: 1 }] }, + { relate: true, noDelete: true }, + ); + + expect(calls).toEqual([3]); + }); + + it('should write the keys and references of moved rows that wait for inserts', async () => { + await knex('graph_edge_person').where('id', 3).update({ parentId: 1, bossId: 2 }); + + await Person.query(knex).upsertGraph( + { + id: 2, + children: [ + { + id: 3, + name: 'is #ref{x.name}', + boss: { name: 'new boss' }, + idols: [{ name: 'new idol' }], + }, + { '#id': 'x', name: 'X' }, + ], + }, + { relate: true, noDelete: true, allowRefs: true }, + ); + + const ids = Object.fromEntries( + (await knex('graph_edge_person')).map((person) => [person.name, person.id]), + ); + expect(await Person.query(knex).findById(3)).toMatchObject({ + name: 'is X', + parentId: 2, + bossId: ids['new boss'], + idolIds: [ids['new idol']], + }); + }); + + it('should write the keys of moved rows that wait for inserts with noUpdate', async () => { + await knex('graph_edge_person').where('id', 3).update({ parentId: 1, bossId: 2 }); + + await Person.query(knex).upsertGraph( + { id: 2, children: [{ id: 3, boss: { name: 'new boss' } }] }, + { relate: true, noDelete: true, noUpdate: true }, + ); + + const boss = await knex('graph_edge_person').where('name', 'new boss').first(); + expect(await knex('graph_edge_person').where('id', 3).first()).toMatchObject({ + parentId: 2, + bossId: boss.id, + }); + }); + + it('should not write the references of moved rows before they are resolved', async () => { + await Person.query(knex).upsertGraph( + { + id: 3, + pets: [ + { id: 3, toys: [{ '#id': 'toy', name: 'new toy' }] }, + { id: 1, favoriteToyId: '#ref{toy.id}' }, + ], + }, + { relate: true, allowRefs: true }, + ); + + const toy = await knex('graph_edge_toy').where('name', 'new toy').first(); + expect(await knex('graph_edge_pet').where('id', 1).first()).toMatchObject({ + ownerId: 3, + favoriteToyId: toy.id, + }); + }); + + it('should give new models the keys of related parents that the graph leaves out', async () => { + await Person.query(knex).upsertGraph( + { id: 1, children: [{ id: 2, petsByCode: [{ name: 'new' }] }] }, + { relate: true, noDelete: true }, + ); + + expect((await knex('graph_edge_pet').where('name', 'new').first()).ownerCode).toBe('friend'); + }); + + it('should give new models the keys of existing parents that the graph leaves out', async () => { + await Pet.query(knex).upsertGraph({ id: 1, owner: { id: 2, petsByCode: [{ name: 'new' }] } }); + + expect((await knex('graph_edge_pet').where('name', 'new').first()).ownerCode).toBe('friend'); + }); + + // Spreading the models into the arguments of a call overflows the stack. + it.skipIf(!session.isSqlite())( + 'should insert more models than a call takes arguments', + async () => { + const count = 200000; + await Toy.query(knex).insertGraph(Array.from({ length: count }, () => ({ name: 'toy' }))); + + expect(await knex('graph_edge_toy').where('name', 'toy').count('* as count')).toEqual([ + { count }, + ]); + }, + ); + }); + + describe('upsertGraph() patches that bind many values', () => { + const { knex } = session; + const columns = Array.from({ length: 80 }, (_, i) => `c${i}`); + + class Wide extends Model { + static get tableName() { + return 'graph_edge_wide'; + } + } + + beforeAll(async () => { + await knex.schema.dropTableIfExists('graph_edge_wide'); + await knex.schema.createTable('graph_edge_wide', (table) => { + table.integer('id').primary(); + + for (const column of columns) { + table.integer(column); + } + }); + }); + + afterAll(() => knex.schema.dropTableIfExists('graph_edge_wide')); + + it('should stay under the number of values that a query can bind', async () => { + const row = (id, value) => ({ + id, + ...Object.fromEntries(columns.map((column) => [column, value])), + }); + const ids = Array.from({ length: 500 }, (_, i) => i + 1); + + for (let i = 0; i < ids.length; i += 10) { + await knex('graph_edge_wide').insert(ids.slice(i, i + 10).map((id) => row(id, 0))); + } + + const queries = await collectQueries(() => + Wide.query(knex).upsertGraph(ids.map((id) => row(id, id))), + ); + // SQLite builds before 3.32 bind 999 values at most: + const maxBindings = session.isSqlite() + ? 999 + : session.isMsSql() + ? 2100 + : session.isPostgres() + ? 65535 + : Infinity; + + expect(queries.every(({ bindings }) => bindings.length <= maxBindings)).toBe(true); + + const rows = await knex('graph_edge_wide').orderBy('id'); + expect(rows.every((row) => columns.every((column) => row[column] === row.id))).toBe(true); + }); + }); + + describe.skipIf(!session.isPostgres())('upsertGraph() patches that a trigger skips', () => { + const { knex } = session; + + class Item extends Model { + static get tableName() { + return 'graph_edge_item'; + } + } + + beforeAll(async () => { + await knex.schema.dropTableIfExists('graph_edge_item'); + await knex.schema.createTable('graph_edge_item', (table) => { + table.integer('id').primary(); + table.string('name'); + }); + await knex.raw(` + create or replace function graph_edge_skip() returns trigger as $$ + begin + if new.name = 'skip' then + return null; + end if; + return new; + end; + $$ language plpgsql + `); + await knex.raw(` + create trigger graph_edge_skip before update on graph_edge_item + for each row execute function graph_edge_skip() + `); + }); + + afterAll(async () => { + await knex.schema.dropTableIfExists('graph_edge_item'); + await knex.raw('drop function if exists graph_edge_skip'); + }); + + it('should not throw for rows that exist', async () => { + await knex('graph_edge_item').insert([ + { id: 1, name: 'a' }, + { id: 2, name: 'b' }, + ]); + + await Item.query(knex).upsertGraph([ + { id: 1, name: 'skip' }, + { id: 2, name: 'c' }, + ]); + + expect(await knex('graph_edge_item').orderBy('id')).toEqual([ + { id: 1, name: 'a' }, + { id: 2, name: 'c' }, + ]); + }); + }); +}; diff --git a/tests/integration/misc/upsertGraphOrder.js b/tests/integration/misc/upsertGraphOrder.js index e231ea8a5..5236be8bc 100644 --- a/tests/integration/misc/upsertGraphOrder.js +++ b/tests/integration/misc/upsertGraphOrder.js @@ -230,6 +230,18 @@ export default (session) => { expect(await selectPets().select('id', 'name')).toEqual([{ id: 3, name: 'c' }]); }); + it('should give #ref models related into a HasManyRelation their foreign key (#311)', async () => { + await createTables(); + await knex('graph_order_person').insert({ id: 1 }); + + await Person.query().upsertGraph( + { id: 1, favoritePet: { '#id': 'x', id: 3, name: 'c' }, pets: [{ '#ref': 'x' }] }, + { relate: true, insertMissing: true, allowRefs: true }, + ); + + expect(await selectPets()).toEqual([{ id: 3, name: 'c', ownerId: 1 }]); + }); + describe('unique foreign key of a HasOneRelation', () => { it('should delete the current row before inserting its replacement', async () => { await createTables({ uniqueOwner: true }); diff --git a/tests/integration/misc/upsertGraphQueries.js b/tests/integration/misc/upsertGraphQueries.js new file mode 100644 index 000000000..70e2485f4 --- /dev/null +++ b/tests/integration/misc/upsertGraphQueries.js @@ -0,0 +1,162 @@ +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import { Model } from 'objection'; + +export default (session) => { + describe('upsertGraph() queries', () => { + const { knex } = session; + + class Pet extends Model { + static get tableName() { + return 'graph_queries_pet'; + } + } + + class Person extends Model { + static get tableName() { + return 'graph_queries_person'; + } + + static get relationMappings() { + return { + pets: { + relation: Model.HasManyRelation, + modelClass: Pet, + join: { from: 'graph_queries_person.id', to: 'graph_queries_pet.ownerId' }, + }, + }; + } + } + + const dropTables = () => + knex.schema.dropTableIfExists('graph_queries_pet').dropTableIfExists('graph_queries_person'); + + beforeAll(async () => { + await dropTables(); + await knex.schema + .createTable('graph_queries_person', (table) => { + table.integer('id').primary(); + }) + .createTable('graph_queries_pet', (table) => { + table.integer('id').primary(); + table.string('name'); + table.integer('age'); + table.integer('ownerId'); + }); + }); + + beforeEach(async () => { + await knex('graph_queries_pet').delete(); + await knex('graph_queries_person').delete(); + await knex('graph_queries_person').insert([{ id: 1 }, { id: 2 }]); + await knex('graph_queries_pet').insert([ + { id: 1, name: 'a', age: 1, ownerId: 1 }, + { id: 2, name: 'b', age: 2, ownerId: 1 }, + { id: 3, name: 'c', age: 3, ownerId: 2 }, + { id: 4, name: 'd', age: 4, ownerId: 2 }, + ]); + }); + + afterAll(dropTables); + + const collectQueries = async (callback) => { + const queries = []; + const onQuery = ({ sql }) => queries.push(sql.toLowerCase()); + knex.on('query', onQuery); + + try { + await callback(); + } finally { + knex.removeListener('query', onQuery); + } + + return queries; + }; + + const selectPets = () => knex('graph_queries_pet').select('id', 'name', 'age').orderBy('id'); + + it('should patch the rows of a table in one query', async () => { + const queries = await collectQueries(() => + Person.query(knex).upsertGraph([ + { + id: 1, + pets: [ + { id: 1, name: 'x' }, + { id: 2, age: 20 }, + ], + }, + { id: 2, pets: [{ id: 3, name: 'y', age: 30 }, { id: 4 }] }, + ]), + ); + + expect(queries.filter((sql) => sql.startsWith('update'))).toHaveLength(1); + expect(await selectPets()).toEqual([ + { id: 1, name: 'x', age: 1 }, + { id: 2, name: 'b', age: 20 }, + { id: 3, name: 'y', age: 30 }, + { id: 4, name: 'd', age: 4 }, + ]); + }); + + it('should patch the rows of models with update hooks one by one', async () => { + class HookedPet extends Pet { + $beforeUpdate() { + this.age = 99; + } + } + + class HookedPerson extends Person { + static get relationMappings() { + return { pets: { ...super.relationMappings.pets, modelClass: HookedPet } }; + } + } + + const queries = await collectQueries(() => + HookedPerson.query(knex).upsertGraph({ + id: 1, + pets: [ + { id: 1, name: 'x' }, + { id: 2, name: 'y' }, + ], + }), + ); + + expect(queries.filter((sql) => sql.startsWith('update'))).toHaveLength(2); + expect(await selectPets()).toEqual([ + { id: 1, name: 'x', age: 99 }, + { id: 2, name: 'y', age: 99 }, + { id: 3, name: 'c', age: 3 }, + { id: 4, name: 'd', age: 4 }, + ]); + }); + + // SQL Server binds at most 2100 values per query. + it('should patch and delete more rows than one query can bind', async () => { + const pets = Array.from({ length: 4200 }, (_, i) => ({ id: 10 + i, name: 'p', ownerId: 1 })); + + for (let i = 0; i < pets.length; i += 100) { + await knex('graph_queries_pet').insert(pets.slice(i, i + 100)); + } + + await Person.query(knex).upsertGraph({ + id: 1, + pets: pets.slice(0, 2100).map(({ id }) => ({ id, name: 'q' })), + }); + + const rows = await knex('graph_queries_pet').where('ownerId', 1); + expect(rows).toHaveLength(2100); + expect(rows.every((row) => row.name === 'q')).toBe(true); + }); + + it('should delete the removed rows of a relation in one query', async () => { + const queries = await collectQueries(() => + Person.query(knex).upsertGraph([ + { id: 1, pets: [{ id: 1 }] }, + { id: 2, pets: [{ id: 3 }] }, + ]), + ); + + expect(queries.filter((sql) => sql.startsWith('delete'))).toHaveLength(1); + expect((await selectPets()).map((pet) => pet.id)).toEqual([1, 3]); + }); + }); +}; diff --git a/tests/integration/misc/upsertGraphRefManyToMany.js b/tests/integration/misc/upsertGraphRefManyToMany.js new file mode 100644 index 000000000..9b034237b --- /dev/null +++ b/tests/integration/misc/upsertGraphRefManyToMany.js @@ -0,0 +1,86 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { Model } from 'objection'; + +export default (session) => { + describe('upsertGraph() #ref to an existing model in a ManyToManyRelation', () => { + const { knex } = session; + + class Pet extends Model { + static get tableName() { + return 'graph_ref_m2m_pet'; + } + } + + class Person extends Model { + static get tableName() { + return 'graph_ref_m2m_person'; + } + + static get relationMappings() { + return { + children: { + relation: Model.HasManyRelation, + modelClass: Person, + join: { from: 'graph_ref_m2m_person.id', to: 'graph_ref_m2m_person.parentId' }, + }, + pets: { + relation: Model.HasManyRelation, + modelClass: Pet, + join: { from: 'graph_ref_m2m_person.id', to: 'graph_ref_m2m_pet.ownerId' }, + }, + friends: { + relation: Model.ManyToManyRelation, + modelClass: Person, + join: { + from: 'graph_ref_m2m_person.id', + through: { from: 'graph_ref_m2m_friend.aId', to: 'graph_ref_m2m_friend.bId' }, + to: 'graph_ref_m2m_person.id', + }, + }, + }; + } + } + + const dropTables = () => + knex.schema + .dropTableIfExists('graph_ref_m2m_friend') + .dropTableIfExists('graph_ref_m2m_pet') + .dropTableIfExists('graph_ref_m2m_person'); + + beforeAll(async () => { + await dropTables(); + await knex.schema + .createTable('graph_ref_m2m_person', (table) => { + table.integer('id').primary(); + table.integer('parentId'); + }) + .createTable('graph_ref_m2m_pet', (table) => { + table.integer('id').primary(); + table.integer('ownerId'); + }) + .createTable('graph_ref_m2m_friend', (table) => { + table.integer('aId'); + table.integer('bId'); + }); + + await knex('graph_ref_m2m_person').insert([{ id: 1 }, { id: 2, parentId: 1 }]); + await knex('graph_ref_m2m_pet').insert([{ id: 10, ownerId: 2 }]); + }); + + afterAll(dropTables); + + it('should relate a #ref with relations to a model that the graph patches elsewhere', async () => { + await Person.query(knex).upsertGraph( + { + id: 1, + children: [{ '#id': 'child', id: 2 }], + friends: [{ '#ref': 'child', pets: [{ id: 10 }] }], + }, + { relate: true, allowRefs: true }, + ); + + expect(await knex('graph_ref_m2m_friend').select('aId', 'bId')).toEqual([{ aId: 1, bId: 2 }]); + expect(await knex('graph_ref_m2m_pet')).toEqual([{ id: 10, ownerId: 2 }]); + }); + }); +}; diff --git a/tests/integration/upsertGraph.js b/tests/integration/upsertGraph.js index 8c9c3787b..c1f6bdecb 100644 --- a/tests/integration/upsertGraph.js +++ b/tests/integration/upsertGraph.js @@ -1899,6 +1899,23 @@ export default (session) => { expect(result.model1Relation1.model1Relation1).not.toHaveProperty('#dbRef'); }); + it('should not return the markers of removed and skipped models', async () => { + const result = await Model1.query(session.knex).upsertGraph( + { + id: 2, + model1Relation2: [ + { idCol: 1, '#delete': true }, + { idCol: 2 }, + { '#id': 'skipped', model2Prop1: 'skipped' }, + ], + }, + { fetchStrategy, allowMarkers: true, noInsert: ['model1Relation2'] }, + ); + + expect(result.model1Relation2[0]).not.toHaveProperty('#delete'); + expect(result.model1Relation2[2]).not.toHaveProperty('#id'); + }); + it('should not fail when trying to #unrelate or #delete a model which either does not exist or is linked elsewhere', () => { const upsert = { id: 2, @@ -4314,6 +4331,25 @@ export default (session) => { }); }); + it('should throw before writing anything if a related model with relations does not exist', async () => { + const upsert = { + id: 2, + model1Prop1: 'updated root 2', + model1Relation3: [{ idCol: NONEXISTENT_ID, model2Relation3: [{ id: 1 }] }], + }; + + const err = await Model1.query(session.knex) + .upsertGraph(upsert, { relate: true, fetchStrategy }) + .catch((err) => err); + + expect(err).toBeInstanceOf(Model1.NotFoundError); + expect(err.message).toBe( + 'model (id=1000) does not exist. If you want to insert it with an id, use the insertMissing option', + ); + expect(err.data.dataPath).toEqual(['model1Relation3', 0]); + expect(await session.knex('Model1').where('model1Prop1', 'updated root 2')).toEqual([]); + }); + for (const [title, options] of [ [ 'flags', @@ -5504,6 +5540,30 @@ export default (session) => { }); }); + it('relate the same model twice with different extras (Vincit#1268)', async () => { + await Model2.query(session.knex).upsertGraph( + { + idCol: 2, + model2Relation1: [ + { id: 5, aliasedExtra: 'foo' }, + { id: 5, aliasedExtra: 'bar' }, + ], + }, + { relate: true, fetchStrategy }, + ); + + const rows = await session + .knex('Model1Model2') + .select('model1Id', 'extra3') + .where('model2Id', 2) + .orderBy('extra3'); + + expect(rows).toEqual([ + { model1Id: 5, extra3: 'bar' }, + { model1Id: 5, extra3: 'foo' }, + ]); + }); + it('update', () => { const upsert = { idCol: 2, diff --git a/tests/unit/queryBuilder/GraphOptions.js b/tests/unit/queryBuilder/GraphOptions.js index aaae84c59..6abb447bd 100644 --- a/tests/unit/queryBuilder/GraphOptions.js +++ b/tests/unit/queryBuilder/GraphOptions.js @@ -7,9 +7,8 @@ describe('GraphOptions', () => { return { relationPath, relationPathKey }; } - function matches(noInsert, relationPathKey, rootRelationPath) { - const options = new GraphOptions({ noInsert }, rootRelationPath); - return options._hasOption(createNode(relationPathKey), 'noInsert'); + function matches(noInsert, relationPathKey) { + return new GraphOptions({ noInsert }).has(createNode(relationPathKey), 'noInsert'); } describe('relation expressions', () => { @@ -41,12 +40,6 @@ describe('GraphOptions', () => { }); } - it('should match the full relation path in recursive upserts', () => { - expect(matches('a.b', 'b', ['a'])).toBe(true); - expect(matches('a.b', '', ['a'])).toBe(false); - expect(matches('a', '', ['a'])).toBe(true); - }); - for (const [expression, message] of [ ['', 'the expression is empty'], [' ', 'the expression is empty'], @@ -72,30 +65,4 @@ describe('GraphOptions', () => { }); } }); - - describe('rebasedOptions', () => { - function rebasedMatches(noInsert, rootRelationPathKey, relationPathKey) { - const options = new GraphOptions({ noInsert }).rebasedOptions( - createNode(rootRelationPathKey), - ); - return options._hasOption(createNode(relationPathKey), 'noInsert'); - } - - for (const noInsert of [['a', 'a.b', 'ab.c', 'x.a'], '[a.b, ab.c, x.a]']) { - it(`should match ${JSON.stringify(noInsert)} against the full relation path`, () => { - expect(rebasedMatches(noInsert, 'a', 'b')).toBe(true); - expect(rebasedMatches(noInsert, 'a', 'c')).toBe(false); - expect(rebasedMatches(noInsert, 'a', 'a')).toBe(false); - expect(rebasedMatches(noInsert, 'x', 'a')).toBe(true); - }); - } - - it('should match the new root', () => { - expect(rebasedMatches(['a'], 'a', '')).toBe(true); - expect(rebasedMatches('a', 'a', '')).toBe(true); - expect(rebasedMatches(['a'], 'b', '')).toBe(false); - expect(rebasedMatches(['a.b'], 'a.b', '')).toBe(true); - expect(rebasedMatches(['b'], 'a.b', '')).toBe(false); - }); - }); }); diff --git a/typings/objection/index.d.ts b/typings/objection/index.d.ts index e440176ac..c3d41f9da 100644 --- a/typings/objection/index.d.ts +++ b/typings/objection/index.d.ts @@ -2565,6 +2565,7 @@ export interface Relation { joinTableOwnerProp: RelationProperty; joinTableRelatedProp: RelationProperty; isOneToOne(): boolean; + keySide: 'owner' | 'related' | 'join'; } export interface RelationProperty {