Allow advanced modifications of properties in Collection.modify (#1910)

Add a new class PropModification with capabilities of modifying properties into computed values based on the previous value, to improve the Operation-based CRDTs support that is already possible with where clauses and `Collection.modify()` but with more precise operations than just replacing a property to another constant value.

The first operation to support is `replacePrefix` that replaces the leading string of a string. This PR also exports a helper function `replacePrefix()` to use in modify operations that returns an instance of PropModification configured to do this operation. Future operations could include `increment` (for numbers), `push` (for arrays), `add` (for unique arrays / sets), `remove` for arrays, and `updateArrayItem` for arrays with object items.

The `replacePrefix` operation could seem a bit special-casey, but the reality is that it solves an important pattern when working with tree structures and letting an indexed property represent the tree path. Using this new operation, it becomes possible to move an entire sub tree in a single modify operation:

```js
db.files
  .where('path')
  .startsWith('old/path')
  .modify({
    path: replacePrefix('old/path', 'new/other/path')
  });
```

Even though this can already be accomplished using a JS callback though doing the equivalent operation, it would not fully propagate the operation to DBCore in order to implement CRDT capabilities:

```js
db.files
  .where('path')
  .startsWith('old/path')
  .modify(file => {
    file.path = 'new/other/path' + file.path.substring('old/path'.length);
  });
```

Using this JS function would not propagate the operation to DBCore because a JS function cannot be interpreted safely for several reasons (security but also missing closures) - so sync implementations would only get the resulting individual put() operations that resulted locally from the modify operation, and would therefore not be able to consistently sync the tree move operation with the server in a truly consistent manner that would resolve correctly across multiple clients in case other clients have added, moved or deleted nodes while being temporarily offline.

Adding this new operation will allow moving trees and propagate the operation consistently in synced environments that supports and propagate the `criteria` and `changeSpec` properties in the resulting put operation to DBCore and further down into sync implementations. Currently only Dexie Cloud Server will respect this and handle it consistently, but the information is available in DBCore to implement in other endpoints.

src/classes/collection/collection.ts

@@ -12,6 +12,8 @@ import { ThenShortcut } from "../../public/types/then-shortcut";
 import { Transaction } from '../transaction';
 import { DBCoreCursor, DBCoreTransaction, DBCoreRangeType, DBCoreMutateResponse, DBCoreKeyRange } from '../../public/types/dbcore';
 import { cmp } from "../../functions/cmp";
+import { PropModification } from "../../helpers/prop-modification";
+import { UpdateSpec } from "../../public/types/update-spec";
 
 /** class Collection
  * 
@@ -457,23 +459,28 @@ export class Collection implements ICollection {
    * https://dexie.org/docs/Collection/Collection.modify()
    * 
    **/
-  modify(changes: { [keyPath: string]: any }) : PromiseExtended<number>
-  modify(changes: (obj: any, ctx:{value: any, primKey: IndexableType}) => void | boolean): PromiseExtended<number> {
+  modify(changes: UpdateSpec<any> | ((obj: any, ctx:{value: any, primKey: IndexableType}) => void | boolean)): PromiseExtended<number> {
     var ctx = this._ctx;
     return this._write(trans => {
       var modifyer: (obj: any, ctx:{value: any, primKey: IndexableType}) => void | boolean
       if (typeof changes === 'function') {
         // Changes is a function that may update, add or delete propterties or even require a deletion the object itself (delete this.item)
-        modifyer = changes;
+        modifyer = changes as (obj: any, ctx:{value: any, primKey: IndexableType}) => void | boolean;
       } else {
         // changes is a set of {keyPath: value} and no one is listening to the updating hook.
         var keyPaths = keys(changes);
         var numKeys = keyPaths.length;
         modifyer = function (item) {
-          var anythingModified = false;
-          for (var i = 0; i < numKeys; ++i) {
-            var keyPath = keyPaths[i], val = changes[keyPath];
-            if (getByKeyPath(item, keyPath) !== val) {
+          let anythingModified = false;
+          for (let i = 0; i < numKeys; ++i) {
+            let keyPath = keyPaths[i];
+            let val = changes[keyPath];
+            let origVal = getByKeyPath(item, keyPath);
+
+            if (val instanceof PropModification) {
+              setByKeyPath(item, keyPath, val.execute(origVal));
+              anythingModified = true;
+            } else if (origVal !== val) {
               setByKeyPath(item, keyPath, val); // Adding {keyPath: undefined} means that the keyPath should be deleted. Handled by setByKeyPath
               anythingModified = true;
             }

src/functions/replace-prefix.ts

@@ -0,0 +1,5 @@
+import { PropModification } from "../helpers/prop-modification";
+
+export function replacePrefix(a: string, b:string) {
+  return new PropModification({replacePrefix: [a, b]});
+}

src/helpers/prop-modification.ts

@@ -0,0 +1,20 @@
+import { PropModSpec } from "../public/types/prop-modification";
+
+export const PropModSymbol: unique symbol = Symbol();
+
+export class PropModification implements PropModSpec {
+  [PropModSymbol]?: true;
+  replacePrefix?: [string, string];
+
+  execute(value: any) {
+    const prefixToReplace = this.replacePrefix?.[0];
+    if (prefixToReplace && typeof value === 'string' && value.startsWith(prefixToReplace)) {
+      return this.replacePrefix[1] + value.substring(prefixToReplace.length);
+    }
+    return value;
+  }
+
+  constructor(spec: PropModSpec) {
+    Object.assign(this, spec);
+  }
+}

src/index.ts

@@ -13,6 +13,9 @@ import './support-bfcache';
 import { liveQuery } from './live-query/live-query';
 import { Entity } from './classes/entity/Entity';
 import { cmp } from './functions/cmp';
+import { PropModification, PropModSymbol } from './helpers/prop-modification';
+import { replacePrefix } from './functions/replace-prefix';
+
 
 // Set rejectionMapper of DexiePromise so that it generally tries to map
 // DOMErrors and DOMExceptions to a DexieError instance with same name but with
@@ -27,4 +30,5 @@ export { RangeSet, mergeRanges, rangesOverlap } from "./helpers/rangeset";
 export { Dexie, liveQuery }; // Comply with public/index.d.ts.
 export { Entity };
 export { cmp };
+export { PropModSymbol, PropModification, replacePrefix };
 export default Dexie;

src/public/index.d.ts

@@ -26,6 +26,8 @@ import { Observable } from './types/observable';
 import { IntervalTree, RangeSetConstructor } from './types/rangeset';
 import { Dexie, TableProp } from './types/dexie';
 export type { TableProp };
+import { PropModification, PropModSpec, PropModSymbol } from './types/prop-modification';
+export { PropModification, PropModSpec, PropModSymbol };
 export * from './types/entity';
 export * from './types/entity-table';
 export { UpdateSpec } from './types/update-spec';
@@ -62,6 +64,7 @@ export function rangesOverlap(
 ): boolean;
 declare var RangeSet: RangeSetConstructor;
 export function cmp(a: any, b: any): number;
+export function replacePrefix(a: string, b: string): PropModification;
 
 /** Exporting 'Dexie' as the default export.
  **/

src/public/types/prop-modification.d.ts

@@ -0,0 +1,12 @@
+export declare const PropModSymbol: unique symbol;
+
+export type PropModSpec = {
+  replacePrefix?: [string, string];
+}
+
+export class PropModification implements PropModSpec {
+  [PropModSymbol]?: true;
+  replacePrefix?: [string, string];
+
+  constructor(spec: PropModSpec);
+}

src/public/types/update-spec.d.ts

@@ -1,3 +1,4 @@
 import { KeyPaths, KeyPathValue } from "./keypaths";
+import { PropModification } from "./prop-modification";
 
-export type UpdateSpec<T> = { [KP in KeyPaths<T>]?: KeyPathValue<T, KP> };
+export type UpdateSpec<T> = { [KP in KeyPaths<T>]?: KeyPathValue<T, KP> | PropModification };

test/deepEqual.js

@@ -0,0 +1,24 @@
+import { equal } from 'QUnit';
+import sortedJSON from "sorted-json";
+import { deepClone } from '../src/functions/utils';
+
+export function deepEqual(actual, expected, description) {
+  actual = JSON.parse(JSON.stringify(actual));
+  expected = JSON.parse(JSON.stringify(expected));
+  actual = sortedJSON.sortify(actual, { sortArray: false });
+  expected = sortedJSON.sortify(expected, { sortArray: false });
+  equal(JSON.stringify(actual, null, 2), JSON.stringify(expected, null, 2), description);
+}
+export function isDeepEqual(actual, expected, allowedExtra, prevActual) {
+  actual = deepClone(actual);
+  expected = deepClone(expected);
+  if (allowedExtra) Array.isArray(allowedExtra) ? allowedExtra.forEach(key => {
+    if (actual[key]) expected[key] = deepClone(prevActual[key]);
+  }) : Object.keys(allowedExtra).forEach(key => {
+    if (actual[key]) expected[key] = deepClone(allowedExtra[key]);
+  });
+
+  actual = sortedJSON.sortify(actual, { sortArray: false });
+  expected = sortedJSON.sortify(expected, { sortArray: false });
+  return JSON.stringify(actual, null, 2) === JSON.stringify(expected, null, 2);
+}

test/tests-collection.js

@@ -1,9 +1,10 @@
-import Dexie from 'dexie';
+import Dexie, {replacePrefix} from 'dexie';
 import {module, stop, start, test, asyncTest, equal, ok} from 'QUnit';
 import {resetDatabase, supports, spawnedTest, promisedTest} from './dexie-unittest-utils';
+import { deepEqual } from './deepEqual';
 
 var db = new Dexie("TestDBCollection");
-db.version(1).stores({ users: "id,first,last,[foo+bar],&username,*&email,*pets" });
+db.version(1).stores({ users: "id,first,last,[foo+bar],&username,*&email,*pets,parentPath" });
 
 var User = db.users.defineClass({
     id:         Number,
@@ -633,4 +634,39 @@ promisedTest("Issue 1381: Collection.filter().primaryKeys() on virtual index", a
     const ids = await db.users.where({foo: "A"}).filter(x => true).primaryKeys();
     ok(ids.length === 1, "Theres one id there");
     equal(ids[0], 1000, "The ID is 1000");
+});
+
+promisedTest("Collection.modify() with replace", async () => {
+    await db.users.bulkAdd([
+        {id: 1000, foo: "A", parentPath: "A/B/C"},
+        {id: 1001, foo: "B", parentPath: "A/B/C"},
+        {id: 1002, foo: "C", parentPath: "A/B"}
+    ]);
+    await db.users.where('parentPath').startsWith("A/B").modify({
+        parentPath: replacePrefix("A/B", "X")
+    });
+    let users = await db.users.where('id').between(1000, 1003).toArray();
+    deepEqual(users, [
+        {id: 1000, foo: "A", parentPath: "X/C"},
+        {id: 1001, foo: "B", parentPath: "X/C"},
+        {id: 1002, foo: "C", parentPath: "X"}
+    ], "All three have moved from A/B to X");
+    await db.users.where('parentPath').startsWith("X/C").modify({
+        parentPath: replacePrefix("X/C", "Y")
+    });
+    users = await db.users.where('id').between(1000, 1003).toArray();
+    deepEqual(users, [
+        {id: 1000, foo: "A", parentPath: "Y"},
+        {id: 1001, foo: "B", parentPath: "Y"},
+        {id: 1002, foo: "C", parentPath: "X"}
+    ], "The first two have moved from X/C to Y");
+    await db.users.toCollection().modify({
+        parentPath: replacePrefix("X", "Z")
+    });
+    users = await db.users.where('id').between(1000, 1003).toArray();
+    deepEqual(users, [
+        {id: 1000, foo: "A", parentPath: "Y"},
+        {id: 1001, foo: "B", parentPath: "Y"},
+        {id: 1002, foo: "C", parentPath: "Z"}
+    ], "Omitting where-criteria will still check the prefix before replacing");
 });

test/tests-live-query.js

@@ -1,10 +1,9 @@
 import Dexie, {liveQuery} from 'dexie';
 import {module, stop, start, asyncTest, equal, ok} from 'QUnit';
 import {resetDatabase, supports, promisedTest, isIE} from './dexie-unittest-utils';
-import sortedJSON from "sorted-json";
 import {from} from "rxjs";
 import {map} from "rxjs/operators";
-import { deepClone } from '../src/functions/utils';
+import { deepEqual, isDeepEqual } from './deepEqual';
 
 const db = new Dexie("TestLiveQuery", {
   cache: 'immutable' // Using immutable cache in tests because it is most likely to fail if not using properly.
@@ -38,28 +37,6 @@ function objectify(map) {
   return rv;
 }
 
-export function deepEqual(actual, expected, description) {
-  actual = JSON.parse(JSON.stringify(actual));
-  expected = JSON.parse(JSON.stringify(expected));
-  actual = sortedJSON.sortify(actual, {sortArray: false});
-  expected = sortedJSON.sortify(expected, {sortArray: false});
-  equal(JSON.stringify(actual, null, 2), JSON.stringify(expected, null, 2), description);
-}
-
-function isDeepEqual(actual, expected, allowedExtra, prevActual) {
-  actual = deepClone(actual);
-  expected = deepClone(expected)
-  if (allowedExtra) Array.isArray(allowedExtra) ? allowedExtra.forEach(key => {
-    if (actual[key]) expected[key] = deepClone(prevActual[key]);
-  }) : Object.keys(allowedExtra).forEach(key => {
-    if (actual[key]) expected[key] = deepClone(allowedExtra[key]);
-  });
-
-  actual = sortedJSON.sortify(actual, {sortArray: false});
-  expected = sortedJSON.sortify(expected, {sortArray: false});
-  return JSON.stringify(actual, null, 2) === JSON.stringify(expected, null, 2);
-}
-
 class Signal {
   promise = new Promise(resolve => this.resolve = resolve);
 }

test/tsconfig.json

@@ -16,6 +16,6 @@
     "downlevelIteration": true
   },
   "files": [
-    "tests-all.js"
+    "tests-all.js", "./deepEqual.js"
   ]
 }

test/typings-test/test-typings.ts

@@ -3,7 +3,7 @@
  * It tests Dexie.d.ts.
  */
 
-import Dexie, { IndexableType, Table } from '../../dist/dexie'; // Imports the source Dexie.d.ts file
+import Dexie, { IndexableType, Table, replacePrefix } from '../../dist/dexie'; // Imports the source Dexie.d.ts file
 import './test-extend-dexie';
 import './test-updatespec';
 
@@ -257,3 +257,21 @@ import './test-updatespec';
         trans.abort();
     });
 }
+
+
+{
+    // Replace modification:
+
+    interface Friend {
+        id?: number;
+        name: string;
+        isGoodFriend: boolean;
+        address: {
+            city: string;
+        }
+    }
+
+    let db = new Dexie('dbname') as Dexie & {friends: Table<Friend, number>};
+
+    db.friends.where({name: 'Kalle'}).modify({name: replacePrefix('K', 'C')});
+}