Implement crdt for texts (#2)

Signed-off-by: Marcos Candeia <[email protected]>

src/bit.ts → src/crdt/bit.ts

@@ -1,28 +1,39 @@
 export class BinaryIndexedTree {
-  bit: number[];
+  bit: Map<number, number>;
+  upperLimit: number;
 
-  constructor(size: number) {
-    this.bit = new Array(size + 1).fill(0);
+  constructor(upperLimit: number = 10_000_000) {
+    this.bit = new Map();
+    this.upperLimit = upperLimit;
   }
 
-  // Updates the value at index i by adding delta to it
+  private increase(idx: number, delta: number): void {
+    let currValue = this.bit.get(idx) || 0;
+    this.bit.set(idx, currValue + delta);
+  }
+
+  // Updates the value at index idx by adding delta to it
   update(idx: number, delta: number): void {
     idx++; // Convert 0-based indexing to 1-based indexing
-    while (idx < this.bit.length) {
-      this.bit[idx] += delta;
+    while (idx <= this.upperLimit) {
+      this.increase(idx, delta);
       idx += idx & -idx; // Move to next index
     }
   }
 
-  // Returns the sum of values in the range [0, i]
-  query(r: number): number {
-    r++; // Convert 0-based indexing to 1-based indexing
-    let ret = 0;
+  private getSum(r: number): number {
+    let sum = 0;
     while (r > 0) {
-      ret += this.bit[r];
+      sum += this.bit.get(r) || 0;
       r -= r & -r; // Move to parent index
     }
-    return ret;
+    return sum;
+  }
+
+  // Returns the sum of values in the range [0, i]
+  query(r: number): number {
+    r++; // Convert 0-based indexing to 1-based indexing
+    return this.getSum(r);
   }
 
   // Returns the sum of values in the range [left, right]

src/crdt/text.ts

@@ -0,0 +1,154 @@
+import type { DeleteAtOperation, TextFilePatchOperation } from "../realtime.ts";
+import { BinaryIndexedTree } from "./bit.ts";
+
+const shrinkOperations = (
+  operations: TextFilePatchOperation[],
+): TextFilePatchOperation[] => {
+  const groupedOperations: TextFilePatchOperation[] = [];
+
+  let currentOperation: TextFilePatchOperation | undefined = operations[0];
+  let lastIdx = currentOperation?.at;
+  for (const operation of operations.slice(1)) {
+    const isConsecutive = operation.at - 1 === lastIdx;
+    if (isConsecutive) {
+      if (
+        "length" in currentOperation && "length" in operation
+      ) {
+        currentOperation.length += operation.length;
+        lastIdx++;
+        continue;
+      } else if ("text" in currentOperation && "text" in operation) {
+        currentOperation.text += operation.text;
+        lastIdx++;
+        continue;
+      }
+    }
+    groupedOperations.push(currentOperation);
+    currentOperation = operation;
+    lastIdx = currentOperation.at;
+  }
+
+  // Add the last operation to the grouped operations list
+  if (currentOperation) {
+    groupedOperations.push(currentOperation);
+  }
+
+  return groupedOperations;
+};
+// GENERATED BY GPT 4.0
+export const diff = (
+  oldStr: string,
+  newStr: string,
+): TextFilePatchOperation[] => {
+  const m = oldStr.length;
+  const n = newStr.length;
+  const dp: number[][] = Array.from(
+    { length: m + 1 },
+    () => Array(n + 1).fill(0),
+  );
+
+  // Building the DP matrix
+  for (let i = 1; i <= m; i++) {
+    for (let j = 1; j <= n; j++) {
+      if (oldStr[i - 1] === newStr[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1] + 1;
+      } else {
+        dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]);
+      }
+    }
+  }
+
+  const operations: TextFilePatchOperation[] = [];
+  let i = m, j = n;
+
+  // Trace back from dp[m][n]
+  while (i > 0 || j > 0) {
+    if (i > 0 && j > 0 && oldStr[i - 1] === newStr[j - 1]) {
+      i--;
+      j--;
+    } else if (j > 0 && (i === 0 || dp[i][j - 1] >= dp[i - 1][j])) {
+      let insertionLength = 1;
+      while (
+        j - insertionLength > 0 &&
+        newStr[j - insertionLength - 1] === newStr[j - 1]
+      ) {
+        insertionLength++;
+      }
+      operations.unshift({
+        at: i,
+        text: newStr.substr(j - insertionLength, insertionLength),
+      });
+      j -= insertionLength;
+    } else {
+      operations.unshift({ at: i - 1, length: 1 });
+      i--;
+    }
+  }
+
+  return shrinkOperations(operations);
+};
+
+const isDeleteOperation = (
+  op: TextFilePatchOperation,
+): op is DeleteAtOperation => {
+  return (op as DeleteAtOperation).length !== undefined;
+};
+
+export const applyPatch =
+  (offsets: BinaryIndexedTree, rollbacks: Array<() => void>) =>
+  (
+    [str, success]: [string, boolean],
+    op: TextFilePatchOperation,
+  ): [string, boolean] => {
+    if (!success) {
+      return [str, success];
+    }
+    if (isDeleteOperation(op)) {
+      const { at, length } = op;
+      const offset = offsets.query(at) + at;
+      if (offset < 0) {
+        return [str, false];
+      }
+      const before = str.slice(0, offset);
+      const after = str.slice(offset + length);
+
+      // Update BIT for deletion operation
+      offsets.update(at, -length); // Subtract length from the index
+      rollbacks.push(() => {
+        offsets.update(at, length);
+      });
+      return [`${before}${after}`, true];
+    }
+    const { at, text } = op;
+    const offset = offsets.query(at) + at;
+    if (offset < 0) {
+      return [str, false];
+    }
+
+    const before = str.slice(0, offset);
+    const after = str.slice(offset); // Use offset instead of at
+
+    // Update BIT for insertion operation
+    offsets.update(at, text.length); // Add length of text at the index
+    rollbacks.push(() => {
+      offsets.update(at, -text.length);
+    });
+    return [`${before}${text}${after}`, true];
+  };
+
+export const apply = (
+  str: string,
+  ops: TextFilePatchOperation[],
+  offsets?: BinaryIndexedTree,
+): [string, boolean] => {
+  const rollbacks: Array<() => void> = [];
+  const [result, success] = ops.reduce(
+    applyPatch(offsets ?? new BinaryIndexedTree(), rollbacks),
+    [str, true],
+  );
+  if (!success) {
+    rollbacks.forEach((rb) => rb());
+  }
+
+  return [result, success];
+};

src/realtime.ts

@@ -1,5 +1,6 @@
 import { applyReducer, type Operation } from "fast-json-patch";
-import { BinaryIndexedTree } from "./bit.ts";
+import { BinaryIndexedTree } from "./crdt/bit.ts";
+import { apply } from "./crdt/text.ts";
 import { type Env } from "./index.ts";
 import { createRouter, Router, Routes } from "./router.ts";
 
@@ -20,15 +21,6 @@ export interface BaseFilePatch {
 }
 export type TextFilePatchOperation = InsertAtOperation | DeleteAtOperation;
 
-export interface TextFilePatch extends BaseFilePatch {
-  operations: TextFilePatchOperation[];
-  timestamp: number;
-}
-
-export interface TextFileSet extends BaseFilePatch {
-  content: string | null;
-}
-
 export interface TextFielPatchOperationBase {
   at: number;
 }
@@ -40,12 +32,14 @@ export interface InsertAtOperation extends TextFielPatchOperationBase {
 export interface DeleteAtOperation extends TextFielPatchOperationBase {
   length: number;
 }
+export interface TextFilePatch extends BaseFilePatch {
+  operations: TextFilePatchOperation[];
+  timestamp: number;
+}
 
-const isDeleteOperation = (
-  op: TextFilePatchOperation,
-): op is DeleteAtOperation => {
-  return (op as DeleteAtOperation).length !== undefined;
-};
+export interface TextFileSet extends BaseFilePatch {
+  content: string | null;
+}
 
 export type FilePatch = JSONFilePatch | TextFilePatch | TextFileSet;
 
@@ -393,48 +387,9 @@ export class Realtime implements DurableObject {
                 results.push({ accepted: false, path, content });
                 continue;
               }
-              const rollbacks: Array<() => void> = [];
               const bit = this.textState.get(timestamp) ??
-                new BinaryIndexedTree(2 ** 8);
-              const [result, success] = operations.reduce(
-                ([txt, success], op) => {
-                  if (!success) {
-                    return [txt, success];
-                  }
-                  if (isDeleteOperation(op)) {
-                    const { at, length } = op;
-                    const offset = bit.rangeQuery(0, at) + at;
-                    if (offset < 0) {
-                      return [txt, false];
-                    }
-                    const before = txt.slice(0, offset);
-                    const after = txt.slice(offset + length);
-
-                    // Update BIT for deletion operation
-                    bit.update(at, -length); // Subtract length from the index
-                    rollbacks.push(() => {
-                      bit.update(at, length);
-                    });
-                    return [`${before}${after}`, true];
-                  }
-                  const { at, text } = op;
-                  const offset = bit.rangeQuery(0, at) + at;
-                  if (offset < 0) {
-                    return [txt, false];
-                  }
-
-                  const before = txt.slice(0, offset);
-                  const after = txt.slice(offset); // Use offset instead of at
-
-                  // Update BIT for insertion operation
-                  bit.update(at, text.length); // Add length of text at the index
-                  rollbacks.push(() => {
-                    bit.update(at, -text.length);
-                  });
-                  return [`${before}${text}${after}`, true];
-                },
-                [content, true],
-              );
+                new BinaryIndexedTree();
+              const [result, success] = apply(content, operations, bit);
               if (success) {
                 this.textState.set(timestamp, bit);
                 results.push({
@@ -443,7 +398,6 @@ export class Realtime implements DurableObject {
                   content: result,
                 });
               } else {
-                rollbacks.map((rollback) => rollback());
                 results.push({
                   accepted: false,
                   path,
@@ -454,7 +408,7 @@ export class Realtime implements DurableObject {
           }
 
           this.timestamp = Date.now();
-          this.textState.set(this.timestamp, new BinaryIndexedTree(2 ** 8));
+          this.textState.set(this.timestamp, new BinaryIndexedTree());
           const shouldWrite = results.every((r) => r.accepted);
 
           if (shouldWrite) {

test.ts

@@ -1,4 +1,5 @@
 import * as fjp from "fast-json-patch";
+import { diff } from "./src/crdt/text.ts";
 import type {
   FileSystemNode,
   ServerEvent,
@@ -94,6 +95,33 @@ const assertAll = (...elems: unknown[]) => {
 };
 
 const tests = {
+  "diff should calculate text diff": async () => {
+    const from = "BC";
+    const to = "ABC";
+    assertEquals(
+      JSON.stringify(diff(from, to)),
+      JSON.stringify([{ at: 0, text: "A" }]),
+    );
+  },
+  "diff should calculate text diff with deletions": async () => {
+    const from = "BC";
+    const to = "AB";
+    assertEquals(
+      JSON.stringify(diff(from, to)),
+      JSON.stringify([{ at: 0, text: "A" }, { at: 1, length: 1 }]),
+    );
+  },
+  "diff should calculate text longer texts": async () => {
+    const from = "Lorem ipsum abcd !";
+    const to = "Lom ips!!!um abcd";
+    assertEquals(
+      JSON.stringify(diff(from, to)),
+      JSON.stringify([{ at: 2, length: 2 }, { at: 9, text: "!!!" }, {
+        at: 16,
+        length: 2,
+      }]),
+    );
+  },
   "Should be accepted": async () => {
     const { results } = await realtime.patch({
       patches: [
@@ -209,10 +237,10 @@ const tests = {
       patches: [
         {
           path: shelf,
-          operations: [{
+          operations: [{ // ABC!
             text: "!",
             at: 3,
-          }, {
+          }, { // AB!
             length: 1,
             at: 2,
           }],