♻️ (crdt) moved re-used utils in actual-server to separate package (#1150)

actual-server does not need to import the full actual-app/api package.
It can import only the CRDT stuff.. so I'm extracting it into a new
package to reduce the size of actual-server and make the link between
things more transparent.

.github/workflows/build.yml

@@ -31,6 +31,22 @@ jobs:
           name: actual-api
           path: packages/api/actual-api.tgz
 
+  crdt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Build CRDT
+        run: cd packages/crdt && yarn build
+      - name: Create package tgz
+        run: cd packages/crdt && yarn pack && mv package.tgz actual-crdt.tgz
+      - name: Upload Build
+        uses: actions/upload-artifact@v3
+        with:
+          name: actual-crdt
+          path: packages/crdt/actual-crdt.tgz
+
   web:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -2,6 +2,7 @@
 !data/.gitkeep
 /data2
 packages/api/dist
+packages/crdt/dist
 packages/desktop-electron/client-build
 packages/desktop-electron/.electron-symbols
 packages/desktop-electron/dist

packages/crdt/.eslintignore

@@ -0,0 +1 @@
+dist

packages/crdt/index.ts

@@ -0,0 +1 @@
+export * from './src/main';

packages/crdt/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  preset: 'ts-jest/presets/js-with-ts-esm',
+  testEnvironment: 'node',
+};

packages/crdt/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@actual-app/crdt",
+  "version": "1.0.0",
+  "license": "MIT",
+  "description": "CRDT layer of Actual",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "lint": "eslint .",
+    "build:node": "tsc --p tsconfig.dist.json",
+    "build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/",
+    "test": "jest -c jest.config.js"
+  },
+  "dependencies": {
+    "google-protobuf": "^3.12.0-rc.1",
+    "murmurhash": "^0.0.2",
+    "uuid": "3.3.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^27.5.0",
+    "jest": "^27.0.0",
+    "ts-jest": "^27.0.0",
+    "typescript": "^5.0.2"
+  }
+}

packages/crdt/src/crdt/timestamp.ts

@@ -1,6 +1,5 @@
 import murmurhash from 'murmurhash';
-
-import * as uuid from '../../platform/uuid';
+import uuid from 'uuid';
 
 /**
  * Hybrid Unique Logical Clock (HULC) timestamp generator
@@ -65,7 +64,7 @@ export function deserializeClock(clock) {
 }
 
 export function makeClientId() {
-  return uuid.v4Sync().replace(/-/g, '').slice(-16);
+  return uuid.v4().replace(/-/g, '').slice(-16);
 }
 
 let config = {

packages/crdt/src/main.ts

@@ -0,0 +1,13 @@
+import * as SyncPb from './proto/sync_pb';
+export {
+  merkle,
+  getClock,
+  setClock,
+  makeClock,
+  makeClientId,
+  serializeClock,
+  deserializeClock,
+  Timestamp,
+} from './crdt';
+
+export const SyncProtoBuf = SyncPb;

packages/crdt/tsconfig.dist.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    // Using ES2021 because that’s the newest version where
+    // the latest Node 16.x release supports all of the features
+    "target": "es2021",
+    "module": "CommonJS",
+    "noEmit": false,
+    "declaration": true,
+    "outDir": "dist"
+  },
+  "include": ["."],
+  "exclude": ["dist"]
+}

packages/loot-core/package.json

@@ -31,7 +31,6 @@
     "csv-parse": "^4.10.1",
     "csv-stringify": "^5.3.6",
     "deep-equal": "^2.0.5",
-    "google-protobuf": "^3.12.0-rc.1",
     "md5": "^2.3.0",
     "mitt": "^3.0.0",
     "node-fetch": "^2.6.9",
@@ -43,6 +42,7 @@
   },
   "devDependencies": {
     "@actual-app/api": "*",
+    "@actual-app/crdt": "*",
     "@actual-app/import-ynab4": "*",
     "@babel/core": "~7.22.5",
     "@babel/preset-env": "^7.22.5",
@@ -65,7 +65,6 @@
     "memfs": "3.1.1",
     "memoize-one": "^4.0.0",
     "mockdate": "^3.0.5",
-    "murmurhash": "^0.0.2",
     "npm-run-all": "^4.1.3",
     "peggy": "3.0.2",
     "snapshot-diff": "^0.10.0",

packages/loot-core/src/mocks/setup.ts

@@ -52,9 +52,11 @@ global.resetRandomId = () => {
   _id = 1;
 };
 
-global.randomId = () => {
-  return 'id' + _id++;
-};
+jest.mock('uuid', () => ({
+  v4: () => {
+    return 'id' + _id++;
+  },
+}));
 
 global.getDatabaseDump = async function (tables) {
   if (!tables) {

packages/loot-core/src/platform/uuid/index.testing.ts

@@ -1,9 +0,0 @@
-import type * as T from '.';
-
-export const v4: T.V4 = function () {
-  return Promise.resolve(global.randomId());
-};
-
-export const v4Sync: T.V4Sync = function () {
-  return global.randomId();
-};

packages/loot-core/src/server/api.ts

@@ -1,3 +1,5 @@
+import { getClock } from '@actual-app/crdt';
+
 import * as connection from '../platform/server/connection';
 import {
   getDownloadError,
@@ -22,7 +24,6 @@ import {
 } from './api-models';
 import { runQuery as aqlQuery } from './aql';
 import * as cloudStorage from './cloud-storage';
-import { getClock } from './crdt';
 import * as db from './db';
 import { runMutator } from './mutators';
 import * as prefs from './prefs';

packages/loot-core/src/server/aql/schema/executors.test.ts

@@ -1,9 +1,9 @@
+import { setClock } from '@actual-app/crdt';
 import fc from 'fast-check';
 
 import * as arbs from '../../../mocks/arbitrary-schema';
 import query from '../../../shared/query';
 import { groupById } from '../../../shared/util';
-import { setClock } from '../../crdt';
 import * as db from '../../db';
 import { batchMessages, setSyncingMode } from '../../sync/index';
 

packages/loot-core/src/server/db/index.ts

@@ -1,3 +1,11 @@
+import {
+  makeClock,
+  setClock,
+  serializeClock,
+  deserializeClock,
+  makeClientId,
+  Timestamp,
+} from '@actual-app/crdt';
 import LRU from 'lru-cache';
 
 import * as fs from '../../platform/server/fs';
@@ -11,14 +19,6 @@ import {
   convertForUpdate,
   convertFromSelect,
 } from '../aql';
-import {
-  makeClock,
-  setClock,
-  serializeClock,
-  deserializeClock,
-  makeClientId,
-  Timestamp,
-} from '../crdt';
 import {
   accountModel,
   categoryModel,

packages/loot-core/src/server/main.test.ts

@@ -1,3 +1,5 @@
+import { getClock, deserializeClock } from '@actual-app/crdt';
+
 import { expectSnapshotWithDiffer } from '../mocks/util';
 import * as connection from '../platform/server/connection';
 import * as fs from '../platform/server/fs';
@@ -5,7 +7,6 @@ import * as monthUtils from '../shared/months';
 
 import * as budgetActions from './budget/actions';
 import * as budget from './budget/base';
-import { getClock, deserializeClock } from './crdt';
 import * as db from './db';
 import { handlers } from './main';
 import {

packages/loot-core/src/server/main.ts

@@ -1,5 +1,6 @@
 import './polyfills';
 import * as injectAPI from '@actual-app/api/injected';
+import * as CRDT from '@actual-app/crdt';
 import * as YNAB4 from '@actual-app/import-ynab4/importer';
 import * as YNAB5 from '@actual-app/import-ynab5/importer';
 
@@ -37,16 +38,6 @@ import {
 import budgetApp from './budget/app';
 import * as budget from './budget/base';
 import * as cloudStorage from './cloud-storage';
-import {
-  getClock,
-  setClock,
-  makeClock,
-  makeClientId,
-  serializeClock,
-  deserializeClock,
-  Timestamp,
-  merkle,
-} from './crdt';
 import * as db from './db';
 import * as mappings from './db/mappings';
 import * as encryption from './encryption';
@@ -73,7 +64,6 @@ import {
   repairSync,
 } from './sync';
 import * as syncMigrations from './sync/migrate';
-import * as SyncPb from './sync/proto/sync_pb';
 import toolsApp from './tools/app';
 import { withUndo, clearUndo, undo, redo } from './undo';
 import { updateVersion } from './update';
@@ -2214,10 +2204,10 @@ async function loadBudget(id) {
     //
     // TODO: The client id should be stored elsewhere. It shouldn't
     // work this way, but it's fine for now.
-    getClock().timestamp.setNode(makeClientId());
+    CRDT.getClock().timestamp.setNode(CRDT.makeClientId());
     await db.runQuery(
       'INSERT OR REPLACE INTO messages_clock (id, clock) VALUES (1, ?)',
-      [serializeClock(getClock())],
+      [CRDT.serializeClock(CRDT.getClock())],
     );
 
     await prefs.savePrefs({ resetClock: false });
@@ -2502,15 +2492,7 @@ export const lib = {
   db,
 
   // Expose CRDT mechanisms so server can use them
-  merkle,
-  timestamp: {
-    getClock,
-    setClock,
-    makeClock,
-    makeClientId,
-    serializeClock,
-    deserializeClock,
-    Timestamp,
-  },
-  SyncProtoBuf: SyncPb,
+  // Backwards compatability
+  ...CRDT,
+  timestamp: CRDT,
 };

packages/loot-core/src/server/prefs.ts

@@ -1,6 +1,7 @@
+import { Timestamp } from '@actual-app/crdt';
+
 import * as fs from '../platform/server/fs';
 
-import { Timestamp } from './crdt';
 import { sendMessages } from './sync';
 
 let prefs = null;

packages/loot-core/src/server/sync/encoder.ts

@@ -1,9 +1,9 @@
+import { SyncProtoBuf } from '@actual-app/crdt';
+
 import * as encryption from '../encryption';
 import { SyncError } from '../errors';
 import * as prefs from '../prefs';
 
-import * as SyncPb from './proto/sync_pb';
-
 function coerceBuffer(value) {
   // The web encryption APIs give us back raw Uint8Array... but our
   // encryption code assumes we can work with it as a buffer. This is
@@ -17,14 +17,14 @@ function coerceBuffer(value) {
 
 export async function encode(groupId, fileId, since, messages) {
   let { encryptKeyId } = prefs.getPrefs();
-  let requestPb = new SyncPb.SyncRequest();
+  let requestPb = new SyncProtoBuf.SyncRequest();
 
   for (let i = 0; i < messages.length; i++) {
     let msg = messages[i];
-    let envelopePb = new SyncPb.MessageEnvelope();
+    let envelopePb = new SyncProtoBuf.MessageEnvelope();
     envelopePb.setTimestamp(msg.timestamp);
 
-    let messagePb = new SyncPb.Message();
+    let messagePb = new SyncProtoBuf.Message();
     messagePb.setDataset(msg.dataset);
     messagePb.setRow(msg.row);
     messagePb.setColumn(msg.column);
@@ -32,7 +32,7 @@ export async function encode(groupId, fileId, since, messages) {
     let binaryMsg = messagePb.serializeBinary();
 
     if (encryptKeyId) {
-      let encrypted = new SyncPb.EncryptedData();
+      let encrypted = new SyncProtoBuf.EncryptedData();
 
       let result;
       try {
@@ -67,7 +67,7 @@ export async function encode(groupId, fileId, since, messages) {
 export async function decode(data) {
   let { encryptKeyId } = prefs.getPrefs();
 
-  let responsePb = SyncPb.SyncResponse.deserializeBinary(data);
+  let responsePb = SyncProtoBuf.SyncResponse.deserializeBinary(data);
   let merkle = JSON.parse(responsePb.getMerkle());
   let list = responsePb.getMessagesList();
   let messages = [];
@@ -79,7 +79,7 @@ export async function decode(data) {
     let msg;
 
     if (encrypted) {
-      let binary = SyncPb.EncryptedData.deserializeBinary(
+      let binary = SyncProtoBuf.EncryptedData.deserializeBinary(
         envelopePb.getContent() as Uint8Array,
       );
 
@@ -98,9 +98,9 @@ export async function decode(data) {
         });
       }
 
-      msg = SyncPb.Message.deserializeBinary(decrypted);
+      msg = SyncProtoBuf.Message.deserializeBinary(decrypted);
     } else {
-      msg = SyncPb.Message.deserializeBinary(
+      msg = SyncProtoBuf.Message.deserializeBinary(
         envelopePb.getContent() as Uint8Array,
       );
     }

packages/loot-core/src/server/sync/index.ts

@@ -1,3 +1,11 @@
+import {
+  serializeClock,
+  deserializeClock,
+  getClock,
+  Timestamp,
+  merkle,
+} from '@actual-app/crdt';
+
 import { captureException } from '../../platform/exceptions';
 import * as asyncStorage from '../../platform/server/asyncStorage';
 import * as connection from '../../platform/server/connection';
@@ -5,13 +13,6 @@ import logger from '../../platform/server/log';
 import { sequential, once } from '../../shared/async';
 import { setIn, getIn } from '../../shared/util';
 import { triggerBudgetChanges, setType as setBudgetType } from '../budget/base';
-import {
-  serializeClock,
-  deserializeClock,
-  getClock,
-  Timestamp,
-  merkle,
-} from '../crdt';
 import * as db from '../db';
 import { PostError, SyncError } from '../errors';
 import app from '../main-app';

packages/loot-core/src/server/sync/make-test-message.ts

@@ -1,13 +1,13 @@
-import * as encryption from '../encryption';
+import { SyncProtoBuf } from '@actual-app/crdt';
 
-import * as SyncPb from './proto/sync_pb';
+import * as encryption from '../encryption';
 
 async function randomString() {
   return (await encryption.randomBytes(12)).toString();
 }
 
 export default async function makeTestMessage(keyId) {
-  let messagePb = new SyncPb.Message();
+  let messagePb = new SyncProtoBuf.Message();
   messagePb.setDataset(await randomString());
   messagePb.setRow(await randomString());
   messagePb.setColumn(await randomString());

packages/loot-core/src/server/sync/migrate.ts

@@ -1,4 +1,4 @@
-import { Timestamp } from '../crdt';
+import { Timestamp } from '@actual-app/crdt';
 
 import { addSyncListener, applyMessages } from './index';
 

packages/loot-core/src/server/sync/repair.ts

@@ -1,4 +1,5 @@
-import { serializeClock, getClock, Timestamp, merkle } from '../crdt';
+import { serializeClock, getClock, Timestamp, merkle } from '@actual-app/crdt';
+
 import * as db from '../db';
 
 export function rebuildMerkleHash() {

packages/loot-core/src/server/sync/sync.property.test.ts

@@ -1,6 +1,6 @@
+import { merkle, getClock, Timestamp } from '@actual-app/crdt';
 import jsc, { type Arbitrary } from 'jsverify';
 
-import { merkle, getClock, Timestamp } from '../crdt';
 import * as db from '../db';
 import * as prefs from '../prefs';
 import * as sheet from '../sheet';

packages/loot-core/src/server/sync/sync.test.ts

@@ -1,4 +1,5 @@
-import { getClock, Timestamp } from '../crdt';
+import { getClock, Timestamp } from '@actual-app/crdt';
+
 import * as db from '../db';
 import * as prefs from '../prefs';
 import * as sheet from '../sheet';

packages/loot-core/src/server/tests/mockSyncServer.ts

@@ -1,5 +1,4 @@
-import { makeClock, Timestamp, merkle } from '../crdt';
-import * as SyncPb from '../sync/proto/sync_pb';
+import { makeClock, Timestamp, merkle, SyncProtoBuf } from '@actual-app/crdt';
 
 import { basic as defaultMockData } from './mockData.json';
 
@@ -29,7 +28,7 @@ handlers['/'] = () => {
 };
 
 handlers['/sync/sync'] = async data => {
-  let requestPb = SyncPb.SyncRequest.deserializeBinary(data);
+  let requestPb = SyncProtoBuf.SyncRequest.deserializeBinary(data);
   let since = requestPb.getSince();
   let messages = requestPb.getMessagesList();
 
@@ -52,11 +51,11 @@ handlers['/sync/sync'] = async data => {
 
   currentClock.merkle = merkle.prune(currentClock.merkle);
 
-  let responsePb = new SyncPb.SyncResponse();
+  let responsePb = new SyncProtoBuf.SyncResponse();
   responsePb.setMerkle(JSON.stringify(currentClock.merkle));
 
   newMessages.forEach(msg => {
-    let envelopePb = new SyncPb.MessageEnvelope();
+    let envelopePb = new SyncProtoBuf.MessageEnvelope();
     envelopePb.setTimestamp(msg.timestamp);
     envelopePb.setIsencrypted(msg.is_encrypted);
     envelopePb.setContent(msg.content);
@@ -112,7 +111,7 @@ export const getClock = () => {
 export const getMessages = () => {
   return currentMessages.map(msg => {
     let { timestamp, content } = msg;
-    let fields = SyncPb.Message.deserializeBinary(content);
+    let fields = SyncProtoBuf.Message.deserializeBinary(content);
 
     return {
       timestamp: timestamp,

packages/loot-core/src/server/undo.ts

@@ -1,7 +1,8 @@
+import { Timestamp } from '@actual-app/crdt';
+
 import * as connection from '../platform/server/connection';
 import { getIn } from '../shared/util';
 
-import { Timestamp } from './crdt';
 import { withMutatorContext, getMutatorContext } from './mutators';
 import { sendMessages } from './sync';
 

upcoming-release-notes/1150.md

@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [MatissJanis]
+---
+
+Extracting CRDT functionality out to `@actual-app/crdt` package

yarn.lock

@@ -23,6 +23,20 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@actual-app/crdt@*, @actual-app/crdt@workspace:packages/crdt":
+  version: 0.0.0-use.local
+  resolution: "@actual-app/crdt@workspace:packages/crdt"
+  dependencies:
+    "@types/jest": ^27.5.0
+    google-protobuf: ^3.12.0-rc.1
+    jest: ^27.0.0
+    murmurhash: ^0.0.2
+    ts-jest: ^27.0.0
+    typescript: ^5.0.2
+    uuid: 3.3.2
+  languageName: unknown
+  linkType: soft
+
 "@actual-app/import-ynab4@*, @actual-app/import-ynab4@workspace:packages/import-ynab4":
   version: 0.0.0-use.local
   resolution: "@actual-app/import-ynab4@workspace:packages/import-ynab4"
@@ -12177,6 +12191,7 @@ __metadata:
   resolution: "loot-core@workspace:packages/loot-core"
   dependencies:
     "@actual-app/api": "*"
+    "@actual-app/crdt": "*"
     "@actual-app/import-ynab4": "*"
     "@babel/core": ~7.22.5
     "@babel/preset-env": ^7.22.5
@@ -12207,7 +12222,6 @@ __metadata:
     deep-equal: ^2.0.5
     fake-indexeddb: ^3.1.3
     fast-check: 3.7.1
-    google-protobuf: ^3.12.0-rc.1
     jest: ^27.0.0
     jsverify: ^0.8.4
     lru-cache: ^5.1.1
@@ -12216,7 +12230,6 @@ __metadata:
     memoize-one: ^4.0.0
     mitt: ^3.0.0
     mockdate: ^3.0.5
-    murmurhash: ^0.0.2
     node-fetch: ^2.6.9
     node-libofx: "*"
     npm-run-all: ^4.1.3