Release notes

* fix bad sort order in demo

* note

* add async back

* fix tests

* fix2

* fix3

* update vrt

* fix image

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+	"name": "Actual development",
+	"dockerComposeFile": [
+		"../docker-compose.yml",
+		"docker-compose.yml"
+	],
+	// Alternatively:
+	// "image": "mcr.microsoft.com/devcontainers/typescript-node:0-16",
+	"service": "actual-development",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"postCreateCommand": "yarn install"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '3.8'
+services:
+  actual-development:
+    volumes:
+      - ..:/workspaces:cached
+    command: /bin/sh -c "while sleep 1000; do :; done"

.eslintignore

@@ -0,0 +1,27 @@
+packages/api/app/bundle.api.js
+packages/api/dist
+packages/api/@types
+packages/api/migrations
+
+packages/crdt/dist
+
+packages/desktop-client/bundle.browser.js
+packages/desktop-client/build/
+packages/desktop-client/build-stats/
+packages/desktop-client/public/kcab/
+packages/desktop-client/public/data/
+packages/desktop-client/**/node_modules/*
+packages/desktop-client/node_modules/
+packages/desktop-client/src/icons/**/*
+packages/desktop-client/test-results/
+
+packages/desktop-electron/client-build/
+packages/desktop-electron/dist/
+
+packages/import-ynab4/**/node_modules/*
+
+packages/import-ynab5/**/node_modules/*
+
+packages/loot-core/**/node_modules/*
+packages/loot-core/**/lib-dist/*
+packages/loot-core/**/proto/*

.eslintrc.js

@@ -1,3 +1,4 @@
+/* eslint-disable rulesdir/typography */
 const path = require('path');
 
 const rulesDirPlugin = require('eslint-plugin-rulesdir');
@@ -9,32 +10,112 @@ rulesDirPlugin.RULES_DIR = path.join(
   'rules',
 );
 
+const ruleFCMsg =
+  'Type the props argument and let TS infer or use ComponentType for a component prop';
+
+const restrictedImportPatterns = [
+  {
+    group: ['*.api', '*.web', '*.electron'],
+    message: 'Don’t directly reference imports from other platforms',
+  },
+  {
+    group: ['uuid'],
+    importNames: ['*'],
+    message: "Use `import { v4 as uuidv4 } from 'uuid'` instead",
+  },
+];
+
+const restrictedImportColors = [
+  {
+    group: ['**/style', '**/colors'],
+    importNames: ['colors'],
+    message: 'Please use themes instead of colors',
+  },
+];
+
 module.exports = {
   plugins: ['prettier', 'import', 'rulesdir', '@typescript-eslint'],
-  extends: ['react-app', 'plugin:@typescript-eslint/recommended'],
+  extends: [
+    'react-app',
+    'plugin:react/recommended',
+    'plugin:prettier/recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:import/typescript',
+  ],
+  parser: '@typescript-eslint/parser',
+  parserOptions: { project: [path.join(__dirname, './tsconfig.json')] },
   reportUnusedDisableDirectives: true,
+  globals: {
+    globalThis: false,
+    vi: true,
+  },
   rules: {
-    'prettier/prettier': 'error',
-    'no-unused-vars': 'off',
+    'prettier/prettier': 'warn',
 
-    'no-restricted-globals': ['error'].concat(
+    // Note: base rule explicitly disabled in favor of the TS one
+    'no-unused-vars': 'off',
+    '@typescript-eslint/no-unused-vars': [
+      'warn',
+      {
+        varsIgnorePattern: '^_',
+        ignoreRestSiblings: true,
+      },
+    ],
+
+    curly: ['warn', 'multi-line', 'consistent'],
+
+    'no-restricted-globals': ['warn'].concat(
       require('confusing-browser-globals').filter(g => g !== 'self'),
     ),
 
-    'react/jsx-no-useless-fragment': 'error',
+    'react/jsx-filename-extension': [
+      'warn',
+      { extensions: ['.jsx', '.tsx'], allow: 'as-needed' },
+    ],
+    'react/jsx-no-useless-fragment': 'warn',
+    'react/self-closing-comp': 'warn',
+    'react/no-unstable-nested-components': [
+      'warn',
+      { allowAsProps: true, customValidators: ['formatter'] },
+    ],
 
-    'rulesdir/typography': 'error',
+    'rulesdir/typography': 'warn',
+    'rulesdir/prefer-if-statement': 'warn',
 
     // https://github.com/eslint/eslint/issues/16954
     // https://github.com/eslint/eslint/issues/16953
     'no-loop-func': 'off',
 
+    // Do don't need this as we're using TypeScript
+    'react/prop-types': 'off',
+
     // TODO: re-enable these rules
     'react-hooks/exhaustive-deps': 'off',
+    'react/display-name': 'off',
+    'react/react-in-jsx-scope': 'off',
+    // 'react-hooks/exhaustive-deps': [
+    //   'warn',
+    //   {
+    //     additionalHooks: 'useLiveQuery',
+    //   },
+    // ],
 
-    'import/no-useless-path-segments': 'error',
+    'no-var': 'warn',
+    'react/jsx-curly-brace-presence': 'warn',
+    'object-shorthand': ['warn', 'properties'],
+
+    'import/extensions': [
+      'warn',
+      'never',
+      {
+        json: 'always',
+      },
+    ],
+    'import/no-useless-path-segments': 'warn',
+    'import/no-duplicates': ['warn', { 'prefer-inline': true }],
+    'import/no-unused-modules': ['warn', { unusedExports: true }],
     'import/order': [
-      'error',
+      'warn',
       {
         alphabetize: {
           caseInsensitive: true,
@@ -62,18 +143,146 @@ module.exports = {
       },
     ],
 
-    // Rules disable during TS migration
-    '@typescript-eslint/no-var-requires': 'off',
-    'prefer-const': 'off',
-    'prefer-spread': 'off',
-    '@typescript-eslint/no-empty-function': 'off',
-    '@typescript-eslint/no-unused-vars': [
-      'error',
+    'no-restricted-syntax': [
+      'warn',
       {
-        args: 'none',
-        varsIgnorePattern: '^_',
-        ignoreRestSiblings: true,
+        // forbid React.* as they are legacy https://twitter.com/dan_abramov/status/1308739731551858689
+        selector:
+          ":matches(MemberExpression[object.name='React'], TSQualifiedName[left.name='React'])",
+        message:
+          'Using default React import is discouraged, please use named exports directly instead.',
+      },
+      {
+        // forbid <a> in favor of <LinkButton> or <ExternalLink>
+        selector: 'JSXOpeningElement[name.name="a"]',
+        message:
+          'Using <a> is discouraged, please use <LinkButton> or <ExternalLink> instead.',
       },
     ],
+    'no-restricted-imports': [
+      'warn',
+      { patterns: [...restrictedImportPatterns, ...restrictedImportColors] },
+    ],
+
+    '@typescript-eslint/ban-ts-comment': [
+      'error',
+      { 'ts-ignore': 'allow-with-description' },
+    ],
+
+    // Rules disable during TS migration
+    '@typescript-eslint/no-var-requires': 'off',
+    'prefer-const': 'warn',
+    'prefer-spread': 'off',
+    '@typescript-eslint/no-empty-function': 'off',
+    'import/no-default-export': 'warn',
+  },
+  overrides: [
+    {
+      files: ['.eslintrc.js', './**/.eslintrc.js'],
+      parserOptions: { project: null },
+      rules: {
+        '@typescript-eslint/consistent-type-exports': 'off',
+      },
+    },
+    {
+      files: [
+        './packages/desktop-client/**/*.{ts,tsx}',
+        './packages/loot-core/src/client/**/*.{ts,tsx}',
+      ],
+      rules: {
+        // enforce type over interface
+        '@typescript-eslint/consistent-type-definitions': ['warn', 'type'],
+        // enforce import type
+        '@typescript-eslint/consistent-type-imports': [
+          'warn',
+          { prefer: 'type-imports', fixStyle: 'inline-type-imports' },
+        ],
+        '@typescript-eslint/ban-types': [
+          'warn',
+          {
+            types: {
+              // forbid FC as superflous
+              FunctionComponent: { message: ruleFCMsg },
+              FC: { message: ruleFCMsg },
+            },
+            extendDefaults: true,
+          },
+        ],
+      },
+    },
+    {
+      files: ['./packages/desktop-client/**/*'],
+      excludedFiles: [
+        './packages/desktop-client/src/hooks/useNavigate.{ts,tsx}',
+      ],
+      rules: {
+        'no-restricted-imports': [
+          'warn',
+          {
+            patterns: [
+              {
+                group: ['react-router-dom'],
+                importNames: ['useNavigate'],
+                message: 'Please use Actual’s useNavigate() hook instead.',
+              },
+            ],
+          },
+        ],
+      },
+    },
+    {
+      files: ['./packages/loot-core/src/**/*'],
+      rules: {
+        'no-restricted-imports': [
+          'warn',
+          {
+            patterns: [
+              ...restrictedImportPatterns,
+              {
+                group: ['loot-core/**'],
+                message:
+                  'Please use relative imports in loot-core instead of importing from `loot-core/*`',
+              },
+            ],
+          },
+        ],
+      },
+    },
+    {
+      files: [
+        'packages/loot-core/src/types/**/*',
+        'packages/loot-core/src/client/state-types/**/*',
+        '**/icons/**/*',
+        '**/{mocks,__mocks__}/**/*',
+        // can't correctly resolve usages
+        '**/*.{testing,electron,browser,web,api}.ts',
+      ],
+      rules: { 'import/no-unused-modules': 'off' },
+    },
+    {
+      files: [
+        './packages/desktop-client/src/style/index.*',
+        './packages/desktop-client/src/style/palette.*',
+      ],
+      rules: {
+        'no-restricted-imports': ['off', { patterns: restrictedImportColors }],
+      },
+    },
+    {
+      files: [
+        './packages/api/migrations/*',
+        './packages/loot-core/migrations/*',
+      ],
+      rules: {
+        'import/no-default-export': 'off',
+      },
+    },
+  ],
+  settings: {
+    'import/resolver': {
+      typescript: {
+        alwaysTryTypes: true,
+      },
+    },
   },
 };

.gitattributes

@@ -8,8 +8,12 @@
 
 # Declare files that will always have LF line endings on checkout.
 *.js text eol=lf
+*.ts text eol=lf
+*.sh text eol=lf
+*.tsx text eol=lf
+
 yarn.lock text eol=lf
 
 # Denote all files that are truly binary and should not be modified.
 *.png binary
-*.jpg binary
+*.jpg binary

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# Funding policies: https://actualbudget.org/docs/contributing/leadership/funding
+open_collective: actual
+github: actualbudget

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,7 +1,7 @@
 name: Bug Report
 description: File a bug report also known as an issue or problem.
 title: '[Bug]: '
-labels: ['bug', 'needs triage']
+labels: ['bug']
 body:
   - type: markdown
     id: intro-md
@@ -18,6 +18,18 @@ body:
           required: true
     validations:
       required: true
+  - type: checkboxes
+    id: bank-sync-issue
+    attributes:
+      label: 'Is this related to GoCardless, Simplefin or another bank-sync provider?'
+      description: 'Most issues with bank-sync providers are due to a lack of a custom bank-mapper (i.e. payee or other fields not coming through). In such cases you can create a custom bank mapper in [actual-server](https://github.com/actualbudget/actual-server/blob/master/src/app-gocardless/README.md) repository. Other likely issue is misconfigured server - in which case please reach out via the [community Discord](https://discord.gg/pRYNYr4W5A) to get support.'
+      options:
+        - label: 'I have checked my server logs and could not see any errors there'
+        - label: 'I will be attaching my server logs to this issue'
+        - label: 'I will be attaching my client-side (browser) logs to this issue'
+        - label: 'I understand that this issue will be automatically closed if insufficient information is provided'
+    validations:
+      required: false
   - type: textarea
     id: what-happened
     attributes:
@@ -48,6 +60,7 @@ body:
         - Docker
         - Fly.io
         - NAS
+        - Desktop App (Electron)
         - Other
     validations:
       required: false
@@ -61,6 +74,7 @@ body:
         - Chrome
         - Safari
         - Microsoft Edge
+        - Desktop App (Electron)
         - Other
   - type: dropdown
     id: operating-system

.github/ISSUE_TEMPLATE/config.yml

@@ -1 +1,5 @@
 blank_issues_enabled: false
+contact_links:
+  - name: Support
+    url: https://discord.gg/pRYNYr4W5A
+    about: Need help with something? Perhaps having issues setting up bank-sync with GoCardless or SimpleFin? Reach out to the community on Discord.

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,7 +1,7 @@
 name: Feature request
 description: Request a missing feature
 title: '[Feature] '
-labels: ['feature', 'needs triage']
+labels: ['feature']
 body:
   - type: markdown
     id: intro-md

.github/actions/check-migrations.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+// overview:
+// 1. Identify the migrations in packages/loot-core/migrations/* on `master` and HEAD
+// 2. Make sure that any new migrations on HEAD are dated after the latest migration on `master`.
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const migrationsDir = path.join(
+  __dirname,
+  '..',
+  '..',
+  'packages',
+  'loot-core',
+  'migrations',
+);
+
+function readMigrations(ref) {
+  const { stdout } = spawnSync('git', [
+    'ls-tree',
+    '--name-only',
+    ref,
+    migrationsDir + '/',
+  ]);
+  const files = stdout.toString().split('\n').filter(Boolean);
+  console.log(`Found ${files.length} migrations on ${ref}.`);
+  return files
+    .map(file => path.basename(file))
+    .filter(file => !file.startsWith('.'))
+    .map(name => ({
+      date: parseInt(name.split('_')[0]),
+      name: name.match(/^\d+_(.+?)(\.sql)?$/)?.[1] ?? '***' + name,
+    }));
+}
+
+spawnSync('git', ['fetch', 'origin', 'master']);
+let masterMigrations = readMigrations('origin/master');
+let headMigrations = readMigrations('HEAD');
+
+let latestMasterMigration = masterMigrations[masterMigrations.length - 1].date;
+let newMigrations = headMigrations.filter(
+  migration => !masterMigrations.find(m => m.name === migration.name),
+);
+let badMigrations = newMigrations.filter(
+  migration => migration.date <= latestMasterMigration,
+);
+
+if (badMigrations.length) {
+  console.error(
+    `The following migrations are dated before the latest migration on master:`,
+  );
+  badMigrations.forEach(migration => {
+    console.error(`  ${migration.name}`);
+  });
+  process.exit(1);
+} else {
+  console.log(`All migrations are dated after the latest migration on master.`);
+}

.github/actions/handle-feature-requests.js

@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+
+// overview:
+// 1. Fetch the issues that are linked to the PR
+// 2. Filter out the issues that are not feature requests
+// 3. For each feature request:
+//   1. Remove the 'help wanted' & 'needs votes' labels
+//   3. Find the automated comment, hide the comment as 'outdated'
+//   5. Post a new comment saying that the feature request has been implemented, and will be released in the next version. Link to the PR.
+
+async function makeAPIRequest(query, variables) {
+  const res = await fetch('https://api.github.com/graphql', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+      'Content-Type': 'application/json',
+      Accept: 'application/json',
+    },
+    body: JSON.stringify({ query, variables }),
+  });
+  return res.json();
+}
+
+function group(name, body) {
+  console.log(`::group::${name}`);
+  const result = body();
+  if (result instanceof Promise) {
+    return result.finally(() => console.log(`::endgroup::`));
+  }
+  console.log(`::endgroup::`);
+  return result;
+}
+
+async function main() {
+  const featureRequests = await group('Pull Request API Response', async () => {
+    const res = await makeAPIRequest(
+      /* GraphQL */ `
+        query FetchLinkedIssues($pr: Int!) {
+          repository(owner: "actualbudget", name: "actual") {
+            pullRequest(number: $pr) {
+              closingIssuesReferences(first: 10) {
+                nodes {
+                  id
+                  number
+                  labels(first: 10) {
+                    nodes {
+                      id
+                      name
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      `,
+      { pr: parseInt(process.env.PR_NUMBER) },
+    );
+
+    console.log(JSON.stringify(res, null, 2));
+
+    return res.data.repository.pullRequest.closingIssuesReferences.nodes.filter(
+      issue => issue.labels.nodes.some(label => label.name === 'feature'),
+    );
+  });
+
+  if (featureRequests.length === 0) {
+    console.log('No linked feature requests found');
+    return;
+  }
+
+  for (const { id, number, labels } of featureRequests) {
+    await group(`Issue #${number}: Remove labels`, async () => {
+      const toRemove = labels.nodes
+        .filter(
+          label =>
+            label.name === 'help wanted' ||
+            label.name === 'needs votes' ||
+            label.name === 'good first issue',
+        )
+        .map(label => label.id);
+      const res = await makeAPIRequest(
+        /* GraphQL */ `
+          mutation RemoveLabels($issue: ID!, $labels: [ID!]!) {
+            removeLabelsFromLabelable(
+              input: {
+                clientMutationId: "1"
+                labelIds: $labels
+                labelableId: $issue
+              }
+            ) {
+              clientMutationId
+            }
+          }
+        `,
+        {
+          issue: id,
+          labels: toRemove,
+        },
+      );
+      console.log(JSON.stringify(res, null, 2));
+    });
+
+    await group(`Issue #${number}: Collapse automatic comment`, async () => {
+      const commentRes = await makeAPIRequest(
+        /* GraphQL */ `
+          query FetchComments($issue: Int!) {
+            repository(owner: "actualbudget", name: "actual") {
+              issue(number: $issue) {
+                comments(first: 100) {
+                  nodes {
+                    id
+                    body
+                    author {
+                      login
+                    }
+                  }
+                }
+              }
+            }
+          }
+        `,
+        { issue: number },
+      );
+      console.log(JSON.stringify(commentRes, null, 2));
+
+      const comments = commentRes.data.repository.issue.comments.nodes.filter(
+        comment => comment.author.login === 'github-actions',
+      );
+      const commentToCollapse =
+        comments.find(comment =>
+          comment.body.includes('<!-- feature-auto-close-comment -->'),
+        ) ||
+        comments.find(comment =>
+          comment.body.includes(
+            ':sparkles: Thanks for sharing your idea! :sparkles:',
+          ),
+        );
+
+      if (!commentToCollapse) {
+        console.log('No comment to collapse found');
+        process.exit(1);
+      }
+
+      const res = await makeAPIRequest(
+        /* GraphQL */ `
+          mutation CollapseComment($comment: ID!) {
+            minimizeComment(
+              input: { classifier: OUTDATED, subjectId: $comment }
+            ) {
+              clientMutationId
+            }
+          }
+        `,
+        { comment: commentToCollapse.id },
+      );
+      console.log(JSON.stringify(res, null, 2));
+    });
+
+    await group(`Issue #${number}: Post comment`, async () => {
+      const res = await makeAPIRequest(
+        /* GraphQL */ `
+          mutation PostComment($issue: ID!, $body: String!) {
+            addComment(
+              input: { subjectId: $issue, body: $body, clientMutationId: "1" }
+            ) {
+              clientMutationId
+            }
+          }
+        `,
+        {
+          issue: id,
+          body: `:tada: This feature has been implemented in #${process.env.PR_NUMBER} and will be released in the next version. Thanks for sharing your idea! :tada:\n\n<!-- feature-implemented-comment -->`,
+        },
+      );
+      console.log(JSON.stringify(res, null, 2));
+    });
+  }
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});

.github/actions/netlify-wait-for-build

@@ -9,8 +9,8 @@ function get_status() {
   curl --header "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/actualbudget/actual/commits/$COMMIT_SHA/statuses" > /tmp/status.json
   cat /tmp/status.json
   echo "::endgroup::"
-  netlify=$(jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json)
-  state=$(jq -r '.state' <<< "$netlify")
+  netlify=$(yarn jq '[.[] | select(.context == "netlify/actualbudget/deploy-preview")][0]' /tmp/status.json)
+  state=$(yarn jq -r '.state' <<< "$netlify")
   echo "::group::Netlify Status"
   echo "$netlify"
   echo "::endgroup::"
@@ -32,7 +32,7 @@ done
 
 if [ "$state" == "success" ]; then
   echo -e "\033[0;32mNetlify build succeeded!\033[0m"
-  jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT
+  yarn jq -r '"url=" + .target_url' <<< "$netlify" > $GITHUB_OUTPUT
   exit 0
 else
   echo -e "\033[0;31mNetlify build failed. Cancelling end-to-end tests.\033[0m"

.github/actions/setup/action.yml

@@ -6,13 +6,13 @@ runs:
     - name: Install node
       uses: actions/setup-node@v3
       with:
-        node-version: 16.15.0
+        node-version: 18.16.0
     - name: Cache
       uses: actions/cache@v3
       id: cache
       with:
         path: '**/node_modules'
-        key: yarn-v1-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+        key: yarn-v1-${{ runner.os }}-${{ hashFiles('.nvmrc') }}-${{ hashFiles('**/yarn.lock') }}
     - name: Install
       run: yarn --immutable
       shell: bash

.github/workflows/build.yml

@@ -12,7 +12,10 @@ on:
     branches:
       - master
   pull_request:
-    branches: '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
 
 jobs:
   api:
@@ -23,6 +26,29 @@ jobs:
         uses: ./.github/actions/setup
       - name: Build API
         run: cd packages/api && yarn build
+      - name: Create package tgz
+        run: cd packages/api && yarn pack && mv package.tgz actual-api.tgz
+      - name: Upload Build
+        uses: actions/upload-artifact@v3
+        with:
+          name: actual-api
+          path: packages/api/actual-api.tgz
+
+  crdt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Build CRDT
+        run: cd packages/crdt && yarn build
+      - name: Create package tgz
+        run: cd packages/crdt && yarn pack && mv package.tgz actual-crdt.tgz
+      - name: Upload Build
+        uses: actions/upload-artifact@v3
+        with:
+          name: actual-crdt
+          path: packages/crdt/actual-crdt.tgz
 
   web:
     runs-on: ubuntu-latest
@@ -37,21 +63,8 @@ jobs:
         with:
           name: actual-web
           path: packages/desktop-client/build
-
-  # TODO: re-enable after solving https://github.com/actualbudget/actual/issues/468
-  # electron:
-  #   # As electron builds take longer, we only run them in master.
-  #   if: github.event_name != 'pull_request'
-  #   strategy:
-  #     matrix:
-  #       os:
-  #         - ubuntu-latest
-  #         - windows-latest
-  #         - macos-latest
-  #   runs-on: ${{ matrix.os }}
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: Set up environment
-  #       uses: ./.github/actions/setup
-  #     - name: Build Electron
-  #       run: ./bin/package
+      - name: Upload Build Stats
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-stats
+          path: packages/desktop-client/build-stats

.github/workflows/check-release-notes.yml

@@ -1,16 +0,0 @@
-name: Check release notes
-
-on:
-  pull_request:
-    branches:
-      - '*'
-      - '!release/*'
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Check release notes
-        uses: actualbudget/actions/release-notes/check@main

.github/workflows/check.yml

@@ -0,0 +1,48 @@
+name: Test
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Lint
+        run: yarn lint
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Typecheck
+        run: yarn typecheck
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Test
+        run: yarn test
+
+  migrations:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '19'
+      - name: Check migrations
+        run: node ./.github/actions/check-migrations.js

.github/workflows/codeql.yml

@@ -4,10 +4,13 @@ on:
   push:
     branches: [master]
   pull_request:
-    branches: [master]
   schedule:
     - cron: '23 11 * * 6'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/e2e-test.yml

@@ -5,26 +5,61 @@ on: [pull_request]
 env:
   GITHUB_PR_NUMBER: ${{github.event.pull_request.number}}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  test:
-    name: Run end-to-end tests on Netlify PR preview
+  netlify:
+    name: Wait for Netlify build to finish
     runs-on: ubuntu-latest
+    outputs:
+      netlify_url: ${{ steps.netlify.outputs.url }}
     steps:
       - uses: actions/checkout@v3
       - name: Set up environment
         uses: ./.github/actions/setup
-      - name: Setup Playwright
-        run: npx playwright install chromium --with-deps
       - name: Wait for Netlify build to finish
         id: netlify
         env:
           COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: ./bin/netlify-wait-for-build
+        run: ./.github/actions/netlify-wait-for-build
+
+  functional:
+    name: Functional
+    needs: netlify
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.41.1-jammy
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
       - name: Run E2E Tests on Netlify URL
         run: yarn e2e
         env:
-          E2E_START_URL: ${{ steps.netlify.outputs.url }}
+          E2E_START_URL: ${{ needs.netlify.outputs.netlify_url }}
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: desktop-client-test-results
+          path: packages/desktop-client/test-results/
+          retention-days: 30
+  vrt:
+    name: Visual regression
+    needs: netlify
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.41.1-jammy
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Run VRT Tests on Netlify URL
+        run: yarn vrt
+        env:
+          E2E_START_URL: ${{ needs.netlify.outputs.netlify_url }}
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/electron-master.yml

@@ -0,0 +1,50 @@
+name: Electron
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  CI: true
+
+on:
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - if: ${{ startsWith(matrix.os, 'windows') }}
+        run: pip.exe install setuptools
+      - if: ${{ ! startsWith(matrix.os, 'windows') }}
+        run: python3 -m pip install setuptools
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Build Electron
+        run: ./bin/package-electron
+        env:
+          CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
+          CSC_LINK: ${{ secrets.CSC_LINK }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
+      - name: Upload Build
+        uses: actions/upload-artifact@v3
+        with:
+          name: actual-electron-${{ matrix.os }}
+          path: |
+            packages/desktop-electron/dist/*.dmg
+            packages/desktop-electron/dist/*.exe
+            packages/desktop-electron/dist/*.AppImage

.github/workflows/electron-pr.yml

@@ -0,0 +1,43 @@
+name: Electron
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  CI: true
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - if: ${{ startsWith(matrix.os, 'windows') }}
+        run: pip.exe install setuptools
+      - if: ${{ ! startsWith(matrix.os, 'windows') }}
+        run: python3 -m pip install setuptools
+      - name: Set up environment
+        uses: ./.github/actions/setup
+      - name: Build Electron
+        run: ./bin/package-electron
+      - name: Upload Build
+        uses: actions/upload-artifact@v3
+        with:
+          name: actual-electron-${{ matrix.os }}
+          path: |
+            packages/desktop-electron/dist/*.dmg
+            packages/desktop-electron/dist/*.exe
+            packages/desktop-electron/dist/*.AppImage

.github/workflows/generate-release-notes.yml

@@ -1,17 +0,0 @@
-name: Generate Release Notes
-
-on:
-  push:
-    branches:
-      - release/*
-
-jobs:
-  generate:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Generate release notes
-        uses: actualbudget/actions/release-notes/generate@main
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/issues-close-feature-requests.yml

@@ -9,8 +9,6 @@ jobs:
     if: ${{ github.event.label.name == 'feature' }}
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
       - uses: actions-ecosystem/action-add-labels@v1
         with:
           labels: needs votes
@@ -26,12 +24,14 @@ jobs:
           body: |
             :sparkles: Thanks for sharing your idea! :sparkles:
 
-            This repository is now using lodash style issue management for enhancements. This means enhancement issues will now be closed instead of leaving them open. This doesn’t mean we don’t accept feature requests, though! We will consider implementing ones that receive many upvotes, and we welcome contributions for any feature requests marked as needing votes (just post a comment first so we can help you make a successful contribution).
+            This repository uses lodash style issue management for enhancements. That means enhancement issues are automatically closed. This doesn’t mean we don’t accept feature requests, though! We will consider implementing ones that receive many upvotes, and we welcome contributions for any feature requests marked as needing votes (just post a comment first so we can help you make a successful contribution).
 
             The enhancement backlog can be found here: https://github.com/actualbudget/actual/issues?q=label%3A%22needs+votes%22+sort%3Areactions-%2B1-desc+
 
             Don’t forget to upvote the top comment with 👍!
+
+            <!-- feature-auto-close-comment -->
       - name: Close Issue
-        run: gh issue close "${{ github.event.issue.number }}"
+        run: gh issue close "https://github.com/actualbudget/actual/issues/${{ github.event.issue.number }}"
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/issues-feature-implemented.yml

@@ -0,0 +1,35 @@
+name: Handle completed feature requests
+
+##########################################################################################
+# WARNING! This workflow uses the 'pull_request_target' event. That mans that it will    #
+# always run in the context of the main actualbudget/actual repo, even if the PR is from #
+# a fork. This is necessary to get access to a GitHub token that can post a comment on   #
+# the PR. Be VERY CAREFUL about adding things to this workflow, since forks can inject   #
+# arbitrary code into their branch, and can pollute the artifacts we download. Arbitrary #
+# code execution in this workflow could lead to a compromise of the main repo.           #
+##########################################################################################
+# See: https://securitylab.github.com/research/github-actions-preventing-pwn-requests    #
+##########################################################################################
+
+on:
+  pull_request_target:
+    types: [closed]
+
+permissions:
+  issues: write
+
+jobs:
+  handle-feature-requests:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      # This is not a security concern because we have approved & merged the PR
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '19'
+      - name: Handle feature requests
+        run: node .github/actions/handle-feature-requests.js
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/issues-remove-help-wanted.yml

@@ -0,0 +1,14 @@
+name: Remove 'help wanted' label from closed issues
+
+on:
+  issues:
+    types: [closed]
+
+jobs:
+  remove-help-wanted:
+    if: ${{ !contains(github.event.issue.labels.*.name, 'feature') && contains(github.event.issue.labels.*.name, 'help wanted') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: help wanted

.github/workflows/lint.yml

@@ -1,18 +0,0 @@
-name: Linter
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches: '*'
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Lint
-        run: yarn lint

.github/workflows/opened-issues-triage.yml

@@ -1,14 +0,0 @@
-name: Mark new issue for triage
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  needs-triage:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-ecosystem/action-add-labels@v1
-        with:
-          labels: needs triage

.github/workflows/release-notes.yml

@@ -0,0 +1,21 @@
+name: Release notes
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release-notes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Check release notes
+        if: startsWith(github.head_ref, 'release/') == false
+        uses: actualbudget/actions/release-notes/check@main
+      - name: Generate release notes
+        if: startsWith(github.head_ref, 'release/') == true
+        uses: actualbudget/actions/release-notes/generate@main

.github/workflows/size-compare.yml

@@ -0,0 +1,85 @@
+name: Compare Sizes
+
+##########################################################################################
+# WARNING! This workflow uses the 'pull_request_target' event. That mans that it will    #
+# always run in the context of the main actualbudget/actual repo, even if the PR is from #
+# a fork. This is necessary to get access to a GitHub token that can post a comment on   #
+# the PR. Be VERY CAREFUL about adding things to this workflow, since forks can inject   #
+# arbitrary code into their branch, and can pollute the artifacts we download. Arbitrary #
+# code execution in this workflow could lead to a compromise of the main repo.           #
+##########################################################################################
+# See: https://securitylab.github.com/research/github-actions-preventing-pwn-requests    #
+##########################################################################################
+
+on:
+  pull_request_target:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  compare:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for ${{github.base_ref}} build to succeed
+        uses: fountainhead/action-wait-for-check@v1.1.0
+        id: master-build
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          checkName: web
+          ref: ${{github.base_ref}}
+
+      - name: Wait for PR build to succeed
+        uses: fountainhead/action-wait-for-check@v1.1.0
+        id: wait-for-build
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          checkName: web
+          ref: ${{github.event.pull_request.head.sha}}
+
+      - name: Report build failure
+        if: steps.wait-for-build.outputs.conclusion == 'failure'
+        run: |
+          echo "Build failed on PR branch or ${{github.base_ref}}"
+          exit 1
+      - name: Download build artifact from ${{github.base_ref}}
+        uses: dawidd6/action-download-artifact@v2
+        id: pr-build
+        with:
+          branch: ${{github.base_ref}}
+          workflow: build.yml
+          name: build-stats
+          path: base
+
+      - name: Download build artifact from PR
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          pr: ${{github.event.pull_request.number}}
+          workflow: build.yml
+          name: build-stats
+          path: head
+
+      - name: Strip content hashes from stats files
+        run: |
+          sed -i -E 's/index\.[0-9a-zA-Z_-]{8,}\./index./g' ./head/web-stats.json
+          sed -i -E 's/\.[0-9a-zA-Z_-]{8,}\.chunk\././g' ./head/web-stats.json
+          sed -i -E 's/\.[0-9a-f]{8,}\././g' ./head/*.json
+          sed -i -E 's/index\.[0-9a-zA-Z_-]{8,}\./index./g' ./base/web-stats.json
+          sed -i -E 's/\.[0-9a-zA-Z_-]{8,}\.chunk\././g' ./base/web-stats.json
+          sed -i -E 's/\.[0-9a-f]{8,}\././g' ./base/*.json
+      - uses: twk3/rollup-size-compare-action@v1.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          current-stats-json-path: ./head/web-stats.json
+          base-stats-json-path: ./base/web-stats.json
+          title: desktop-client
+
+      - uses: github/webpack-bundlesize-compare-action@v1.8.2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          current-stats-json-path: ./head/loot-core-stats.json
+          base-stats-json-path: ./base/loot-core-stats.json
+          title: loot-core

.github/workflows/stale.yml

@@ -1,20 +1,16 @@
-name: Close inactive issues
+name: 'Close stale PRs'
 on:
   schedule:
-    - cron: "30 1 * * *"
+    - cron: '30 1 * * *'
 
 jobs:
-  close-issues:
+  stale:
     runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
-      - uses: actions/stale@v7
+      - uses: actions/stale@v8
         with:
-          days-before-issue-stale: 90
-          days-before-issue-close: -1
-          stale-issue-label: "stale"
-          stale-issue-message: "🚧🚨 This issue is being marked as stale due to 90 days of inactivity. 🚧🚨"
-          only-labels: 'needs triage'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
+          close-pr-message: 'This PR was closed because it has been stalled for 5 days with no activity.'
+          days-before-stale: 30
+          days-before-close: 5
+          days-before-issue-stale: -1

.github/workflows/test.yml

@@ -1,18 +0,0 @@
-name: Test
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches: '*'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Test
-        run: yarn test

.github/workflows/typecheck.yml

@@ -1,18 +0,0 @@
-name: Typecheck
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches: '*'
-
-jobs:
-  typecheck:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Typecheck
-        run: yarn typecheck

.gitignore

@@ -2,6 +2,8 @@
 !data/.gitkeep
 /data2
 packages/api/dist
+packages/api/@types
+packages/crdt/dist
 packages/desktop-electron/client-build
 packages/desktop-electron/.electron-symbols
 packages/desktop-electron/dist
@@ -19,6 +21,7 @@ bundle.mobile.js
 bundle.mobile.js.map
 export-2020-01-10.csv
 .idea
+.vscode
 
 **/*.log
 
@@ -30,3 +33,6 @@ export-2020-01-10.csv
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+
+# VSCode
+.vscode

.nvmrc

@@ -1 +1 @@
-v16.15.0
+v18.16.0

.prettierignore

@@ -0,0 +1 @@
+sync_pb.*

.secret-tokens.example

@@ -0,0 +1,2 @@
+export APPLE_ID=example@email.com
+export APPLE_APP_SPECIFIC_PASSWORD=password

.yarnrc.yml

@@ -1,9 +1,7 @@
+compressionLevel: mixed
+
+enableGlobalCache: false
+
 nodeLinker: node-modules
 
-plugins:
-  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
-    spec: "@yarnpkg/plugin-workspace-tools"
-  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
-    spec: "@yarnpkg/plugin-interactive-tools"
-
-yarnPath: .yarn/releases/yarn-3.4.1.cjs
+yarnPath: .yarn/releases/yarn-4.0.1.cjs

CONTRIBUTING.md

@@ -1,34 +1 @@
-## Expectations
-
-For smaller improvements or features - feel free to submit a PR or an issue if you don't have the necessary skills to build it yourself. For larger features we would recommend first opening an issue to discuss it with the team.
-
-We aren't going to take every single little change. Don't be offended if we close your PR. In order for the project to stay healthy, we need to guard our bandwidth and also only take changes that align with Actual.
-
-Here are some initial guidelines for how contributions will be treated:
-
-- The mental health of the maintainers will be prioritized above all else. If this means some things get lost and PRs are unreviewed because maintainers are spending time with family or on themselves, we celebrate that.
-
-- Multiple maintainers are key to this being a healthy project. Currently a few people have maintainer rights (see list below). We are actively looking for more people to come on as maintainers. If nobody steps up, expect less activity on this project.
-
-- An open PR does not automatically deserve time for a full review and acceptance. It's up to the PR author to convince the maintainers that the change is good and worth reviewing. This involves a clear description for why the the change is being made, detailing the tradeoffs.
-
-- We especially welcome improvements in automation: creating github actions to automatically generate builds, making the release process easier, etc.
-
-## Main contributors
-
-(sorted alphabetically)
-
-- @albertogasparin
-- @j-f1
-- @jlongster
-- @MatissJanis
-- @rich-howell
-- @trevdor
-
-## Project ideas
-
-We welcome all contributions from the community. If you have an idea for a feature you want to build - please go ahead and submit a PR with the implementation or if it's a larger feature - open a new issue so we can discuss it.
-
-If you do not have ideas what to build: the issue list is always a good starting point. Look for issues labeled with "[help wanted](https://github.com/actualbudget/actual/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22)".
-
-For first time contributions you can also filter the issues labeled with "[good first issue](https://github.com/actualbudget/actual/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)".
+Please review the contributing documentation on our website: https://actualbudget.org/docs/contributing/

Dockerfile

@@ -0,0 +1,11 @@
+###################################################
+# This Dockerfile is used by the docker-compose.yml
+# file to build the development container.
+# Do not make any changes here unless you know what
+# you are doing.
+###################################################
+
+FROM node:18-bullseye as dev
+RUN apt-get update -y && apt-get upgrade -y && apt-get install -y openssl
+WORKDIR /app
+CMD ["sh", "./bin/docker-start"]

README.md

@@ -6,14 +6,14 @@
 
 Actual is a local-first personal finance tool. It is 100% free and open-source, written in NodeJS, it has a synchronization element so that all your changes can move between devices without any heavy lifting.
 
-If you are interested in contributing, or want to know how development works, see [CONTRIBUTING.md](https://github.com/actualbudget/actual/blob/master/CONTRIBUTING.md) we would love to have you.
+If you are interested in contributing, or want to know how development works, see our [contributing](https://actualbudget.org/docs/contributing/) document we would love to have you.
 
 Want to say thanks? Click the ⭐ at the top of the page.
 
 ## Key Links
 
 - Actual [discord](https://discord.gg/pRYNYr4W5A) community.
-- Actual [Community Documentation](https://actualbudget.github.io/docs)
+- Actual [Community Documentation](https://actualbudget.org/docs)
 
 ## Installation
 
@@ -23,22 +23,28 @@ If you are only interested in running the latest version and not contributing to
 
 The easiest way to get Actual running is to use the [actual-server](https://github.com/actualbudget/actual-server) project. That is the server for syncing changes across devices, and it comes with the latest version of Actual. The server will provide both the web project and a server for syncing.
 
-You can get up and running quickly and easily by following our [Running Actual Locally Guide](https://actualbudget.github.io/docs/Installing/Local/your-own-machine)
+You can get up and running quickly and easily by following our [Running Actual Locally Guide](https://actualbudget.org/docs/install/local)
 
 ## Documentation
 
-We have a wide range of documentation on how to use Actual, this is all available in our [Community Documentation](https://actualbudget.github.io/docs), this includes topics on Budgeting, Account Management, Tips & Tricks and some documentation for developers.
+We have a wide range of documentation on how to use Actual, this is all available in our [Community Documentation](https://actualbudget.org/docs), this includes topics on Budgeting, Account Management, Tips & Tricks and some documentation for developers.
 
 ## Code structure
 
 The Actual app is split up into a few packages:
 
 - loot-core - The core application that runs on any platform
-- loot-design - The generic design components that make up the UI
 - desktop-client - The desktop UI
 - desktop-electron - The desktop app
 
-More information on the project structure is available in our [community documentation](https://actualbudget.github.io/docs/Developers/project-layout).
+More information on the project structure is available in our [community documentation](https://actualbudget.org/docs/contributing/project-details).
+
+## Feature Requests
+
+Current feature requests can be seen [here](https://github.com/actualbudget/actual/issues?q=is%3Aissue+label%3A%22needs+votes%22+sort%3Areactions-%2B1-desc).
+Vote for your favorite requests by reacting :+1: to the top comment of the request.
+
+To add new feature requests, open a new Issue of the "Feature Request" type.
 
 ## Sponsors
 

bin/docker-start

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+#####################################################
+# This startup script is used by the docker container
+# to check if the node_modules folder is empty and
+# if so, run yarn to install the dependencies.
+#####################################################
+
+if [ ! -d "node_modules" ] || [ "$(ls -A node_modules)" = "" ]; then
+    yarn
+fi
+
+yarn start:browser

bin/format-staged

@@ -1,11 +0,0 @@
-#!/bin/sh
-FILES=$(git diff --cached --name-only --diff-filter=ACMR "*.js" "*.jsx" | sed 's| |\\ |g')
-[ -z "$FILES" ] && exit 0
-
-# Prettify all selected files
-echo "$FILES" | xargs ./node_modules/.bin/prettier --write
-
-# Add back the modified/prettified files to staging
-echo "$FILES" | xargs git add
-
-exit 0

bin/import-open-source-package

@@ -1,27 +0,0 @@
-#!/bin/sh -e
-
-ROOT=$(cd "`dirname $0`"; pwd)
-NPM_NAME="$1"
-NAME="$2"
-PACKAGE_DIR="`dirname "$ROOT"`/packages/$NAME"
-
-if [ -z "$NAME" ] || [ -z "$NPM_NAME" ]; then
-    echo "Usage: `basename $0` <npm-name> <local-name>"
-    exit 1
-fi
-
-if [ -d "$PACKAGE_DIR" ]; then
-    read -p "Package exists, remove $PACKAGE_DIR? [y/N] " -r
-    if [ -z "$REPLY" ] || [ "$REPLY" != "y" ]; then
-        exit 2
-    fi
-fi
-
-rm -rf "$PACKAGE_DIR"
-URL="`npm view "$NPM_NAME" dist.tarball`"
-TMPDIR="`mktemp -d`"
-
-cd "$TMPDIR"
-wget -O tar.tgz "$URL"
-tar xvzf tar.tgz
-mv package "$PACKAGE_DIR"

bin/make-release

@@ -1,50 +0,0 @@
-#!/bin/bash -e
-
-VERSION=""
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
-  key="$1"
-
-  case $key in
-      --version)
-      VERSION="$2"
-      shift
-      shift
-      ;;
-      *)
-      POSITIONAL+=("$1")
-      shift
-      ;;
-  esac
-done
-set -- "${POSITIONAL[@]}"
-
-NOTES="$@"
-
-if [ -z "$VERSION" ]; then
-    echo "--version is required";
-    exit 1
-fi
-
-echo "Version: $VERSION"
-echo "Notes: $NOTES"
-read -p "Make release? [y/N] " -r
-if [ -z "$REPLY" ] || [ "$REPLY" != "y" ]; then
-    exit 2
-fi
-
-source ./.secret-tokens
-
-# Tag and push to make windows and linux versions
-git push origin master
-git tag -a "$VERSION" -m "$NOTES"
-git push origin "$VERSION"
-
-# Make a macOS version
-./bin/package --release --version "$VERSION"
-
-# TODO: browser version
-
-# Finally, update github issues
-curl -X POST -H "x-release-token: $RELEASE_TOKEN"  https://actual-automoto.fly.dev/release/"$VERSION"

bin/package

@@ -1,111 +0,0 @@
-#!/bin/bash -e
-
-
-ROOT=`dirname $0`
-VERSION=""
-BETA=""
-RELEASE=""
-RELEASE_NOTES=""
-CI=${CI:-false}
-
-cd "$ROOT/.."
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
-  key="$1"
-
-  case $key in
-      --version)
-      VERSION="$2"
-      shift
-      shift
-      ;;
-      --beta)
-      RELEASE="beta"
-      shift
-      ;;
-      --release)
-      RELEASE="production"
-      shift
-      ;;
-      *)
-      POSITIONAL+=("$1")
-      shift
-      ;;
-  esac
-done
-set -- "${POSITIONAL[@]}"
-
-if [ -z "$VERSION" ] && [ -n "$RELEASE" ]; then
-    echo "Version is required if making a release"
-    exit 1
-fi
-
-if [ -n "$RELEASE" ]; then
-    if [ -z "$CIRCLE_TAG" ]; then
-        read -p "Make release: $RELEASE v$VERSION? [y/N] " -r
-        if [ -z "$REPLY" ] || [ "$REPLY" != "y" ]; then
-            exit 2
-        fi
-    fi
-
-    if [ "$RELEASE" == "production" ]; then
-        if [ -z "$CIRCLE_TAG" ]; then
-            RELEASE_NOTES=`git tag -l --format="%(contents:subject)" "$VERSION"`
-        else
-            RELEASE_NOTES=`git tag -l --format="%(contents:subject)" "$CIRCLE_TAG"`
-        fi
-    fi
-
-    PACKAGE_VERSION=`node -p -e "require('./packages/desktop-electron/package.json').version"`
-    if [ "$VERSION" != "$PACKAGE_VERSION" ]; then
-        echo "Version in desktop-electron/package.json does not match given version! ($PACKAGE_VERSION)"
-        exit 4
-    fi
-fi
-
-if [ "$OSTYPE" == "msys" ]; then
-    if [ $CI != true ]; then
-        read -s -p "Windows certificate password: " -r CSC_KEY_PASSWORD
-        export CSC_KEY_PASSWORD
-    elif [ -n "$CIRCLE_TAG" ]; then
-        # We only want to run this on CircleCI as Github doesn't have the CSC_KEY_PASSWORD secret set.
-        certutil -f -p ${CSC_KEY_PASSWORD} -importPfx ~/windows-shift-reset-llc.p12
-    fi
-fi
-
-# We only need to run linting once (and this doesn't seem to work on
-# Windows for some reason)
-if [[ $CI != true && "$OSTYPE" == "darwin"* ]]; then
-    yarn lint
-fi
-
-yarn patch-package
-
-yarn workspace loot-core build:node
-
-yarn workspace @actual-app/web build
-
-yarn workspace Actual update-client
-
-(
-    cd packages/desktop-electron;
-    yarn clean;
-
-    export npm_config_better_sqlite3_binary_host="https://static.actualbudget.com/prebuild/better-sqlite3"
-
-    if [ "$RELEASE" == "production" ]; then
-        if [ -f ../../.secret-tokens ]; then
-            source ../../.secret-tokens
-        fi
-        yarn build --publish always -c.releaseInfo.releaseNotes="$RELEASE_NOTES" --arm64 --x64
-
-        echo "\nCreated release $VERSION with release notes \"$RELEASE_NOTES\""
-    elif [ "$RELEASE" == "beta" ]; then
-        yarn build --publish never --arm64 --x64
-        
-        echo "\nCreated beta release $VERSION"
-    else
-        SKIP_NOTARIZATION=true yarn build --publish never --x64
-    fi
-)

bin/package-browser

@@ -1,63 +1,10 @@
 #!/bin/bash -e
 
 ROOT=`dirname $0`
-VERSION=""
-RELEASE=""
-CI=${CI:-false}
 
 cd "$ROOT/.."
 
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
-  key="$1"
-
-  case $key in
-      --version)
-      VERSION="$2"
-      shift
-      shift
-      ;;
-      --beta)
-      RELEASE="beta"
-      shift
-      ;;
-      --release)
-      RELEASE="production"
-      shift
-      ;;
-      *)
-      POSITIONAL+=("$1")
-      shift
-      ;;
-  esac
-done
-set -- "${POSITIONAL[@]}"
-
-if [ -z "$VERSION" ] && [ -n "$RELEASE" ]; then
-    echo "Version is required if making a release"
-    exit 1
-fi
-
-if [ -n "$RELEASE" ]; then
-    read -p "Deploy release for browser: $RELEASE v$VERSION? [y/N] " -r
-    if [ -z "$REPLY" ] || [ "$REPLY" != "y" ]; then
-        exit 2
-    fi
-
-    PACKAGE_VERSION=`node -p -e "require('./packages/desktop-electron/package.json').version"`
-    if [ "$VERSION" != "$PACKAGE_VERSION" ] && [ "$VERSION-next" != "$PACKAGE_VERSION" ]; then
-        echo "Version in desktop-electron/package.json does not match given version! ($PACKAGE_VERSION)"
-        exit 4
-    fi
-fi
-
-# There's no need to check linting in CI as it'll be done in a different step.
-if [ $CI != true ]; then
-    yarn lint
-fi
-
-ACTUAL_RELEASE_TYPE=$RELEASE yarn workspace loot-core build:browser
-
-REACT_APP_RELEASE_TYPE=$RELEASE yarn workspace @actual-app/web build:browser
+yarn workspace loot-core build:browser
+yarn workspace @actual-app/web build:browser
 
 echo "packages/desktop-client/build"

bin/package-electron

@@ -0,0 +1,59 @@
+#!/bin/bash -e
+
+
+ROOT=`dirname $0`
+RELEASE=""
+CI=${CI:-false}
+
+cd "$ROOT/.."
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+      --release)
+      RELEASE="production"
+      shift
+      ;;
+      *)
+      POSITIONAL+=("$1")
+      shift
+      ;;
+  esac
+done
+set -- "${POSITIONAL[@]}"
+
+if [ "$OSTYPE" == "msys" ]; then
+    if [ $CI != true ]; then
+        read -s -p "Windows certificate password: " -r CSC_KEY_PASSWORD
+        export CSC_KEY_PASSWORD
+    elif [ -n "$CIRCLE_TAG" ]; then
+        # We only want to run this on CircleCI as Github doesn't have the CSC_KEY_PASSWORD secret set.
+        certutil -f -p ${CSC_KEY_PASSWORD} -importPfx ~/windows-shift-reset-llc.p12
+    fi
+fi
+
+yarn rebuild-electron
+
+yarn workspace loot-core build:node
+
+yarn workspace @actual-app/web build
+
+yarn workspace desktop-electron update-client
+
+(
+    cd packages/desktop-electron;
+    yarn clean;
+
+    if [ "$RELEASE" == "production" ]; then
+        if [ -f ../../.secret-tokens ]; then
+            source ../../.secret-tokens
+        fi
+        yarn build --publish never --arm64 --x64
+
+        echo "\nCreated release"
+    else
+        SKIP_NOTARIZATION=true yarn build --publish never --x64
+    fi
+)

docker-compose.yml

@@ -0,0 +1,16 @@
+###################################################
+# This creates and stands up the development
+# docker container. Depends on the Dockerfile and
+# docker-start.sh files.
+###################################################
+
+services:
+  actual-development:
+    build: .
+    image: actual-development
+    ports:
+      - '3001:3001'
+    volumes:
+      - '.:/app'
+    restart: 'no'
+

package.json

@@ -19,42 +19,52 @@
   },
   "scripts": {
     "start": "yarn start:browser",
-    "start:desktop": "npm-run-all --parallel 'start:desktop-*'",
+    "start:desktop": "yarn rebuild-electron && npm-run-all --parallel 'start:desktop-*'",
     "start:desktop-node": "yarn workspace loot-core watch:node",
     "start:desktop-client": "yarn workspace @actual-app/web watch",
-    "start:desktop-electron": "yarn workspace Actual watch",
+    "start:desktop-electron": "yarn workspace desktop-electron watch",
+    "start:electron": "yarn start:desktop",
     "start:browser": "npm-run-all --parallel 'start:browser-*'",
     "start:browser-backend": "yarn workspace loot-core watch:browser",
     "start:browser-frontend": "yarn workspace @actual-app/web start:browser",
     "build:browser": "./bin/package-browser",
-    "test": "yarn workspaces foreach --parallel --verbose run test",
-    "test:debug": "yarn workspaces foreach --verbose run test",
-    "e2e": "yarn workspaces foreach --parallel --verbose run e2e",
+    "build:desktop": "./bin/package-electron",
+    "build:api": "yarn workspace @actual-app/api build",
+    "test": "yarn workspaces foreach --all --parallel --verbose run test",
+    "test:debug": "yarn workspaces foreach --all --verbose run test",
+    "e2e": "yarn workspaces foreach --all --parallel --verbose run e2e",
+    "vrt": "yarn workspaces foreach --all --parallel --verbose run vrt",
     "rebuild-electron": "./node_modules/.bin/electron-rebuild -f -m ./packages/loot-core",
     "rebuild-node": "yarn workspace loot-core rebuild",
-    "lint": "cross-env NODE_ENV=development yarn workspaces foreach --verbose run lint --max-warnings 0",
-    "typecheck": "yarn tsc",
-    "postinstall": "patch-package"
+    "lint": "eslint . --max-warnings 0 --ext .js,.jsx,.ts,.tsx",
+    "lint:verbose": "DEBUG=eslint:cli-engine eslint . --max-warnings 0",
+    "typecheck": "yarn tsc && tsc-strict",
+    "jq": "./node_modules/node-jq/bin/jq"
   },
   "devDependencies": {
-    "cross-env": "^5.1.5",
+    "cross-env": "^7.0.3",
     "eslint": "^8.37.0",
+    "eslint-config-prettier": "^9.1.0",
     "eslint-config-react-app": "7.0.1",
-    "eslint-plugin-prettier": "4.2.1",
+    "eslint-import-resolver-typescript": "3.5.5",
+    "eslint-plugin-import": "2.27.5",
+    "eslint-plugin-prettier": "5.1.3",
+    "eslint-plugin-react": "7.32.2",
     "eslint-plugin-rulesdir": "^0.2.2",
+    "node-jq": "^4.0.1",
     "npm-run-all": "^4.1.3",
-    "patch-package": "^6.1.2",
-    "prettier": "2.8.2",
+    "prettier": "3.2.4",
     "react-refresh": "^0.14.0",
     "source-map-support": "^0.5.21",
-    "typescript": "^5.0.2"
+    "typescript": "^5.0.2",
+    "typescript-strict-plugin": "^2.2.2-beta.2"
   },
-  "resolutions": {
-    "react-error-overlay": "6.0.9"
+  "engines": {
+    "node": ">=18.0.0"
   },
-  "packageManager": "yarn@3.4.1",
+  "packageManager": "yarn@4.0.1",
   "browserslist": [
-    "electron 12.0",
+    "electron 24.0",
     "defaults"
   ]
 }

packages/api/.eslintignore

@@ -1,2 +0,0 @@
-app/bundle.api.js
-dist

packages/api/.gitignore

@@ -1,3 +1,5 @@
 app/bundle.api.js*
+app/stats.json
 migrations
 default-db.sqlite
+mocks/budgets/**/*

packages/api/README.md

@@ -2,4 +2,4 @@
 npm install @actual-app/api
 ```
 
-View docs here: https://actualbudget.github.io/docs/Developers/using-the-API
+View docs here: https://actualbudget.org/docs/api/

packages/api/__snapshots__/methods.test.ts.snap

@@ -0,0 +1,21 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`API setup and teardown successfully loads budget 1`] = `
+Array [
+  "2016-10",
+  "2016-11",
+  "2016-12",
+  "2017-01",
+  "2017-02",
+  "2017-03",
+  "2017-04",
+  "2017-05",
+  "2017-06",
+  "2017-07",
+  "2017-08",
+  "2017-09",
+  "2017-10",
+  "2017-11",
+  "2017-12",
+]
+`;

packages/api/app/query.js

@@ -23,7 +23,7 @@ class Query {
   }
 
   unfilter(exprs) {
-    let exprSet = new Set(exprs);
+    const exprSet = new Set(exprs);
     return new Query({
       ...this.state,
       filterExpressions: this.state.filterExpressions.filter(
@@ -37,13 +37,13 @@ class Query {
       exprs = [exprs];
     }
 
-    let query = new Query({ ...this.state, selectExpressions: exprs });
+    const query = new Query({ ...this.state, selectExpressions: exprs });
     query.state.calculation = false;
     return query;
   }
 
   calculate(expr) {
-    let query = this.select({ result: expr });
+    const query = this.select({ result: expr });
     query.state.calculation = true;
     return query;
   }
@@ -99,6 +99,6 @@ class Query {
   }
 }
 
-export default function q(table) {
+export function q(table) {
   return new Query({ table });
 }

packages/api/index.js

@@ -1,31 +0,0 @@
-import fetch from 'node-fetch';
-
-import * as bundle from './app/bundle.api';
-import * as injected from './injected';
-
-let actualApp;
-export const internal = bundle.lib;
-
-export * as methods from './methods';
-export * as utils from './utils';
-
-export async function init(config = {}) {
-  if (actualApp) {
-    return;
-  }
-
-  global.fetch = fetch;
-
-  await bundle.init(config);
-  actualApp = bundle.lib;
-
-  injected.override(bundle.lib.send);
-  return bundle.lib;
-}
-
-export async function shutdown() {
-  if (actualApp) {
-    await actualApp.send('close-budget');
-    actualApp = null;
-  }
-}

packages/api/index.ts

@@ -0,0 +1,53 @@
+import type {
+  RequestInfo as FetchInfo,
+  RequestInit as FetchInit,
+  // @ts-ignore: false-positive commonjs module error on build until typescript 5.3
+} from 'node-fetch'; // with { 'resolution-mode': 'import' };
+
+// loot-core types
+import type { InitConfig } from 'loot-core/server/main';
+
+// @ts-ignore: bundle not available until we build it
+// eslint-disable-next-line import/extensions
+import * as bundle from './app/bundle.api.js';
+import * as injected from './injected';
+import { validateNodeVersion } from './validateNodeVersion';
+
+let actualApp: null | typeof bundle.lib;
+export const internal = bundle.lib;
+
+// DEPRECATED: remove the next line in @actual-app/api v7
+export * as methods from './methods';
+
+export * from './methods';
+export * as utils from './utils';
+
+export async function init(config: InitConfig = {}) {
+  if (actualApp) {
+    return;
+  }
+
+  validateNodeVersion();
+
+  if (!globalThis.fetch) {
+    globalThis.fetch = (url: URL | RequestInfo, init?: RequestInit) => {
+      return import('node-fetch').then(({ default: fetch }) =>
+        fetch(url as unknown as FetchInfo, init as unknown as FetchInit),
+      ) as unknown as Promise<Response>;
+    };
+  }
+
+  await bundle.init(config);
+  actualApp = bundle.lib;
+
+  injected.override(bundle.lib.send);
+  return bundle.lib;
+}
+
+export async function shutdown() {
+  if (actualApp) {
+    await actualApp.send('sync');
+    await actualApp.send('close-budget');
+    actualApp = null;
+  }
+}

packages/api/jest.config.js

@@ -0,0 +1,24 @@
+module.exports = {
+  moduleFileExtensions: [
+    'testing.js',
+    'testing.ts',
+    'api.js',
+    'api.ts',
+    'api.tsx',
+    'electron.js',
+    'electron.ts',
+    'mjs',
+    'js',
+    'ts',
+    'tsx',
+    'json',
+  ],
+  testEnvironment: 'node',
+  testPathIgnorePatterns: ['/node_modules/'],
+  watchPathIgnorePatterns: ['<rootDir>/mocks/budgets/'],
+  setupFilesAfterEnv: ['<rootDir>/../loot-core/src/mocks/setup.ts'],
+  transformIgnorePatterns: ['/node_modules/'],
+  transform: {
+    '^.+\\.(t|j)sx?$': '@swc/jest',
+  },
+};

packages/api/methods.js

@@ -1,151 +0,0 @@
-import * as injected from './injected';
-
-export { default as q } from './app/query';
-
-function send(name, args) {
-  return injected.send(name, args);
-}
-
-export async function runImport(name, func) {
-  await send('api/start-import', { budgetName: name });
-  try {
-    await func();
-  } catch (e) {
-    await send('api/abort-import');
-    throw e;
-  }
-  await send('api/finish-import');
-}
-
-export async function loadBudget(budgetId) {
-  return send('api/load-budget', { id: budgetId });
-}
-
-export async function downloadBudget(syncId, { password } = {}) {
-  return send('api/download-budget', { syncId, password });
-}
-
-export async function batchBudgetUpdates(func) {
-  await send('api/batch-budget-start');
-  try {
-    await func();
-  } finally {
-    await send('api/batch-budget-end');
-  }
-}
-
-export function runQuery(query) {
-  return send('api/query', { query: query.serialize() });
-}
-
-export function getBudgetMonths() {
-  return send('api/budget-months');
-}
-
-export function getBudgetMonth(month) {
-  return send('api/budget-month', { month });
-}
-
-export function setBudgetAmount(month, categoryId, value) {
-  return send('api/budget-set-amount', { month, categoryId, amount: value });
-}
-
-export function setBudgetCarryover(month, categoryId, flag) {
-  return send('api/budget-set-carryover', { month, categoryId, flag });
-}
-
-export function addTransactions(accountId, transactions) {
-  return send('api/transactions-add', { accountId, transactions });
-}
-
-export function importTransactions(accountId, transactions) {
-  return send('api/transactions-import', { accountId, transactions });
-}
-
-export function getTransactions(accountId, startDate, endDate) {
-  return send('api/transactions-get', { accountId, startDate, endDate });
-}
-
-export function filterTransactions(accountId, text) {
-  return send('api/transactions-filter', { accountId, text });
-}
-
-export function updateTransaction(id, fields) {
-  return send('api/transaction-update', { id, fields });
-}
-
-export function deleteTransaction(id) {
-  return send('api/transaction-delete', { id });
-}
-
-export function getAccounts() {
-  return send('api/accounts-get');
-}
-
-export function createAccount(account, initialBalance) {
-  return send('api/account-create', { account, initialBalance });
-}
-
-export function updateAccount(id, fields) {
-  return send('api/account-update', { id, fields });
-}
-
-export function closeAccount(id, transferAccountId, transferCategoryId) {
-  return send('api/account-close', {
-    id,
-    transferAccountId,
-    transferCategoryId,
-  });
-}
-
-export function reopenAccount(id) {
-  return send('api/account-reopen', { id });
-}
-
-export function deleteAccount(id) {
-  return send('api/account-delete', { id });
-}
-
-export function createCategoryGroup(group) {
-  return send('api/category-group-create', { group });
-}
-
-export function updateCategoryGroup(id, fields) {
-  return send('api/category-group-update', { id, fields });
-}
-
-export function deleteCategoryGroup(id, transferCategoryId) {
-  return send('api/category-group-delete', { id, transferCategoryId });
-}
-
-export function getCategories() {
-  return send('api/categories-get', { grouped: false });
-}
-
-export function createCategory(category) {
-  return send('api/category-create', { category });
-}
-
-export function updateCategory(id, fields) {
-  return send('api/category-update', { id, fields });
-}
-
-export function deleteCategory(id, transferCategoryId) {
-  return send('api/category-delete', { id, transferCategoryId });
-}
-
-export function getPayees() {
-  return send('api/payees-get');
-}
-
-export function createPayee(payee) {
-  return send('api/payee-create', { payee });
-}
-
-export function updatePayee(id, fields) {
-  return send('api/payee-update', { id, fields });
-}
-
-export function deletePayee(id) {
-  return send('api/payee-delete', { id });
-}

packages/api/methods.test.ts

@@ -0,0 +1,389 @@
+// @ts-strict-ignore
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+import * as api from './index';
+
+const budgetName = 'test-budget';
+
+beforeEach(async () => {
+  // we need real datetime if we are going to mix new timestamps with our mock data
+  global.restoreDateNow();
+
+  const budgetPath = path.join(__dirname, '/mocks/budgets/', budgetName);
+  await fs.rm(budgetPath, { force: true, recursive: true });
+
+  await createTestBudget('default-budget-template', budgetName);
+  await api.init({
+    dataDir: path.join(__dirname, '/mocks/budgets/'),
+  });
+});
+
+afterEach(async () => {
+  global.currentMonth = null;
+  await api.shutdown();
+});
+
+async function createTestBudget(templateName: string, name: string) {
+  const templatePath = path.join(
+    __dirname,
+    '/../loot-core/src/mocks/files',
+    templateName,
+  );
+  const budgetPath = path.join(__dirname, '/mocks/budgets/', name);
+
+  await fs.mkdir(budgetPath);
+  await fs.copyFile(
+    path.join(templatePath, 'metadata.json'),
+    path.join(budgetPath, 'metadata.json'),
+  );
+  await fs.copyFile(
+    path.join(templatePath, 'db.sqlite'),
+    path.join(budgetPath, 'db.sqlite'),
+  );
+}
+
+describe('API setup and teardown', () => {
+  // apis: loadBudget, getBudgetMonths
+  test('successfully loads budget', async () => {
+    await expect(api.loadBudget(budgetName)).resolves.toBeUndefined();
+
+    await expect(api.getBudgetMonths()).resolves.toMatchSnapshot();
+  });
+});
+
+describe('API CRUD operations', () => {
+  beforeEach(async () => {
+    // load test budget
+    await api.loadBudget(budgetName);
+  });
+
+  // apis: createCategoryGroup, updateCategoryGroup, deleteCategoryGroup
+  test('CategoryGroups: successfully update category groups', async () => {
+    const month = '2023-10';
+    global.currentMonth = month;
+
+    // create our test category group
+    const mainGroupId = await api.createCategoryGroup({
+      name: 'test-group',
+    });
+
+    let budgetMonth = await api.getBudgetMonth(month);
+    expect(budgetMonth.categoryGroups).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: mainGroupId,
+        }),
+      ]),
+    );
+
+    // update group
+    await api.updateCategoryGroup(mainGroupId, {
+      name: 'update-tests',
+    });
+
+    budgetMonth = await api.getBudgetMonth(month);
+    expect(budgetMonth.categoryGroups).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: mainGroupId,
+        }),
+      ]),
+    );
+
+    // delete group
+    await api.deleteCategoryGroup(mainGroupId);
+
+    budgetMonth = await api.getBudgetMonth(month);
+    expect(budgetMonth.categoryGroups).toEqual(
+      expect.arrayContaining([
+        expect.not.objectContaining({
+          id: mainGroupId,
+        }),
+      ]),
+    );
+  });
+
+  // apis: createCategory, getCategories, updateCategory, deleteCategory
+  test('Categories: successfully update categories', async () => {
+    const month = '2023-10';
+    global.currentMonth = month;
+
+    // create our test category group
+    const mainGroupId = await api.createCategoryGroup({
+      name: 'test-group',
+    });
+    const secondaryGroupId = await api.createCategoryGroup({
+      name: 'test-secondary-group',
+    });
+    const categoryId = await api.createCategory({
+      name: 'test-budget',
+      group_id: mainGroupId,
+    });
+    const categoryIdHidden = await api.createCategory({
+      name: 'test-budget-hidden',
+      group_id: mainGroupId,
+      hidden: true,
+    });
+
+    let categories = await api.getCategories();
+    expect(categories).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: categoryId,
+          name: 'test-budget',
+          hidden: false,
+          group_id: mainGroupId,
+        }),
+        expect.objectContaining({
+          id: categoryIdHidden,
+          name: 'test-budget-hidden',
+          hidden: true,
+          group_id: mainGroupId,
+        }),
+      ]),
+    );
+
+    // update/move category
+    await api.updateCategory(categoryId, {
+      name: 'updated-budget',
+      group_id: secondaryGroupId,
+    });
+
+    await api.updateCategory(categoryIdHidden, {
+      name: 'updated-budget-hidden',
+      group_id: secondaryGroupId,
+      hidden: false,
+    });
+
+    categories = await api.getCategories();
+    expect(categories).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: categoryId,
+          name: 'updated-budget',
+          hidden: false,
+          group_id: secondaryGroupId,
+        }),
+        expect.objectContaining({
+          id: categoryIdHidden,
+          name: 'updated-budget-hidden',
+          hidden: false,
+          group_id: secondaryGroupId,
+        }),
+      ]),
+    );
+
+    // delete categories
+    await api.deleteCategory(categoryId);
+
+    expect(categories).toEqual(
+      expect.arrayContaining([
+        expect.not.objectContaining({
+          id: categoryId,
+        }),
+      ]),
+    );
+  });
+
+  // apis: setBudgetAmount, setBudgetCarryover, getBudgetMonth
+  test('Budgets: successfully update budgets', async () => {
+    const month = '2023-10';
+    global.currentMonth = month;
+
+    // create some new categories to test with
+    const groupId = await api.createCategoryGroup({
+      name: 'tests',
+    });
+    const categoryId = await api.createCategory({
+      name: 'test-budget',
+      group_id: groupId,
+    });
+
+    await api.setBudgetAmount(month, categoryId, 100);
+    await api.setBudgetCarryover(month, categoryId, true);
+
+    const budgetMonth = await api.getBudgetMonth(month);
+    expect(budgetMonth.categoryGroups).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: groupId,
+          categories: expect.arrayContaining([
+            expect.objectContaining({
+              id: categoryId,
+              budgeted: 100,
+              carryover: true,
+            }),
+          ]),
+        }),
+      ]),
+    );
+  });
+
+  //apis: createAccount, getAccounts, updateAccount, closeAccount, deleteAccount, reopenAccount
+  test('Accounts: successfully complete account operators', async () => {
+    const accountId1 = await api.createAccount(
+      { name: 'test-account1', offbudget: true },
+      1000,
+    );
+    const accountId2 = await api.createAccount({ name: 'test-account2' }, 0);
+    let accounts = await api.getAccounts();
+
+    // accounts successfully created
+    expect(accounts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: accountId1,
+          name: 'test-account1',
+          offbudget: true,
+        }),
+        expect.objectContaining({ id: accountId2, name: 'test-account2' }),
+      ]),
+    );
+
+    await api.updateAccount(accountId1, { offbudget: false });
+    await api.closeAccount(accountId1, accountId2, null);
+    await api.deleteAccount(accountId2);
+
+    // accounts successfully updated, and one of them deleted
+    accounts = await api.getAccounts();
+    expect(accounts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: accountId1,
+          name: 'test-account1',
+          closed: true,
+          offbudget: false,
+        }),
+        expect.not.objectContaining({ id: accountId2 }),
+      ]),
+    );
+
+    await api.reopenAccount(accountId1);
+
+    // the non-deleted account is reopened
+    accounts = await api.getAccounts();
+    expect(accounts).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: accountId1,
+          name: 'test-account1',
+          closed: false,
+        }),
+      ]),
+    );
+  });
+
+  // apis: createPayee, getPayees, updatePayee, deletePayee
+  test('Payees: successfully update payees', async () => {
+    const payeeId1 = await api.createPayee({ name: 'test-payee1' });
+    const payeeId2 = await api.createPayee({ name: 'test-payee2' });
+    let payees = await api.getPayees();
+
+    // payees successfully created
+    expect(payees).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: payeeId1,
+          name: 'test-payee1',
+        }),
+        expect.objectContaining({
+          id: payeeId2,
+          name: 'test-payee2',
+        }),
+      ]),
+    );
+
+    await api.updatePayee(payeeId1, { name: 'test-updated-payee' });
+    await api.deletePayee(payeeId2);
+
+    // confirm update and delete were successful
+    payees = await api.getPayees();
+    expect(payees).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: payeeId1,
+          name: 'test-updated-payee',
+        }),
+        expect.not.objectContaining({
+          name: 'test-payee1',
+        }),
+        expect.not.objectContaining({
+          id: payeeId2,
+        }),
+      ]),
+    );
+  });
+
+  // apis: addTransactions, getTransactions, importTransactions, updateTransaction, deleteTransaction
+  test('Transactions: successfully update transactions', async () => {
+    const accountId = await api.createAccount({ name: 'test-account' }, 0);
+
+    let newTransaction = [
+      { date: '2023-11-03', imported_id: '11', amount: 100 },
+      { date: '2023-11-03', imported_id: '11', amount: 100 },
+    ];
+
+    const addResult = await api.addTransactions(accountId, newTransaction, {
+      learnCategories: true,
+      runTransfers: true,
+    });
+    expect(addResult).toBe('ok');
+
+    // confirm added transactions exist
+    let transactions = await api.getTransactions(
+      accountId,
+      '2023-11-01',
+      '2023-11-30',
+    );
+    expect(transactions).toEqual(
+      expect.arrayContaining(
+        newTransaction.map(trans => expect.objectContaining(trans)),
+      ),
+    );
+    expect(transactions).toHaveLength(2);
+
+    newTransaction = [
+      { date: '2023-12-03', imported_id: '11', amount: 100 },
+      { date: '2023-12-03', imported_id: '22', amount: 200 },
+    ];
+
+    const reconciled = await api.importTransactions(accountId, newTransaction);
+
+    // Expect it to reconcile and to have updated one of the previous transactions
+    expect(reconciled.added).toHaveLength(1);
+    expect(reconciled.updated).toHaveLength(1);
+
+    // confirm imported transactions exist
+    transactions = await api.getTransactions(
+      accountId,
+      '2023-12-01',
+      '2023-12-31',
+    );
+    expect(transactions).toEqual(
+      expect.arrayContaining(
+        newTransaction.map(trans => expect.objectContaining(trans)),
+      ),
+    );
+    expect(transactions).toHaveLength(2);
+
+    const idToUpdate = reconciled.added[0];
+    const idToDelete = reconciled.updated[0];
+    await api.updateTransaction(idToUpdate, { amount: 500 });
+    await api.deleteTransaction(idToDelete);
+
+    // confirm updates and deletions work
+    transactions = await api.getTransactions(
+      accountId,
+      '2023-12-01',
+      '2023-12-31',
+    );
+    expect(transactions).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ id: idToUpdate, amount: 500 }),
+        expect.not.objectContaining({ id: idToDelete }),
+      ]),
+    );
+    expect(transactions).toHaveLength(1);
+  });
+});

packages/api/methods.ts

@@ -0,0 +1,166 @@
+// @ts-strict-ignore
+import type { Handlers } from 'loot-core/src/types/handlers';
+
+import * as injected from './injected';
+
+export { q } from './app/query';
+
+function send<K extends keyof Handlers, T extends Handlers[K]>(
+  name: K,
+  args?: Parameters<T>[0],
+): Promise<Awaited<ReturnType<T>>> {
+  return injected.send(name, args);
+}
+
+export async function runImport(name, func) {
+  await send('api/start-import', { budgetName: name });
+  try {
+    await func();
+  } catch (e) {
+    await send('api/abort-import');
+    throw e;
+  }
+  await send('api/finish-import');
+}
+
+export async function loadBudget(budgetId) {
+  return send('api/load-budget', { id: budgetId });
+}
+
+export async function downloadBudget(syncId, { password }: { password? } = {}) {
+  return send('api/download-budget', { syncId, password });
+}
+
+export async function sync() {
+  return send('api/sync');
+}
+
+export async function batchBudgetUpdates(func) {
+  await send('api/batch-budget-start');
+  try {
+    await func();
+  } finally {
+    await send('api/batch-budget-end');
+  }
+}
+
+export function runQuery(query) {
+  return send('api/query', { query: query.serialize() });
+}
+
+export function getBudgetMonths() {
+  return send('api/budget-months');
+}
+
+export function getBudgetMonth(month) {
+  return send('api/budget-month', { month });
+}
+
+export function setBudgetAmount(month, categoryId, value) {
+  return send('api/budget-set-amount', { month, categoryId, amount: value });
+}
+
+export function setBudgetCarryover(month, categoryId, flag) {
+  return send('api/budget-set-carryover', { month, categoryId, flag });
+}
+
+export function addTransactions(
+  accountId,
+  transactions,
+  { learnCategories = false, runTransfers = false } = {},
+) {
+  return send('api/transactions-add', {
+    accountId,
+    transactions,
+    learnCategories,
+    runTransfers,
+  });
+}
+
+export function importTransactions(accountId, transactions) {
+  return send('api/transactions-import', { accountId, transactions });
+}
+
+export function getTransactions(accountId, startDate, endDate) {
+  return send('api/transactions-get', { accountId, startDate, endDate });
+}
+
+export function updateTransaction(id, fields) {
+  return send('api/transaction-update', { id, fields });
+}
+
+export function deleteTransaction(id) {
+  return send('api/transaction-delete', { id });
+}
+
+export function getAccounts() {
+  return send('api/accounts-get');
+}
+
+export function createAccount(account, initialBalance?) {
+  return send('api/account-create', { account, initialBalance });
+}
+
+export function updateAccount(id, fields) {
+  return send('api/account-update', { id, fields });
+}
+
+export function closeAccount(id, transferAccountId?, transferCategoryId?) {
+  return send('api/account-close', {
+    id,
+    transferAccountId,
+    transferCategoryId,
+  });
+}
+
+export function reopenAccount(id) {
+  return send('api/account-reopen', { id });
+}
+
+export function deleteAccount(id) {
+  return send('api/account-delete', { id });
+}
+
+export function createCategoryGroup(group) {
+  return send('api/category-group-create', { group });
+}
+
+export function updateCategoryGroup(id, fields) {
+  return send('api/category-group-update', { id, fields });
+}
+
+export function deleteCategoryGroup(id, transferCategoryId?) {
+  return send('api/category-group-delete', { id, transferCategoryId });
+}
+
+export function getCategories() {
+  return send('api/categories-get', { grouped: false });
+}
+
+export function createCategory(category) {
+  return send('api/category-create', { category });
+}
+
+export function updateCategory(id, fields) {
+  return send('api/category-update', { id, fields });
+}
+
+export function deleteCategory(id, transferCategoryId?) {
+  return send('api/category-delete', { id, transferCategoryId });
+}
+
+export function getPayees() {
+  return send('api/payees-get');
+}
+
+export function createPayee(payee) {
+  return send('api/payee-create', { payee });
+}
+
+export function updatePayee(id, fields) {
+  return send('api/payee-update', { id, fields });
+}
+
+export function deletePayee(id) {
+  return send('api/payee-delete', { id });
+}

packages/api/package.json

@@ -1,27 +1,38 @@
 {
   "name": "@actual-app/api",
-  "version": "6.0.0",
+  "version": "6.4.0",
   "license": "MIT",
   "description": "An API for Actual",
+  "engines": {
+    "node": ">=18.12.0"
+  },
   "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "types": "@types/index.d.ts",
   "files": [
-    "dist",
-    "default-db.sqlite",
-    "migrations"
+    "dist"
   ],
   "scripts": {
-    "lint": "eslint .",
     "build:app": "yarn workspace loot-core build:api",
-    "build:node": "tsc --p tsconfig.dist.json",
-    "build": "yarn run build:app && yarn run build:node"
+    "build:node": "tsc --p tsconfig.dist.json && tsc-alias -p tsconfig.dist.json",
+    "build:migrations": "cp migrations/*.sql dist/migrations",
+    "build:default-db": "cp default-db.sqlite dist/",
+    "build": "yarn run clean && yarn run build:app && yarn run build:node && yarn run build:migrations && yarn run build:default-db",
+    "test": "yarn run build:app && jest -c jest.config.js",
+    "clean": "rm -rf dist @types"
   },
   "dependencies": {
-    "better-sqlite3": "^8.2.0",
-    "node-fetch": "^2.6.9",
-    "uuid": "3.3.2"
+    "better-sqlite3": "^9.2.2",
+    "compare-versions": "^6.1.0",
+    "node-fetch": "^3.3.2",
+    "uuid": "^9.0.0"
   },
   "devDependencies": {
+    "@swc/core": "^1.3.105",
+    "@swc/jest": "^0.2.31",
+    "@types/jest": "^27.5.0",
+    "@types/uuid": "^9.0.2",
+    "jest": "^27.0.0",
+    "tsc-alias": "^1.8.8",
     "typescript": "^5.0.2"
   }
 }

packages/api/test.js

@@ -1,8 +0,0 @@
-import * as api from './index';
-
-async function run() {
-  let app = await api.init({ config: { dataDir: '/tmp' } });
-  await app.send('create-budget', { testMode: true });
-}
-
-run();

packages/api/tsconfig.dist.json

@@ -1,11 +1,18 @@
 {
   "extends": "../../tsconfig.json",
   "compilerOptions": {
-    "target": "es5",
+    // Using ES2021 because that’s the newest version where
+    // the latest Node 16.x release supports all of the features
+    "target": "ES2021",
     "module": "CommonJS",
     "noEmit": false,
     "declaration": true,
-    "outDir": "dist"
+    "outDir": "dist",
+    "declarationDir": "@types",
+    "paths": {
+      "loot-core/*": ["./@types/loot-core/*"],
+    }
   },
-  "include": ["."]
+  "include": ["."],
+  "exclude": ["**/node_modules/*", "dist", "@types"]
 }

packages/api/validateNodeVersion.js

@@ -0,0 +1,16 @@
+import { satisfies } from 'compare-versions';
+
+import * as packageJson from './package.json';
+
+export function validateNodeVersion() {
+  if (process?.versions?.node) {
+    const nodeVersion = process?.versions?.node;
+    const minimumNodeVersion = packageJson.engines.node;
+
+    if (!satisfies(nodeVersion, minimumNodeVersion)) {
+      throw new Error(
+        `@actual-app/api requires a node version ${minimumNodeVersion}. Found that you are using: ${nodeVersion}. Please upgrade to a higher version`,
+      );
+    }
+  }
+}

packages/crdt/.gitignore

@@ -0,0 +1,2 @@
+bin/protoc-gen-js
+bin/protoc

packages/crdt/README.md

@@ -0,0 +1,27 @@
+# `@actual-app/crdt`
+
+This package contains the core CRDT logic that enables Actual’s syncing. It is shared between the client and server. We may or may not follow semver when updating this package; any usage of it outside Actual is undocumented and at your own risk.
+
+## protobuf
+
+We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.
+
+### Generating protobuf
+
+The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.
+
+This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). You’ll also need to [download the latest version of `protoc-gen-js`](https://github.com/protocolbuffers/protobuf-javascript/releases/latest). For convenience, you can put both of these binaries in `./bin`.
+
+Once installed, the protobuf can be generated by running `./bin/generate-proto`.
+
+However there is one very important thing to remember! The default output includes this near the top:
+
+```
+var global = (function() { return this || window || global || self || Function('return this')(); }).call(null);
+```
+
+This will not work with our CSP directives. You must manually modify this to this:
+
+```
+var global = globalThis;
+```

packages/crdt/bin/generate-proto

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+cd "$(dirname "$(dirname "$0")")"
+
+if ! [ -x "$(command -v protoc)" ]; then
+  echo 'Error: protoc is not installed. See the readme for installation instructions.' >&2
+  exit 1
+fi
+
+export PATH="$PWD/bin:$PATH"
+
+protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \
+  --ts_opt=esModuleInterop=true \
+  --ts_out="src/proto" \
+  --js_out=import_style=commonjs,binary:src/proto \
+  --proto_path=src/proto \
+  sync.proto
+
+../../node_modules/.bin/prettier --write src/proto/*.d.ts
+
+echo 'One more step! Find the `var global = ...` declaration in src/proto/sync_pb.js and change it to `var global = globalThis;`'

packages/crdt/index.ts

@@ -0,0 +1 @@
+export * from './src/main';

packages/crdt/jest.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.(t|j)sx?$': '@swc/jest',
+  },
+};

packages/crdt/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@actual-app/crdt",
+  "version": "2.1.0",
+  "license": "MIT",
+  "description": "CRDT layer of Actual",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build:node": "tsc --p tsconfig.dist.json",
+    "proto:generate": "./bin/generate-proto",
+    "build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/",
+    "test": "jest -c jest.config.js"
+  },
+  "dependencies": {
+    "google-protobuf": "^3.12.0-rc.1",
+    "murmurhash": "^2.0.1",
+    "uuid": "^9.0.0"
+  },
+  "devDependencies": {
+    "@swc/core": "^1.3.105",
+    "@swc/jest": "^0.2.31",
+    "@types/jest": "^27.5.0",
+    "@types/uuid": "^9.0.2",
+    "jest": "^27.0.0",
+    "ts-protoc-gen": "^0.15.0",
+    "typescript": "^5.0.2"
+  }
+}

packages/crdt/src/crdt/index.ts

@@ -0,0 +1,13 @@
+import * as merkle from './merkle';
+
+export { merkle };
+export {
+  getClock,
+  setClock,
+  makeClock,
+  makeClientId,
+  serializeClock,
+  deserializeClock,
+  type Clock,
+  Timestamp,
+} from './timestamp';

packages/crdt/src/crdt/merkle.test.ts

@@ -1,13 +1,17 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
 import * as merkle from './merkle';
 import { Timestamp } from './timestamp';
 
-function message(timestamp, hash) {
-  timestamp = Timestamp.parse(timestamp);
+function message(timestampStr: string, hash: number) {
+  const timestamp = Timestamp.parse(timestampStr)!;
   timestamp.hash = () => hash;
   return { timestamp };
 }
 
-function insertMessages(trie, messages) {
+function insertMessages(
+  trie: merkle.TrieNode,
+  messages: ReturnType<typeof message>[],
+) {
   messages.forEach(msg => {
     trie = merkle.insert(trie, msg.timestamp);
   });
@@ -17,19 +21,19 @@ function insertMessages(trie, messages) {
 describe('merkle trie', () => {
   test('adding an item works', () => {
     let trie = merkle.insert(
-      {},
-      Timestamp.parse('2018-11-12T13:21:40.122Z-0000-0123456789ABCDEF'),
+      merkle.emptyTrie(),
+      Timestamp.parse('2018-11-12T13:21:40.122Z-0000-0123456789ABCDEF')!,
     );
     trie = merkle.insert(
       trie,
-      Timestamp.parse('2018-11-13T13:21:40.122Z-0000-0123456789ABCDEF'),
+      Timestamp.parse('2018-11-13T13:21:40.122Z-0000-0123456789ABCDEF')!,
     );
     expect(trie).toMatchSnapshot();
   });
 
   test('diff returns the correct time difference', () => {
-    let trie1: { hash?: unknown } = {};
-    let trie2: { hash?: unknown } = {};
+    let trie1 = merkle.emptyTrie();
+    let trie2 = merkle.emptyTrie();
 
     const messages = [
       // First client messages
@@ -51,7 +55,7 @@ describe('merkle trie', () => {
     trie2 = merkle.insert(trie2, messages[4].timestamp);
     expect(trie2.hash).toBe(108);
 
-    expect(new Date(merkle.diff(trie1, trie2)).toISOString()).toBe(
+    expect(new Date(merkle.diff(trie1, trie2)!).toISOString()).toBe(
       '2018-11-02T17:15:00.000Z',
     );
 
@@ -65,17 +69,17 @@ describe('merkle trie', () => {
   });
 
   test('diffing works with empty tries', () => {
-    let trie1 = {};
-    let trie2 = merkle.insert(
-      {},
-      Timestamp.parse('2009-01-02T10:17:37.789Z-0000-0000testinguuid1'),
+    const trie1 = merkle.emptyTrie();
+    const trie2 = merkle.insert(
+      merkle.emptyTrie(),
+      Timestamp.parse('2009-01-02T10:17:37.789Z-0000-0000testinguuid1')!,
     );
 
     expect(merkle.diff(trie1, trie2)).toBe(0);
   });
 
   test('pruning works and keeps correct hashes', () => {
-    let messages = [
+    const messages = [
       message('2018-11-01T01:00:00.000Z-0000-0123456789ABCDEF', 1000),
       message('2018-11-01T01:09:00.000Z-0000-0123456789ABCDEF', 1100),
       message('2018-11-01T01:18:00.000Z-0000-0123456789ABCDEF', 1200),
@@ -90,20 +94,20 @@ describe('merkle trie', () => {
       message('2018-11-01T02:37:00.000Z-0000-0123456789ABCDEF', 2100),
     ];
 
-    let trie: { hash?: unknown } = {};
+    let trie = merkle.emptyTrie();
     messages.forEach(msg => {
       trie = merkle.insert(trie, msg.timestamp);
     });
     expect(trie.hash).toBe(2496);
     expect(trie).toMatchSnapshot();
 
-    let pruned = merkle.prune(trie);
+    const pruned = merkle.prune(trie);
     expect(pruned.hash).toBe(2496);
     expect(pruned).toMatchSnapshot();
   });
 
   test('diffing differently shaped tries returns correct time', () => {
-    let messages = [
+    const messages = [
       message('2018-11-01T01:00:00.000Z-0000-0123456789ABCDEF', 1000),
       message('2018-11-01T01:09:00.000Z-0000-0123456789ABCDEF', 1100),
       message('2018-11-01T01:18:00.000Z-0000-0123456789ABCDEF', 1200),
@@ -118,70 +122,70 @@ describe('merkle trie', () => {
       message('2018-11-01T02:37:00.000Z-0000-0123456789ABCDEF', 2100),
     ];
 
-    let trie = insertMessages({}, messages);
+    const trie = insertMessages({}, messages);
 
     // Case 0: It always returns a base time when comparing with an
     // empty trie
-    expect(new Date(merkle.diff({}, trie)).toISOString()).toBe(
+    expect(new Date(merkle.diff(merkle.emptyTrie(), trie)!).toISOString()).toBe(
       '1970-01-01T00:00:00.000Z',
     );
-    expect(new Date(merkle.diff(trie, {})).toISOString()).toBe(
+    expect(new Date(merkle.diff(trie, merkle.emptyTrie())!).toISOString()).toBe(
       '1970-01-01T00:00:00.000Z',
     );
 
     // Case 1: Add an older message that modifies the trie in such a
     // way that it modifies the 1st out of 3 branches (so it will be
     // pruned away)
-    let trie1 = insertMessages(trie, [
+    const trie1 = insertMessages(trie, [
       message('2018-11-01T00:59:00.000Z-0000-0123456789ABCDEF', 900),
     ]);
 
-    // Normal comparision works
-    expect(new Date(merkle.diff(trie1, trie)).toISOString()).toBe(
+    // Normal comparison works
+    expect(new Date(merkle.diff(trie1, trie)!).toISOString()).toBe(
       '2018-11-01T00:54:00.000Z',
     );
 
     // Comparing the pruned new trie is lossy, so it returns an even older time
-    expect(new Date(merkle.diff(merkle.prune(trie1), trie)).toISOString()).toBe(
-      '2018-11-01T00:45:00.000Z',
-    );
+    expect(
+      new Date(merkle.diff(merkle.prune(trie1), trie)!).toISOString(),
+    ).toBe('2018-11-01T00:45:00.000Z');
 
     // Comparing the pruned original trie is just as lossy
-    expect(new Date(merkle.diff(trie1, merkle.prune(trie))).toISOString()).toBe(
-      '2018-11-01T00:45:00.000Z',
-    );
+    expect(
+      new Date(merkle.diff(trie1, merkle.prune(trie))!).toISOString(),
+    ).toBe('2018-11-01T00:45:00.000Z');
 
     // Pruning both tries is just as lossy as well, since the changed
     // key is pruned away in both cases and it won't find a changed
     // key so it bails at the point
     expect(
       new Date(
-        merkle.diff(merkle.prune(trie1), merkle.prune(trie)),
+        merkle.diff(merkle.prune(trie1), merkle.prune(trie))!,
       ).toISOString(),
     ).toBe('2018-11-01T00:45:00.000Z');
 
     // Case 2: Add two messages similar to the above case, but the
     // second message modifies the 2nd key at the same level as the
-    // first message modifiying the 1st key
-    let trie2 = insertMessages(trie, [
+    // first message modifying the 1st key
+    const trie2 = insertMessages(trie, [
       message('2018-11-01T00:59:00.000Z-0000-0123456789ABCDEF', 900),
       message('2018-11-01T01:15:00.000Z-0000-0123456789ABCDEF', 1422),
     ]);
 
-    // Normal comparision works
-    expect(new Date(merkle.diff(trie2, trie)).toISOString()).toBe(
+    // Normal comparison works
+    expect(new Date(merkle.diff(trie2, trie)!).toISOString()).toBe(
       '2018-11-01T00:54:00.000Z',
     );
 
     // Same as case 1
-    expect(new Date(merkle.diff(merkle.prune(trie2), trie)).toISOString()).toBe(
-      '2018-11-01T00:45:00.000Z',
-    );
+    expect(
+      new Date(merkle.diff(merkle.prune(trie2), trie)!).toISOString(),
+    ).toBe('2018-11-01T00:45:00.000Z');
 
     // Same as case 1
-    expect(new Date(merkle.diff(trie2, merkle.prune(trie))).toISOString()).toBe(
-      '2018-11-01T00:45:00.000Z',
-    );
+    expect(
+      new Date(merkle.diff(trie2, merkle.prune(trie))!).toISOString(),
+    ).toBe('2018-11-01T00:45:00.000Z');
 
     // Pruning both tries is very lossy and this ends up returning a
     // time that only covers the second message. Syncing will need
@@ -190,7 +194,7 @@ describe('merkle trie', () => {
     // ignores the first message.
     expect(
       new Date(
-        merkle.diff(merkle.prune(trie2), merkle.prune(trie)),
+        merkle.diff(merkle.prune(trie2), merkle.prune(trie))!,
       ).toISOString(),
     ).toBe('2018-11-01T01:12:00.000Z');
   });

packages/crdt/src/crdt/merkle.ts

@@ -0,0 +1,180 @@
+// TODO: Ok, several problems:
+//
+// * If nothing matches between two merkle trees, we should fallback
+// * to the last window instead the front one (use 0 instead of the
+// * key)
+//
+// * Need to check to make sure if account exists when handling
+// * transaction changes in syncing
+
+import { Timestamp } from './timestamp';
+
+/**
+ * Represents a node within a trinary radix trie.
+ */
+export type TrieNode = {
+  '0'?: TrieNode;
+  '1'?: TrieNode;
+  '2'?: TrieNode;
+  hash?: number;
+};
+
+type NumberTrieNodeKey = keyof Omit<TrieNode, 'hash'>;
+
+export function emptyTrie(): TrieNode {
+  return { hash: 0 };
+}
+
+function isNumberTrieNodeKey(input: string): input is NumberTrieNodeKey {
+  return ['0', '1', '2'].includes(input);
+}
+
+export function getKeys(trie: TrieNode): NumberTrieNodeKey[] {
+  return Object.keys(trie).filter(isNumberTrieNodeKey);
+}
+
+export function keyToTimestamp(key: string): number {
+  // 16 is the length of the base 3 value of the current time in
+  // minutes. Ensure it's padded to create the full value
+  const fullkey = key + '0'.repeat(16 - key.length);
+
+  // Parse the base 3 representation
+  return parseInt(fullkey, 3) * 1000 * 60;
+}
+
+/**
+ * Mutates `trie` to insert a node at `timestamp`
+ */
+export function insert(trie: TrieNode, timestamp: Timestamp) {
+  const hash = timestamp.hash();
+  const key = Number(Math.floor(timestamp.millis() / 1000 / 60)).toString(3);
+
+  trie = Object.assign({}, trie, { hash: (trie.hash || 0) ^ hash });
+  return insertKey(trie, key, hash);
+}
+
+function insertKey(trie: TrieNode, key: string, hash: number): TrieNode {
+  if (key.length === 0) {
+    return trie;
+  }
+  const c = key[0];
+  const t = isNumberTrieNodeKey(c) ? trie[c] : undefined;
+  const n = t || {};
+  return Object.assign({}, trie, {
+    [c]: Object.assign({}, n, insertKey(n, key.slice(1), hash), {
+      hash: (n.hash || 0) ^ hash,
+    }),
+  });
+}
+
+export function build(timestamps: Timestamp[]) {
+  const trie = emptyTrie();
+  for (const timestamp of timestamps) {
+    insert(trie, timestamp);
+  }
+  return trie;
+}
+
+export function diff(trie1: TrieNode, trie2: TrieNode): number | null {
+  if (trie1.hash === trie2.hash) {
+    return null;
+  }
+
+  let node1 = trie1;
+  let node2 = trie2;
+  let k = '';
+
+  // This loop will eventually stop when it traverses down to find
+  // where the hashes differ, or otherwise when there are no leaves
+  // left (this shouldn't happen, if that's the case the hash check at
+  // the top of this function should pass)
+  while (1) {
+    const keyset = new Set([...getKeys(node1), ...getKeys(node2)]);
+    const keys = [...keyset.values()];
+    keys.sort();
+
+    let diffkey: null | '0' | '1' | '2' = null;
+
+    // Traverse down the trie through keys that aren't the same. We
+    // traverse down the keys in order. Stop in two cases: either one
+    // of the nodes doesn't have the key, or a different key isn't
+    // found. For the former case, we have to that because pruning is
+    // lossy. We don't know if we've pruned off a changed key so we
+    // can't traverse down anymore. For the latter case, it means two
+    // things: either we've hit the bottom of the tree, or the changed
+    // key has been pruned off. In the latter case we have a "partial"
+    // key and will fill the rest with 0s. Note that if multiple older
+    // messages were added into one trie, it's possible we will
+    // generate a time that only encompasses *some* of the those
+    // messages. Pruning is lossy, and we traverse down the left-most
+    // changed time that we know of, because of pruning it might take
+    // multiple passes to sync up a trie.
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i];
+
+      const next1 = node1[key];
+      const next2 = node2[key];
+
+      if (!next1 || !next2) {
+        break;
+      }
+
+      if (next1.hash !== next2.hash) {
+        diffkey = key;
+        break;
+      }
+    }
+
+    if (!diffkey) {
+      return keyToTimestamp(k);
+    }
+
+    k += diffkey;
+    node1 = node1[diffkey] || emptyTrie();
+    node2 = node2[diffkey] || emptyTrie();
+  }
+
+  return null;
+}
+
+export function prune(trie: TrieNode, n = 2): TrieNode {
+  // Do nothing if empty
+  if (!trie.hash) {
+    return trie;
+  }
+
+  const keys = getKeys(trie);
+  keys.sort();
+
+  const next: TrieNode = { hash: trie.hash };
+
+  // Prune child nodes.
+  for (const k of keys.slice(-n)) {
+    const node = trie[k];
+
+    if (!node) {
+      throw new Error(`TrieNode for key ${k} could not be found`);
+    }
+
+    next[k] = prune(node, n);
+  }
+
+  return next;
+}
+
+export function debug(trie: TrieNode, k = '', indent = 0): string {
+  const str =
+    ' '.repeat(indent) +
+    (k !== '' ? `k: ${k} ` : '') +
+    `hash: ${trie.hash || '(empty)'}\n`;
+  return (
+    str +
+    getKeys(trie)
+      .map(key => {
+        const node = trie[key];
+        if (!node) return '';
+        return debug(node, key, indent + 2);
+      })
+      .join('')
+  );
+}

packages/crdt/src/crdt/timestamp.test.ts

@@ -1,8 +1,9 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
 import { Timestamp } from './timestamp';
 
 describe('Timestamp', function () {
   let now = 0;
-  let prevNow;
+  let prevNow: typeof Date.now;
 
   beforeEach(function () {
     prevNow = Date.now;
@@ -16,16 +17,18 @@ describe('Timestamp', function () {
 
   describe('comparison', function () {
     it('should be in order', function () {
-      expect(Timestamp.zero()).toBe(Timestamp.zero());
-      expect(Timestamp.max() > Timestamp.zero()).toBeTruthy();
-      expect(Timestamp.send() > Timestamp.zero()).toBeTruthy();
-      expect(Timestamp.send() < Timestamp.max()).toBeTruthy();
+      const sendTimestamp = Timestamp.send();
+
+      expect(Timestamp.zero).toBe(Timestamp.zero);
+      expect(Timestamp.max > Timestamp.zero).toBeTruthy();
+      expect(sendTimestamp && sendTimestamp > Timestamp.zero).toBeTruthy();
+      expect(sendTimestamp && sendTimestamp < Timestamp.max).toBeTruthy();
     });
   });
 
   describe('parsing', function () {
     it('should not parse', function () {
-      let invalidInputs = [
+      const invalidInputs = [
         null,
         undefined,
         {},
@@ -41,19 +44,19 @@ describe('Timestamp', function () {
         '9999-12-31T23:59:59.999Z-10000-FFFFFFFFFFFFFFFF',
         '9999-12-31T23:59:59.999Z-FFFF-10000000000000000',
       ];
-      for (let invalidInput of invalidInputs) {
-        expect(Timestamp.parse(invalidInput)).toBe(null);
+      for (const invalidInput of invalidInputs) {
+        expect(Timestamp.parse(invalidInput as string)).toBe(null);
       }
     });
 
     it('should parse', function () {
-      let validInputs = [
+      const validInputs = [
         '1970-01-01T00:00:00.000Z-0000-0000000000000000',
         '2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF',
         '9999-12-31T23:59:59.999Z-FFFF-FFFFFFFFFFFFFFFF',
       ];
-      for (let validInput of validInputs) {
-        let parsed = Timestamp.parse(validInput);
+      for (const validInput of validInputs) {
+        const parsed = Timestamp.parse(validInput)!;
         expect(typeof parsed).toBe('object');
         expect(parsed.millis() >= 0).toBeTruthy();
         expect(parsed.millis() < 253402300800000).toBeTruthy();
@@ -133,7 +136,7 @@ describe('Timestamp', function () {
       now = 52;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.051Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.051Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.052Z-0000-0000000000000001'),
@@ -141,7 +144,7 @@ describe('Timestamp', function () {
       now = 54;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.053Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.053Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.054Z-0000-0000000000000001'),
@@ -149,7 +152,7 @@ describe('Timestamp', function () {
       now = 56;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.055Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.055Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.056Z-0000-0000000000000001'),
@@ -160,7 +163,7 @@ describe('Timestamp', function () {
       now = 61;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.062Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.062Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.062Z-0001-0000000000000001'),
@@ -168,7 +171,7 @@ describe('Timestamp', function () {
       now = 62;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.062Z-0001-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.062Z-0001-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.062Z-0002-0000000000000001'),
@@ -176,7 +179,7 @@ describe('Timestamp', function () {
       now = 62;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.062Z-0002-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.062Z-0002-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.062Z-0003-0000000000000001'),
@@ -184,7 +187,7 @@ describe('Timestamp', function () {
       now = 63;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.062Z-0004-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.062Z-0004-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.063Z-0000-0000000000000001'),
@@ -195,7 +198,7 @@ describe('Timestamp', function () {
       now = 73;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.071Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.071Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.073Z-0000-0000000000000001'),
@@ -203,7 +206,7 @@ describe('Timestamp', function () {
       now = 73;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.072Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.072Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.073Z-0001-0000000000000001'),
@@ -211,7 +214,7 @@ describe('Timestamp', function () {
       now = 74;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.073Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.073Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.074Z-0000-0000000000000001'),
@@ -222,7 +225,7 @@ describe('Timestamp', function () {
       now = 81;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.083Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.083Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.083Z-0001-0000000000000001'),
@@ -230,7 +233,7 @@ describe('Timestamp', function () {
       now = 82;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.083Z-0001-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.083Z-0001-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.083Z-0002-0000000000000001'),
@@ -238,7 +241,7 @@ describe('Timestamp', function () {
       now = 83;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.083Z-0002-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.083Z-0002-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.083Z-0003-0000000000000001'),
@@ -246,7 +249,7 @@ describe('Timestamp', function () {
       now = 84;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.083Z-0003-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.083Z-0003-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.084Z-0000-0000000000000001'),
@@ -256,7 +259,7 @@ describe('Timestamp', function () {
       now = 93;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.091Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.091Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.093Z-0000-0000000000000001'),
@@ -264,7 +267,7 @@ describe('Timestamp', function () {
       now = 92;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.092Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.092Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.093Z-0001-0000000000000001'),
@@ -272,7 +275,7 @@ describe('Timestamp', function () {
       now = 91;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.093Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.093Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.093Z-0002-0000000000000001'),
@@ -282,7 +285,7 @@ describe('Timestamp', function () {
       now = 101;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.103Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.103Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.103Z-0001-0000000000000001'),
@@ -290,7 +293,7 @@ describe('Timestamp', function () {
       now = 102;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.102Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.102Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.103Z-0002-0000000000000001'),
@@ -298,7 +301,7 @@ describe('Timestamp', function () {
       now = 103;
       expect(
         Timestamp.recv(
-          Timestamp.parse('1970-01-01T00:00:00.101Z-0000-0000000000000002'),
+          Timestamp.parse('1970-01-01T00:00:00.101Z-0000-0000000000000002')!,
         ),
       ).toEqual(
         Timestamp.parse('1970-01-01T00:00:00.103Z-0003-0000000000000001'),
@@ -316,7 +319,7 @@ describe('Timestamp', function () {
     it('should fail with clock drift', function () {
       expect(function () {
         Timestamp.recv(
-          Timestamp.parse('1980-01-01T00:00:00.101Z-0000-0000000000000002'),
+          Timestamp.parse('1980-01-01T00:00:00.101Z-0000-0000000000000002')!,
         );
       }).toThrow(Timestamp.ClockDriftError);
     });

packages/crdt/src/crdt/timestamp.ts

@@ -0,0 +1,358 @@
+import murmurhash from 'murmurhash';
+import { v4 as uuidv4 } from 'uuid';
+
+import { TrieNode } from './merkle';
+
+/**
+ * Hybrid Unique Logical Clock (HULC) timestamp generator
+ *
+ * Globally-unique, monotonic timestamps are generated from the
+ *    combination of the unreliable system time, a counter, and an
+ *    identifier for the current node (instance, machine, process, etc.).
+ *    These timestamps can accommodate clock stuttering (duplicate values),
+ *    regression, and node differences within the configured maximum drift.
+ *
+ * In order to generate timestamps locally or for transmission to another
+ *    node, use the send() method. For global causality, timestamps must
+ *    be included in each message processed by the system. Whenever a
+ *    message is received, its timestamp must be passed to the recv()
+ *    method.
+ *
+ * Timestamps serialize into a 46-character collatable string
+ *    example: 2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF
+ *    example: 2015-04-24T22:23:42.123Z-1000-A219E7A71CC18912
+ *
+ * The 64-bit hybrid clock is based on the HLC specification,
+ * http://www.cse.buffalo.edu/tech-reports/2014-04.pdf
+ */
+
+export type Clock = {
+  timestamp: MutableTimestamp;
+  merkle: TrieNode;
+};
+
+// A mutable global clock
+let clock: Clock;
+
+export function setClock(clock_: Clock): void {
+  clock = clock_;
+}
+
+export function getClock(): Clock {
+  return clock;
+}
+
+export function makeClock(timestamp: Timestamp, merkle: TrieNode = {}) {
+  return { timestamp: MutableTimestamp.from(timestamp), merkle };
+}
+
+export function serializeClock(clock: Clock): string {
+  return JSON.stringify({
+    timestamp: clock.timestamp.toString(),
+    merkle: clock.merkle,
+  });
+}
+
+export function deserializeClock(clock: string): Clock {
+  let data;
+  try {
+    data = JSON.parse(clock);
+  } catch (e) {
+    data = {
+      timestamp: '1970-01-01T00:00:00.000Z-0000-' + makeClientId(),
+      merkle: {},
+    };
+  }
+
+  const ts = Timestamp.parse(data.timestamp);
+
+  if (!ts) {
+    throw new Timestamp.InvalidError(data.timestamp);
+  }
+
+  return {
+    timestamp: MutableTimestamp.from(ts),
+    merkle: data.merkle,
+  };
+}
+
+export function makeClientId() {
+  return uuidv4().replace(/-/g, '').slice(-16);
+}
+
+const config = {
+  // Allow 5 minutes of clock drift
+  maxDrift: 5 * 60 * 1000,
+};
+
+const MAX_COUNTER = parseInt('0xFFFF');
+const MAX_NODE_LENGTH = 16;
+
+/**
+ * timestamp instance class
+ */
+export class Timestamp {
+  _state: { millis: number; counter: number; node: string };
+
+  constructor(millis: number, counter: number, node: string) {
+    this._state = {
+      millis,
+      counter,
+      node,
+    };
+  }
+
+  valueOf() {
+    return this.toString();
+  }
+
+  toString() {
+    return [
+      new Date(this.millis()).toISOString(),
+      ('0000' + this.counter().toString(16).toUpperCase()).slice(-4),
+      ('0000000000000000' + this.node()).slice(-16),
+    ].join('-');
+  }
+
+  millis() {
+    return this._state.millis;
+  }
+
+  counter() {
+    return this._state.counter;
+  }
+
+  node() {
+    return this._state.node;
+  }
+
+  hash() {
+    return murmurhash.v3(this.toString());
+  }
+
+  // Timestamp generator initialization
+  // * sets the node ID to an arbitrary value
+  // * useful for mocking/unit testing
+  static init(options: { maxDrift?: number; node?: string } = {}) {
+    if (options.maxDrift) {
+      config.maxDrift = options.maxDrift;
+    }
+
+    setClock(
+      makeClock(
+        new Timestamp(
+          0,
+          0,
+          options.node
+            ? ('0000000000000000' + options.node).toString().slice(-16)
+            : '',
+        ),
+      ),
+    );
+  }
+
+  /**
+   * maximum timestamp
+   */
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  static max = Timestamp.parse(
+    '9999-12-31T23:59:59.999Z-FFFF-FFFFFFFFFFFFFFFF',
+  )!;
+
+  /**
+   * timestamp parsing
+   * converts a fixed-length string timestamp to the structured value
+   */
+  static parse(timestamp: string | Timestamp): Timestamp | null {
+    if (timestamp instanceof Timestamp) {
+      return timestamp;
+    }
+    if (typeof timestamp === 'string') {
+      const parts = timestamp.split('-');
+      if (parts && parts.length === 5) {
+        const millis = Date.parse(parts.slice(0, 3).join('-')).valueOf();
+        const counter = parseInt(parts[3], 16);
+        const node = parts[4];
+        if (
+          !isNaN(millis) &&
+          millis >= 0 &&
+          !isNaN(counter) &&
+          counter <= MAX_COUNTER &&
+          typeof node === 'string' &&
+          node.length <= MAX_NODE_LENGTH
+        ) {
+          return new Timestamp(millis, counter, node);
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Timestamp send. Generates a unique, monotonic timestamp suitable
+   * for transmission to another system in string format
+   */
+  static send(): Timestamp | null {
+    if (!clock) {
+      return null;
+    }
+
+    // retrieve the local wall time
+    const phys = Date.now();
+
+    // unpack the clock.timestamp logical time and counter
+    const lOld = clock.timestamp.millis();
+    const cOld = clock.timestamp.counter();
+
+    // calculate the next logical time and counter
+    // * ensure that the logical time never goes backward
+    // * increment the counter if phys time does not advance
+    const lNew = Math.max(lOld, phys);
+    const cNew = lOld === lNew ? cOld + 1 : 0;
+
+    // check the result for drift and counter overflow
+    if (lNew - phys > config.maxDrift) {
+      throw new Timestamp.ClockDriftError(lNew, phys, config.maxDrift);
+    }
+    if (cNew > MAX_COUNTER) {
+      throw new Timestamp.OverflowError();
+    }
+
+    // repack the logical time/counter
+    clock.timestamp.setMillis(lNew);
+    clock.timestamp.setCounter(cNew);
+
+    return new Timestamp(
+      clock.timestamp.millis(),
+      clock.timestamp.counter(),
+      clock.timestamp.node(),
+    );
+  }
+
+  // Timestamp receive. Parses and merges a timestamp from a remote
+  // system with the local timeglobal uniqueness and monotonicity are
+  // preserved
+  static recv(msg: Timestamp): Timestamp | null {
+    if (!clock) {
+      return null;
+    }
+
+    // retrieve the local wall time
+    const phys = Date.now();
+
+    // unpack the message wall time/counter
+    const lMsg = msg.millis();
+    const cMsg = msg.counter();
+
+    // assert the node id and remote clock drift
+    // if (msg.node() === clock.timestamp.node()) {
+    //   throw new Timestamp.DuplicateNodeError(clock.timestamp.node());
+    // }
+    if (lMsg - phys > config.maxDrift) {
+      throw new Timestamp.ClockDriftError();
+    }
+
+    // unpack the clock.timestamp logical time and counter
+    const lOld = clock.timestamp.millis();
+    const cOld = clock.timestamp.counter();
+
+    // calculate the next logical time and counter
+    // . ensure that the logical time never goes backward
+    // . if all logical clocks are equal, increment the max counter
+    // . if max = old > message, increment local counter
+    // . if max = messsage > old, increment message counter
+    // . otherwise, clocks are monotonic, reset counter
+    const lNew = Math.max(Math.max(lOld, phys), lMsg);
+    const cNew =
+      lNew === lOld && lNew === lMsg
+        ? Math.max(cOld, cMsg) + 1
+        : lNew === lOld
+          ? cOld + 1
+          : lNew === lMsg
+            ? cMsg + 1
+            : 0;
+
+    // check the result for drift and counter overflow
+    if (lNew - phys > config.maxDrift) {
+      throw new Timestamp.ClockDriftError();
+    }
+    if (cNew > MAX_COUNTER) {
+      throw new Timestamp.OverflowError();
+    }
+
+    // repack the logical time/counter
+    clock.timestamp.setMillis(lNew);
+    clock.timestamp.setCounter(cNew);
+
+    return new Timestamp(
+      clock.timestamp.millis(),
+      clock.timestamp.counter(),
+      clock.timestamp.node(),
+    );
+  }
+
+  /**
+   * zero/minimum timestamp
+   */
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  static zero = Timestamp.parse(
+    '1970-01-01T00:00:00.000Z-0000-0000000000000000',
+  )!;
+
+  static since = (isoString: string) => isoString + '-0000-0000000000000000';
+
+  /**
+   * error classes
+   */
+  static DuplicateNodeError = class DuplicateNodeError extends Error {
+    constructor(node: string) {
+      super('duplicate node identifier ' + node);
+      this.name = 'DuplicateNodeError';
+    }
+  };
+
+  static ClockDriftError = class ClockDriftError extends Error {
+    constructor(...args: unknown[]) {
+      super(
+        ['maximum clock drift exceeded'].concat(args as string[]).join(' '),
+      );
+      this.name = 'ClockDriftError';
+    }
+  };
+
+  static OverflowError = class OverflowError extends Error {
+    constructor() {
+      super('timestamp counter overflow');
+      this.name = 'OverflowError';
+    }
+  };
+
+  static InvalidError = class InvalidError extends Error {
+    constructor(...args: unknown[]) {
+      super(['timestamp is not valid'].concat(args.map(String)).join(' '));
+      this.name = 'InvalidError';
+    }
+  };
+}
+
+class MutableTimestamp extends Timestamp {
+  static from(timestamp: Timestamp) {
+    return new MutableTimestamp(
+      timestamp.millis(),
+      timestamp.counter(),
+      timestamp.node(),
+    );
+  }
+
+  setMillis(n: number) {
+    this._state.millis = n;
+  }
+
+  setCounter(n: number) {
+    this._state.counter = n;
+  }
+
+  setNode(n: string) {
+    this._state.node = n;
+  }
+}

packages/crdt/src/main.ts

@@ -0,0 +1,14 @@
+import * as SyncPb from './proto/sync_pb';
+export {
+  merkle,
+  getClock,
+  setClock,
+  makeClock,
+  makeClientId,
+  serializeClock,
+  deserializeClock,
+  type Clock,
+  Timestamp,
+} from './crdt';
+
+export const SyncProtoBuf = SyncPb;

packages/crdt/src/proto/sync_pb.d.ts

@@ -20,11 +20,11 @@ export class EncryptedData extends jspb.Message {
   setData(value: Uint8Array | string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): EncryptedDataAsObject;
+  toObject(includeInstance?: boolean): EncryptedData.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: EncryptedData,
-  ): EncryptedDataAsObject;
+  ): EncryptedData.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -40,11 +40,13 @@ export class EncryptedData extends jspb.Message {
   ): EncryptedData;
 }
 
-export type EncryptedDataAsObject = {
-  iv: Uint8Array | string;
-  authtag: Uint8Array | string;
-  data: Uint8Array | string;
-};
+export namespace EncryptedData {
+  export type AsObject = {
+    iv: Uint8Array | string;
+    authtag: Uint8Array | string;
+    data: Uint8Array | string;
+  };
+}
 
 export class Message extends jspb.Message {
   getDataset(): string;
@@ -60,8 +62,8 @@ export class Message extends jspb.Message {
   setValue(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): MessageAsObject;
-  static toObject(includeInstance: boolean, msg: Message): MessageAsObject;
+  toObject(includeInstance?: boolean): Message.AsObject;
+  static toObject(includeInstance: boolean, msg: Message): Message.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -77,12 +79,14 @@ export class Message extends jspb.Message {
   ): Message;
 }
 
-export type MessageAsObject = {
-  dataset: string;
-  row: string;
-  column: string;
-  value: string;
-};
+export namespace Message {
+  export type AsObject = {
+    dataset: string;
+    row: string;
+    column: string;
+    value: string;
+  };
+}
 
 export class MessageEnvelope extends jspb.Message {
   getTimestamp(): string;
@@ -97,11 +101,11 @@ export class MessageEnvelope extends jspb.Message {
   setContent(value: Uint8Array | string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): MessageEnvelopeAsObject;
+  toObject(includeInstance?: boolean): MessageEnvelope.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: MessageEnvelope,
-  ): MessageEnvelopeAsObject;
+  ): MessageEnvelope.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -117,11 +121,13 @@ export class MessageEnvelope extends jspb.Message {
   ): MessageEnvelope;
 }
 
-export type MessageEnvelopeAsObject = {
-  timestamp: string;
-  isencrypted: boolean;
-  content: Uint8Array | string;
-};
+export namespace MessageEnvelope {
+  export type AsObject = {
+    timestamp: string;
+    isencrypted: boolean;
+    content: Uint8Array | string;
+  };
+}
 
 export class SyncRequest extends jspb.Message {
   clearMessagesList(): void;
@@ -142,11 +148,11 @@ export class SyncRequest extends jspb.Message {
   setSince(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): SyncRequestAsObject;
+  toObject(includeInstance?: boolean): SyncRequest.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: SyncRequest,
-  ): SyncRequestAsObject;
+  ): SyncRequest.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -162,13 +168,15 @@ export class SyncRequest extends jspb.Message {
   ): SyncRequest;
 }
 
-export type SyncRequestAsObject = {
-  messagesList: Array<MessageEnvelopeAsObject>;
-  fileid: string;
-  groupid: string;
-  keyid: string;
-  since: string;
-};
+export namespace SyncRequest {
+  export type AsObject = {
+    messagesList: Array<MessageEnvelope.AsObject>;
+    fileid: string;
+    groupid: string;
+    keyid: string;
+    since: string;
+  };
+}
 
 export class SyncResponse extends jspb.Message {
   clearMessagesList(): void;
@@ -180,11 +188,11 @@ export class SyncResponse extends jspb.Message {
   setMerkle(value: string): void;
 
   serializeBinary(): Uint8Array;
-  toObject(includeInstance?: boolean): SyncResponseAsObject;
+  toObject(includeInstance?: boolean): SyncResponse.AsObject;
   static toObject(
     includeInstance: boolean,
     msg: SyncResponse,
-  ): SyncResponseAsObject;
+  ): SyncResponse.AsObject;
   static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
   static extensionsBinary: {
     [key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -200,7 +208,9 @@ export class SyncResponse extends jspb.Message {
   ): SyncResponse;
 }
 
-export type SyncResponseAsObject = {
-  messagesList: Array<MessageEnvelopeAsObject>;
-  merkle: string;
-};
+export namespace SyncResponse {
+  export type AsObject = {
+    messagesList: Array<MessageEnvelope.AsObject>;
+    merkle: string;
+  };
+}

packages/crdt/tsconfig.dist.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    // Using ES2021 because that’s the newest version where
+    // the latest Node 16.x release supports all of the features
+    "target": "ES2021",
+    "module": "CommonJS",
+    "noEmit": false,
+    "declaration": true,
+    "strict": true,
+    "outDir": "dist"
+  },
+  "include": ["."],
+  "exclude": ["dist"]
+}

packages/desktop-client/.eslintignore

@@ -1,3 +0,0 @@
-bundle.browser.js
-build/
-public/kcab/

packages/desktop-client/.gitignore

@@ -9,13 +9,17 @@ test-results
 
 # production
 build
+build-stats
+stats.json
 
 # misc
 .DS_Store
 .env
 npm-debug.log
+.swc
 
 *kcab.*
 public/kcab
 public/data
 public/data-file-index.txt
+public/*.wasm

packages/desktop-client/.swcrc

@@ -0,0 +1,16 @@
+{
+  "jsc": {
+    "target": "es2022",
+    "transform": {
+      "react": {
+        "runtime": "automatic"
+      }
+    },
+    "externalHelpers": true,
+    "parser": {
+      "syntax": "typescript",
+      "tsx": true
+    }
+  },
+  "sourceMaps": true
+}

packages/desktop-client/README.md

@@ -4,11 +4,13 @@ Actual on the web
 
 E2E (end-to-end) tests use [Playwright](https://playwright.dev/). Running them requires an Actual server to be running either locally or on a remote server.
 
+### Functional
+
 Running against the local server:
 
 ```sh
 # Start the development server
-yarn start:browser
+yarn start
 
 # Run against the local server (localhost:3001)
 yarn e2e
@@ -19,3 +21,44 @@ Running against a remote server:
 ```sh
 E2E_START_URL=http://my-remote-server.com yarn e2e
 ```
+
+### Visual regression
+
+Visual regression tests (also known as screenshot tests) check that the visual appearance of the product has not regressed. Each environment has slightly different colors, fonts etc. Mac differs from Windows which differs from Linux. In order to have a stable test environment for visual comparisons - you must use a standartised docker container. This ensures that the tests are always ran in a consistent environment.
+
+Prerequisites:
+
+- Docker installed
+
+#### Running against the local server
+
+First start a dev instance:
+
+```sh
+HTTPS=true yarn start
+```
+Note the network IP address and port the dev instance is listening on.
+
+Next, navigate to the root of your project folder, run the standartised docker container, and launch the visual regression tests from within it.
+
+```sh
+# Run docker container
+docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.41.1-jammy /bin/bash
+
+    # If you receive an error such as "docker: invalid reference format", please instead use the following command:
+    docker run --rm --network host -v ${pwd}:/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.41.1-jammy /bin/bash
+
+# Run the VRT tests: important - they MUST be ran against a HTTPS server.  Use the ip and port noted earlier
+E2E_START_URL=https://ip:port yarn vrt
+
+    # To update snapshots, use the following command:
+    E2E_START_URL=https://ip:port yarn vrt --update-snapshots
+```
+
+#### Running against a remote server
+
+You can also run the tests against a remote server by passing the URL:
+
+```sh
+E2E_START_URL=https://my-remote-server.com yarn vrt
+```

packages/desktop-client/bin/build-browser

@@ -3,15 +3,16 @@
 ROOT=`dirname $0`
 cd "$ROOT/.."
 
-VERSION=`cat "$ROOT/../../desktop-client/package.json" | grep version | head -n1 | awk -F "\"" '{print $4}' | tr -d '\r\n'`
-
-echo "Building version $VERSION for the browser..."
+echo "Building the browser..."
 
 rm -fr build
 
 export IS_GENERIC_BROWSER=1
-export INLINE_RUNTIME_CHUNK=false
 export REACT_APP_BACKEND_WORKER_HASH=`ls "$ROOT"/../public/kcab/kcab.worker.*.js |  sed 's/.*kcab\.worker\.\(.*\)\.js/\1/'`
-export REACT_APP_ACTUAL_VERSION="$VERSION"
 
 yarn build
+
+rm -fr build-stats
+mkdir build-stats
+mv build/kcab/stats.json build-stats/loot-core-stats.json
+mv ./stats.json build-stats/web-stats.json

packages/desktop-client/bin/watch-browser

@@ -3,11 +3,8 @@
 ROOT=`dirname $0`
 cd "$ROOT/.."
 
-VERSION=`cat "$ROOT/../../desktop-client/package.json" | grep version | head -n1 | awk -F "\"" '{print $4}' | tr -d '\r\n'`
-
 export IS_GENERIC_BROWSER=1
 export PORT=3001
 export REACT_APP_BACKEND_WORKER_HASH="dev"
-export REACT_APP_ACTUAL_VERSION="$VERSION"
 
 yarn start

packages/desktop-client/config-overrides.js

@@ -1,45 +0,0 @@
-const path = require('path');
-
-const {
-  addWebpackResolve,
-  override,
-  overrideDevServer,
-  babelInclude,
-} = require('customize-cra');
-
-if (process.env.CI) {
-  process.env.DISABLE_ESLINT_PLUGIN = 'true';
-}
-
-module.exports = {
-  webpack: override(
-    babelInclude([path.resolve('src'), path.resolve('../loot-core')]),
-    addWebpackResolve({
-      extensions: [
-        ...(process.env.IS_GENERIC_BROWSER
-          ? ['.browser.js', '.browser.ts', '.browser.tsx']
-          : []),
-        '.web.js',
-        '.web.ts',
-        '.web.tsx',
-        '.js',
-        '.ts',
-        '.tsx',
-      ],
-    }),
-    config => {
-      config.cache = false;
-      return config;
-    },
-  ),
-  devServer: overrideDevServer(config => {
-    return {
-      ...config,
-      headers: {
-        ...config.headers,
-        'Cross-Origin-Opener-Policy': 'same-origin',
-        'Cross-Origin-Embedder-Policy': 'require-corp',
-      },
-    };
-  }),
-};

packages/desktop-client/e2e/accounts.test.js

@@ -24,18 +24,17 @@ test.describe('Accounts', () => {
   test('creates a new account and views the initial balance transaction', async () => {
     const accountPage = await navigation.createAccount({
       name: 'New Account',
-      type: 'Checking / Cash',
       offBudget: false,
       balance: 100,
     });
 
-    expect(await accountPage.getNthTransaction(0)).toMatchObject({
-      payee: 'Starting Balance',
-      notes: '',
-      category: 'Starting Balances',
-      debit: '',
-      credit: '100.00',
-    });
+    const transaction = accountPage.getNthTransaction(0);
+    await expect(transaction.payee).toHaveText('Starting Balance');
+    await expect(transaction.notes).toHaveText('');
+    await expect(transaction.category).toHaveText('Starting Balances');
+    await expect(transaction.debit).toHaveText('');
+    await expect(transaction.credit).toHaveText('100.00');
+    await expect(page).toMatchThemeScreenshots();
   });
 
   test('closes an account', async () => {
@@ -45,8 +44,10 @@ test.describe('Accounts', () => {
 
     const modal = await accountPage.clickCloseAccount();
     await modal.selectTransferAccount('Vanguard 401k');
+    await expect(page).toMatchThemeScreenshots();
     await modal.closeAccount();
 
     await expect(accountPage.accountName).toHaveText('Closed: Roth IRA');
+    await expect(page).toMatchThemeScreenshots();
   });
 });