[AI] Add continuous benchmarking with vitest bench + CodSpeed

Introduces runtime performance benchmarks for hot paths that can silently regress, complementing the existing bundle-size guard: - CRDT merkle trie (build/diff/prune) - AQL query compilation - OFX/QIF/CAMT import parsers Benchmarks live next to their code as `*.bench.ts` files and run via `yarn bench` (root `vitest.bench.config.mts`). A new `benchmark.yml` workflow runs them under CodSpeed in CI to report perf deltas on PRs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016UwZpHTjFGo5jBTp8SZVad
2026-07-15 14:43:36 -05:00 · 2026-06-22 21:24:55 +00:00
parent 98c096a3d0
commit 8d040bee62
10 changed files with 428 additions and 2 deletions
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,40 @@
+name: Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - 'packages/loot-core/**'
+      - 'packages/crdt/**'
+      - 'vitest.bench.config.mts'
+      - 'package.json'
+      - 'yarn.lock'
+      - '.github/workflows/benchmark.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+  benchmark:
+    runs-on: depot-ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up environment
+        uses: ./.github/actions/setup
+        with:
+          download-translations: 'false'
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@1aa3d227c0eaa4157abb72f64750a032cac12a9b # v4.8.2
+        with:
+          run: yarn bench
+          token: ${{ secrets.CODSPEED_TOKEN }}
--- a/package.json
+++ b/package.json
@@ -47,6 +47,7 @@
    "generate:release-notes": "ts-node ./bin/release-note-generator.ts",
    "test": "lage test --continue",
    "test:debug": "lage test --no-cache --continue",
+    "bench": "vitest bench --run --config vitest.bench.config.mts",
    "e2e": "yarn workspace @actual-app/web run e2e",
    "e2e:desktop": "yarn build:desktop --skip-exe-build --skip-translations && yarn workspace desktop-electron e2e",
    "playwright": "yarn workspace @actual-app/web run playwright",
@@ -64,6 +65,7 @@
    "prepare": "husky"
  },
  "devDependencies": {
+    "@codspeed/vitest-plugin": "^5.6.0",
    "@monorepo-utils/workspaces-to-typescript-project-references": "^2.11.0",
    "@octokit/rest": "^22.0.1",
    "@types/node": "^22.19.21",
--- a/packages/crdt/src/crdt/merkle.bench.ts
+++ b/packages/crdt/src/crdt/merkle.bench.ts
@@ -0,0 +1,46 @@
+import { bench, describe } from 'vitest';
+
+import * as merkle from './merkle';
+import { Timestamp } from './timestamp';
+
+const NODE = '0123456789abcdef';
+const MINUTE = 1000 * 60;
+const BASE = Date.UTC(2020, 0, 1);
+
+function makeTimestamps(count: number, startMillis = BASE): Timestamp[] {
+  const timestamps: Timestamp[] = [];
+  for (let i = 0; i < count; i++) {
+    // Spread messages across time so the trie gains realistic depth.
+    timestamps.push(new Timestamp(startMillis + i * MINUTE, i % 100, NODE));
+  }
+  return timestamps;
+}
+
+const small = makeTimestamps(100);
+const large = makeTimestamps(5000);
+const largeTrie = merkle.build(large);
+
+// A second trie that diverges from `largeTrie` near the end, simulating two
+// clients that have synced most of their history but differ on recent edits.
+const diverged = merkle.build([
+  ...large.slice(0, large.length - 10),
+  ...makeTimestamps(10, BASE + large.length * MINUTE * 2),
+]);
+
+describe('merkle trie', () => {
+  bench('build trie from 100 messages', () => {
+    merkle.build(small);
+  });
+
+  bench('build trie from 5000 messages', () => {
+    merkle.build(large);
+  });
+
+  bench('diff two large tries', () => {
+    merkle.diff(largeTrie, diverged);
+  });
+
+  bench('prune large trie', () => {
+    merkle.prune(largeTrie);
+  });
+});
--- a/packages/loot-core/src/server/aql/compiler.bench.ts
+++ b/packages/loot-core/src/server/aql/compiler.bench.ts
@@ -0,0 +1,87 @@
+import { bench, describe } from 'vitest';
+
+import { q } from '#shared/query';
+
+import { generateSQLWithState } from './compiler';
+
+const schema = {
+  transactions: {
+    id: { type: 'id' },
+    date: { type: 'date' },
+    amount: { type: 'integer' },
+    notes: { type: 'string' },
+    cleared: { type: 'boolean' },
+    payee: { type: 'id', ref: 'payees' },
+    category: { type: 'id', ref: 'categories' },
+    account: { type: 'id', ref: 'accounts' },
+    tombstone: { type: 'boolean' },
+  },
+  payees: {
+    id: { type: 'id' },
+    name: { type: 'string' },
+    tombstone: { type: 'boolean' },
+  },
+  categories: {
+    id: { type: 'id' },
+    name: { type: 'string' },
+    is_income: { type: 'boolean' },
+    tombstone: { type: 'boolean' },
+  },
+  accounts: {
+    id: { type: 'id' },
+    name: { type: 'string' },
+    offbudget: { type: 'boolean' },
+    tombstone: { type: 'boolean' },
+  },
+};
+
+const simpleQuery = q('transactions')
+  .filter({ account: 'abc' })
+  .select(['date', 'amount', 'notes'])
+  .serialize();
+
+const complexQuery = q('transactions')
+  .filter({
+    $and: [
+      { 'account.offbudget': false },
+      { date: { $gte: '2023-01-01' } },
+      { date: { $lte: '2023-12-31' } },
+      {
+        $or: [
+          { 'category.is_income': true },
+          { amount: { $lt: 0 } },
+          { 'payee.name': { $like: '%market%' } },
+        ],
+      },
+    ],
+  })
+  .select([
+    'date',
+    'amount',
+    'notes',
+    'payee.name',
+    'category.name',
+    'account.name',
+  ])
+  .orderBy([{ date: 'desc' }, 'amount'])
+  .serialize();
+
+const groupedQuery = q('transactions')
+  .filter({ 'account.offbudget': false })
+  .groupBy(['category.name'])
+  .select(['category.name', { total: { $sum: '$amount' } }])
+  .serialize();
+
+describe('aql compiler', () => {
+  bench('compile simple query', () => {
+    generateSQLWithState(simpleQuery, schema);
+  });
+
+  bench('compile complex filtered/joined query', () => {
+    generateSQLWithState(complexQuery, schema);
+  });
+
+  bench('compile grouped aggregate query', () => {
+    generateSQLWithState(groupedQuery, schema);
+  });
+});
--- a/packages/loot-core/src/server/transactions/import/ofx2json.bench.ts
+++ b/packages/loot-core/src/server/transactions/import/ofx2json.bench.ts
@@ -0,0 +1,62 @@
+import { bench, describe } from 'vitest';
+
+import { ofx2json } from './ofx2json';
+
+function generateOfx(transactionCount: number): string {
+  const header = [
+    'OFXHEADER:100',
+    'DATA:OFXSGML',
+    'VERSION:102',
+    'SECURITY:NONE',
+    'ENCODING:USASCII',
+    'CHARSET:1252',
+    'COMPRESSION:NONE',
+    'OLDFILEUID:NONE',
+    'NEWFILEUID:NONE',
+    '',
+  ].join('\n');
+
+  const transactions: string[] = [];
+  for (let i = 0; i < transactionCount; i++) {
+    const day = String((i % 28) + 1).padStart(2, '0');
+    transactions.push(
+      '<STMTTRN>',
+      '<TRNTYPE>DEBIT',
+      `<DTPOSTED>201801${day}120000`,
+      `<TRNAMT>-${(i % 1000) + 0.99}`,
+      `<FITID>${i}`,
+      `<NAME>Payee Name ${i % 50}`,
+      `<MEMO>Memo for transaction ${i}`,
+      '</STMTTRN>',
+    );
+  }
+
+  return `${header}
+<OFX>
+<BANKMSGSRSV1>
+<STMTTRNRS>
+<STMTRS>
+<CURDEF>USD
+<BANKTRANLIST>
+<DTSTART>20180101120000
+<DTEND>20181231120000
+${transactions.join('\n')}
+</BANKTRANLIST>
+</STMTRS>
+</STMTTRNRS>
+</BANKMSGSRSV1>
+</OFX>`;
+}
+
+const small = generateOfx(100);
+const large = generateOfx(2000);
+
+describe('ofx2json', () => {
+  bench('parse 100 transactions', async () => {
+    await ofx2json(small);
+  });
+
+  bench('parse 2000 transactions', async () => {
+    await ofx2json(large);
+  });
+});
--- a/packages/loot-core/src/server/transactions/import/qif2json.bench.ts
+++ b/packages/loot-core/src/server/transactions/import/qif2json.bench.ts
@@ -0,0 +1,34 @@
+import { bench, describe } from 'vitest';
+
+import { qif2json } from './qif2json';
+
+function generateQif(transactionCount: number): string {
+  const lines = ['!Type:Bank'];
+  for (let i = 0; i < transactionCount; i++) {
+    const day = (i % 28) + 1;
+    const month = (i % 12) + 1;
+    lines.push(
+      `D${String(month).padStart(2, '0')}/${String(day).padStart(2, '0')}/18`,
+      `T-${(i % 1000) + 0.99}`,
+      `N${i}`,
+      `PPayee Name ${i % 50}`,
+      `MMemo for transaction ${i}`,
+      `LCategory:Subcategory${i % 20}`,
+      '^',
+    );
+  }
+  return lines.join('\n');
+}
+
+const small = generateQif(100);
+const large = generateQif(5000);
+
+describe('qif2json', () => {
+  bench('parse 100 transactions', () => {
+    qif2json(small, { dateFormat: 'MM/dd/yy' });
+  });
+
+  bench('parse 5000 transactions', () => {
+    qif2json(large, { dateFormat: 'MM/dd/yy' });
+  });
+});
--- a/packages/loot-core/src/server/transactions/import/xmlcamt2json.bench.ts
+++ b/packages/loot-core/src/server/transactions/import/xmlcamt2json.bench.ts
@@ -0,0 +1,80 @@
+import { bench, describe } from 'vitest';
+
+import { xmlCAMT2json } from './xmlcamt2json';
+
+function generateEntry(i: number): string {
+  const day = String((i % 28) + 1).padStart(2, '0');
+  const isDebit = i % 2 === 0;
+  return `
+            <Ntry>
+                <Amt Ccy="EUR">${(i % 1000) + 0.6}</Amt>
+                <CdtDbtInd>${isDebit ? 'DBIT' : 'CRDT'}</CdtDbtInd>
+                <Sts>BOOK</Sts>
+                <BookgDt>
+                    <Dt>2014-01-${day}</Dt>
+                </BookgDt>
+                <ValDt>
+                    <Dt>2014-01-${day}</Dt>
+                </ValDt>
+                <AcctSvcrRef>2013123001153870${String(i).padStart(3, '0')}</AcctSvcrRef>
+                <NtryDtls>
+                    <TxDtls>
+                        <Refs>
+                            <EndToEndId>STZV-EtE-${i}</EndToEndId>
+                        </Refs>
+                        <RltdPties>
+                            <Dbtr>
+                                <Nm>Debtor ${i % 50}</Nm>
+                            </Dbtr>
+                            <Cdtr>
+                                <Nm>Creditor ${i % 50}</Nm>
+                            </Cdtr>
+                        </RltdPties>
+                        <RmtInf>
+                            <Ustrd>Payment reference for transaction ${i}</Ustrd>
+                        </RmtInf>
+                    </TxDtls>
+                </NtryDtls>
+            </Ntry>`;
+}
+
+function generateCamt(transactionCount: number): string {
+  const entries: string[] = [];
+  for (let i = 0; i < transactionCount; i++) {
+    entries.push(generateEntry(i));
+  }
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<Document
+    xmlns="urn:iso:std:iso:20022:tech:xsd:camt.053.001.04"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <BkToCstmrStmt>
+        <GrpHdr>
+            <MsgId>053D2014</MsgId>
+            <CreDtTm>2014-01-03T22:01:36.0+01:00</CreDtTm>
+        </GrpHdr>
+        <Stmt>
+            <Id>0352C5320140103220142</Id>
+            <Acct>
+                <Id>
+                    <IBAN>DE14740618130000033626</IBAN>
+                </Id>
+                <Ccy>EUR</Ccy>
+            </Acct>
+            ${entries.join('')}
+        </Stmt>
+    </BkToCstmrStmt>
+</Document>`;
+}
+
+const small = generateCamt(100);
+const large = generateCamt(2000);
+
+describe('xmlCAMT2json', () => {
+  bench('parse 100 transactions', async () => {
+    await xmlCAMT2json(small);
+  });
+
+  bench('parse 2000 transactions', async () => {
+    await xmlCAMT2json(large);
+  });
+});
--- a/upcoming-release-notes/benchmark-codspeed.md
+++ b/upcoming-release-notes/benchmark-codspeed.md
@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [MatissJanis]
+---
+
+Add continuous benchmarking with vitest bench and CodSpeed to guard runtime performance of hot paths (CRDT merge, AQL query compilation, and OFX/QIF/CAMT import parsers)
--- a/vitest.bench.config.mts
+++ b/vitest.bench.config.mts
@@ -0,0 +1,23 @@
+import codspeedPlugin from '@codspeed/vitest-plugin';
+import { defineConfig } from 'vitest/config';
+
+// Continuous benchmarking config. Benchmarks live next to the code they measure
+// as `*.bench.ts` files and run with `yarn bench`. In CI the CodSpeed plugin
+// instruments them so performance deltas can be reported on pull requests.
+export default defineConfig({
+  plugins: [codspeedPlugin()],
+  // Mirror loot-core's node test resolution so platform-specific (`#platform`,
+  // `#shared`, …) subpath imports resolve the same way they do under `yarn test`.
+  resolve: {
+    conditions: ['electron', 'module', 'browser', 'development'],
+  },
+  ssr: {
+    resolve: { conditions: ['electron', 'module', 'node', 'development'] },
+  },
+  test: {
+    benchmark: {
+      include: ['packages/*/src/**/*.bench.ts'],
+      exclude: ['**/node_modules/**', '**/dist/**', '**/lib-dist/**'],
+    },
+  },
+});
--- a/yarn.lock
+++ b/yarn.lock
@@ -2019,6 +2019,32 @@ __metadata:
  languageName: node
  linkType: hard

+"@codspeed/core@npm:^5.6.0":
+  version: 5.6.0
+  resolution: "@codspeed/core@npm:5.6.0"
+  dependencies:
+    axios: "npm:^1.4.0"
+    find-up: "npm:^6.3.0"
+    form-data: "npm:^4.0.4"
+    node-gyp-build: "npm:^4.6.0"
+    stack-trace: "npm:1.0.0-pre2"
+  checksum: 10/36ef33a56a0f6dccf9ec19e420ed02ffdbd22d9e86c467a66d72f48b9eb75217ca0c91763cec13c0b27916bc843e9347a2afae800da556638d1ef240d4aee880
+  languageName: node
+  linkType: hard
+
+"@codspeed/vitest-plugin@npm:^5.6.0":
+  version: 5.6.0
+  resolution: "@codspeed/vitest-plugin@npm:5.6.0"
+  dependencies:
+    "@codspeed/core": "npm:^5.6.0"
+  peerDependencies:
+    tinybench: ">=2.9.0"
+    vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0
+    vitest: ^3.2 || ^4
+  checksum: 10/476cbea5f230fae5126f1b0f97fe844e43480781b709a77a10ca6d6e9e117ce6ca277da1b5a17c26b6c2cbcfceb1690f9bdca2e129e4fddec9f4911fe6c9ff95
+  languageName: node
+  linkType: hard
+
 "@colors/colors@npm:1.5.0":
  version: 1.5.0
  resolution: "@colors/colors@npm:1.5.0"
@@ -9409,6 +9435,7 @@ __metadata:
  version: 0.0.0-use.local
  resolution: "actual@workspace:."
  dependencies:
+    "@codspeed/vitest-plugin": "npm:^5.6.0"
    "@monorepo-utils/workspaces-to-typescript-project-references": "npm:^2.11.0"
    "@octokit/rest": "npm:^22.0.1"
    "@types/node": "npm:^22.19.21"
@@ -10010,6 +10037,18 @@ __metadata:
  languageName: node
  linkType: hard

+"axios@npm:^1.4.0":
+  version: 1.18.0
+  resolution: "axios@npm:1.18.0"
+  dependencies:
+    follow-redirects: "npm:^1.16.0"
+    form-data: "npm:^4.0.5"
+    https-proxy-agent: "npm:^5.0.1"
+    proxy-from-env: "npm:^2.1.0"
+  checksum: 10/586a1a9534531c6ec4ee8fbab07a536ecaa8d29bd16b40ab0f9fce361fcd24da1538a54aadde0562f08f30b55bf80f9b7ededf1987e6506f722e1fb338b09d5d
+  languageName: node
+  linkType: hard
+
 "b4a@npm:^1.6.4":
  version: 1.7.3
  resolution: "b4a@npm:1.7.3"
@@ -14950,7 +14989,7 @@ __metadata:
  languageName: node
  linkType: hard

-"form-data@npm:^4.0.0, form-data@npm:^4.0.5":
+"form-data@npm:^4.0.0, form-data@npm:^4.0.4, form-data@npm:^4.0.5":
  version: 4.0.6
  resolution: "form-data@npm:4.0.6"
  dependencies:
@@ -19784,7 +19823,7 @@ __metadata:
  languageName: node
  linkType: hard

-"node-gyp-build@npm:^4.8.4":
+"node-gyp-build@npm:^4.6.0, node-gyp-build@npm:^4.8.4":
  version: 4.8.4
  resolution: "node-gyp-build@npm:4.8.4"
  bin:
@@ -24870,6 +24909,13 @@ __metadata:
  languageName: node
  linkType: hard

+"stack-trace@npm:1.0.0-pre2":
+  version: 1.0.0-pre2
+  resolution: "stack-trace@npm:1.0.0-pre2"
+  checksum: 10/a64099f86acc01980b0a7fbc662f3233bf8626daf95c53e31c835b2252ae11fc3dbfe8f3e77a7f8310132dd488af2795057cd7db599de0c41a6fa99b16068273
+  languageName: node
+  linkType: hard
+
 "stackback@npm:0.0.2":
  version: 0.0.2
  resolution: "stackback@npm:0.0.2"