Experimental OPFS - Phase 3 (#9271)

* phase 3

* fix coordinator

* fix tab handover

* new write failure notification

* phase 4

* embed wasm bytes to the sqlite-wasm engine

* making the storage=opfs qs set the experimental setting gate

* improving error messages

* fix timer on migration

* better message for what engine is in use

* relesae even if flush errors

* fixup some issues

* hardening

* iphone fix for worker/leader and opfs

* mobile fix

* release note consolidation

* feedback

* remove perf test

* cleanup

* feedback

* more feedback

* feedback

* fix suspended tabs by keeping alive in

* update test

* feedback

* feedback

* feedback fixed

* feedback

* fix navigation lock

* feedback
This commit is contained in:
Michael Clark
2026-11-11 21:39:54 +00:00
committed by GitHub
parent a353f6a0bb
commit 27f0dbfc68
64 changed files with 3659 additions and 201 deletions
+47
View File
@@ -107,6 +107,53 @@ jobs:
retention-days: 1
overwrite: true
# The same suite on the sqlite-wasm browser storage (opt-in while it is
# being tried out, the default once it ships). Kept out of the Depot
# report so the two runs are not merged.
functional-new-storage:
name: Functional, new storage (shard ${{ matrix.shard }}/${{ vars.E2E_SHARD_TOTAL || '3' }})
runs-on: depot-ubuntu-latest-4
needs: build-web
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3]
container:
image: mcr.microsoft.com/playwright:v1.61.1-jammy
env:
E2E_USE_BUILD: '1'
E2E_STORAGE_BACKEND: opfs
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Set up environment
uses: ./.github/actions/setup
with:
download-translations: 'false'
- name: Trust the repository directory
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Download web build
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: desktop-client-build
path: packages/desktop-client/build/
- name: Set E2E port
if: ${{ matrix.shard <= fromJson(vars.E2E_SHARD_TOTAL || '3') }}
run: echo "E2E_PORT=$((3000 + ${{ matrix.shard }}))" >> "$GITHUB_ENV"
- name: Run E2E Tests
if: ${{ matrix.shard <= fromJson(vars.E2E_SHARD_TOTAL || '3') }}
env:
SHARD_TOTAL: ${{ vars.E2E_SHARD_TOTAL || '3' }}
run: yarn e2e --shard=${{ matrix.shard }}/$SHARD_TOTAL
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: ${{ failure() && matrix.shard <= fromJson(vars.E2E_SHARD_TOTAL || '3') }}
with:
name: desktop-client-test-results-new-storage-shard-${{ matrix.shard }}
path: packages/desktop-client/test-results/
retention-days: 30
overwrite: true
report-e2e:
name: Report E2E results to Depot
needs: functional
+15 -2
View File
@@ -1,11 +1,16 @@
// Web Worker entry for the browser build; owns the real loot-core backend.
//
// Fully self-contained: the sql.js wasm and the default filesystem data
// Fully self-contained: the SQLite wasm and the default filesystem data
// (migrations + default budget DB) are embedded at build time. The worker
// performs NO `PUBLIC_URL` asset fetches, so a consumer needs to serve no extra
// files and configure no bundler — just `import` the package and `init()`.
//
// Only the sqlite-wasm engine ships here: budgets live in the origin private
// file system, or in IndexedDB where sync access handles are unavailable.
// Nothing uses SharedArrayBuffer, so the page needs no cross-origin isolation.
import * as connection from '@actual-app/core/platform/server/connection';
import { logger } from '@actual-app/core/platform/server/log';
import { setWasmBinary } from '@actual-app/core/platform/server/sqlite';
import { handlers, init } from '@actual-app/core/server/main';
import type { InitConfig } from '@actual-app/core/server/main';
@@ -67,8 +72,16 @@ self.fetch = function patchedFetch(
'api-browser/init'
] = async function (args?: unknown) {
const { config } = (args ?? {}) as { config?: InitConfig };
// absurd-sql is not bundled, so the legacy engine cannot be requested
let storageBackend = config?.storageBackend ?? 'opfs';
if (storageBackend === 'legacy') {
logger.warn(
'The browser build of @actual-app/api no longer includes the legacy storage engine; using the sqlite-wasm engine instead',
);
storageBackend = 'opfs';
}
// There is no meaningful `process.cwd()` to fall back to in a worker
await init({ dataDir: '/documents', ...config });
await init({ dataDir: '/documents', ...config, storageBackend });
};
connection.init(self, handlers);
+518 -4
View File
@@ -1,11 +1,13 @@
import { expect, test } from '@playwright/test';
import type { Page } from '@playwright/test';
// Structural view of the module harness.html loads from dist/browser.js
// (`typeof import` would pull the whole loot-core type graph in here).
type Api = {
init(config: { dataDir: string }): Promise<unknown>;
init(config: { dataDir: string; storageBackend?: string }): Promise<unknown>;
shutdown(): Promise<void>;
getBudgets(): Promise<unknown[]>;
getBudgets(): Promise<Array<{ id: string }>>;
loadBudget(id: string): Promise<unknown>;
getAccounts(): Promise<Array<{ id: string; name: string }>>;
runImport(budgetName: string, func: () => Promise<void>): Promise<void>;
createAccount(account: { name: string }, balance?: number): Promise<string>;
@@ -24,14 +26,55 @@ declare global {
// oxlint-disable-next-line typescript/consistent-type-definitions -- global Window augmentation requires interface
interface Window {
apiReady: Promise<Api>;
unansweredBackendRequests: Set<unknown>;
}
}
// Where the open page keeps its budgets: absurd-sql's per-file databases
// must be gone, and every sqlite row must carry the sqlite-wasm marker.
async function readStorageLayout(page: Page) {
return page.evaluate(async () => {
const names = (await indexedDB.databases()).map(
database => database.name ?? '',
);
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open('actual');
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
type FileRow = { filepath: string; storage?: string };
const rows = await new Promise<FileRow[]>((resolve, reject) => {
const request = db
.transaction(['files'], 'readonly')
.objectStore('files')
.getAll();
request.onsuccess = () => resolve(request.result as FileRow[]);
request.onerror = () => reject(request.error);
});
db.close();
return {
legacyDatabases: names.filter(name => name.startsWith('documents-')),
sqliteStorage: rows
.filter(row => row.filepath.endsWith('.sqlite'))
.map(row => row.storage),
};
});
}
test('boots, imports a budget, reads it back, and persists', async ({
page,
}) => {
await page.goto('/e2e/harness.html');
// The harness is served without cross-origin isolation, so the engine
// has to manage without SharedArrayBuffer
expect(
await page.evaluate(() => ({
isolated: window.crossOriginIsolated,
sharedArrayBuffer: typeof SharedArrayBuffer,
})),
).toEqual({ isolated: false, sharedArrayBuffer: 'undefined' });
await page.evaluate(async () => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents' });
@@ -77,7 +120,14 @@ test('boots, imports a budget, reads it back, and persists', async ({
expect(result.accountNames).toEqual(['Checking']);
expect(result.amounts).toEqual([-1250, 50000]);
// Reload the page: the budget must survive in IndexedDB.
const layout = await readStorageLayout(page);
expect(layout.legacyDatabases).toEqual([]);
expect(layout.sqliteStorage.length).toBeGreaterThan(0);
for (const kind of layout.sqliteStorage) {
expect(['opfs', 'memory-idb']).toContain(kind);
}
// Reload the page: the budget must survive in the browser's storage.
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
@@ -112,7 +162,7 @@ test('supports a custom dataDir and persists it', async ({ page }) => {
});
expect(result.accountNames).toEqual(['Savings']);
// Reload the page: the budget must survive in IndexedDB.
// Reload the page: the budget must survive in the browser's storage.
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
@@ -127,3 +177,467 @@ test('supports a custom dataDir and persists it', async ({ page }) => {
});
expect(budgetCount).toBe(1);
});
// Left alone, the fallback saves this long after the last change
const FALLBACK_DEBOUNCE_MS = 1000;
// The fallback saves a budget whole to IndexedDB; its row revision moves on
// every save
async function readFallbackRevision(page: Page) {
return page.evaluate(async () => {
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open('actual');
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
type FileRow = { storage?: string; rev?: number };
const rows = await new Promise<FileRow[]>((resolve, reject) => {
const request = db
.transaction(['files'], 'readonly')
.objectStore('files')
.getAll();
request.onsuccess = () => resolve(request.result as FileRow[]);
request.onerror = () => reject(request.error);
});
db.close();
return rows
.filter(row => row.storage === 'memory-idb')
.reduce((sum, row) => sum + (row.rev ?? 0), 0);
});
}
test('saves fallback changes when the page is hidden, so a reload without shutdown keeps them', async ({
page,
}) => {
await page.goto('/e2e/harness.html');
await page.evaluate(async () => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: 'memory-idb' });
await api.runImport('e2e-fallback-budget', async () => {
await api.createAccount({ name: 'Checking' }, 0);
});
});
const saved = await readFallbackRevision(page);
// A change the fallback would save only after its debounce; hiding the
// page asks for the save at once
const changeStartedAt = await page.evaluate(async () => {
const api = await window.apiReady;
const [account] = await api.getAccounts();
const startedAt = Date.now();
await api.addTransactions(account.id, [
{ date: '2024-02-01', amount: -4200, notes: 'hidden' },
]);
Object.defineProperty(document, 'visibilityState', {
value: 'hidden',
configurable: true,
});
document.dispatchEvent(new Event('visibilitychange'));
return startedAt;
});
// Patient, for a loaded runner; what counts is how long the save took
await expect
.poll(() => readFallbackRevision(page), {
timeout: 5000,
intervals: [25, 50, 100],
})
.toBeGreaterThan(saved);
// Counted from before the change, so a save the debounce made can never
// pass: that one lands a full interval after the change at the earliest
expect(Date.now() - changeStartedAt).toBeLessThan(FALLBACK_DEBOUNCE_MS);
// No shutdown(): the saved budget still has the change
await page.reload();
const amounts = await page.evaluate(async () => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: 'memory-idb' });
const [budget] = await api.getBudgets();
await api.loadBudget(budget.id);
const [account] = await api.getAccounts();
const transactions = await api.getTransactions(
account.id,
'2024-02-01',
'2024-02-01',
);
await api.shutdown();
return transactions.map(transaction => transaction.amount);
});
expect(amounts).toEqual([-4200]);
});
// Makes the backend's IndexedDB writes fail the way a full disk does: the
// request is accepted, then its transaction aborts
async function setBackendWritesFailing(page: Page, isFailing: boolean) {
const [backend] = page.workers();
await backend.evaluate(failing => {
const scope = globalThis as typeof globalThis & {
workingPut?: IDBObjectStore['put'];
};
scope.workingPut ??= IDBObjectStore.prototype.put;
const workingPut = scope.workingPut;
IDBObjectStore.prototype.put = failing
? function failingPut(
this: IDBObjectStore,
...args: Parameters<IDBObjectStore['put']>
) {
const request = workingPut.apply(this, args);
this.transaction.abort();
return request;
}
: workingPut;
}, isFailing);
}
test('keeps the worker and the budget when the last changes cannot be saved, and shuts down once they can', async ({
page,
}) => {
await page.goto('/e2e/harness.html');
await page.evaluate(async () => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: 'memory-idb' });
await api.runImport('e2e-unsaved-budget', async () => {
await api.createAccount({ name: 'Checking' }, 0);
});
});
await setBackendWritesFailing(page, true);
const failure = await page.evaluate(async () => {
const api = await window.apiReady;
const [account] = await api.getAccounts();
await api.addTransactions(account.id, [
{ date: '2024-04-01', amount: -900, notes: 'not saved yet' },
]);
return api.shutdown().then(
() => null,
(error: Error) => error.message,
);
});
expect(failure).toContain('Could not save the last changes');
// Nothing was shut down: the worker is there, with the budget open and
// the change in it
async function readAmounts() {
return page.evaluate(async () => {
const api = await window.apiReady;
const [account] = await api.getAccounts();
const transactions = await api.getTransactions(
account.id,
'2024-04-01',
'2024-04-01',
);
return transactions.map(transaction => transaction.amount);
});
}
expect(page.workers()).toHaveLength(1);
expect(await readAmounts()).toEqual([-900]);
// Nor is it replaced: a second init is refused while it runs
const refusal = await page.evaluate(async () => {
const api = await window.apiReady;
return api
.init({ dataDir: '/documents', storageBackend: 'memory-idb' })
.then(
() => null,
(error: Error) => error.message,
);
});
expect(refusal).toContain('already running');
expect(page.workers()).toHaveLength(1);
expect(await readAmounts()).toEqual([-900]);
// With room again the same call saves the change and ends the worker
await setBackendWritesFailing(page, false);
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
await expect.poll(() => page.workers().length).toBe(0);
await page.reload();
await page.evaluate(async () => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: 'memory-idb' });
const [budget] = await api.getBudgets();
await api.loadBudget(budget.id);
});
expect(await readAmounts()).toEqual([-900]);
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
});
// Keeps count of the requests the backend has yet to answer. Installed
// before the page loads, as it has to see the worker being created
async function trackBackendRequests(page: Page) {
await page.addInitScript(() => {
const unanswered = new Set<unknown>();
window.unansweredBackendRequests = unanswered;
const NativeWorker = window.Worker;
window.Worker = class TrackedWorker extends NativeWorker {
constructor(...args: ConstructorParameters<typeof Worker>) {
super(...args);
const post = this.postMessage.bind(this);
this.postMessage = (message: { id?: unknown }) => {
if (message.id != null) {
unanswered.add(message.id);
}
post(message);
};
this.addEventListener('message', event => {
unanswered.delete(event.data?.id);
});
}
};
});
}
// The API moves on to its next request as each answer arrives, so once
// nothing is unanswered it has finished whatever a departure or a return
// set off
async function backendToSettle(page: Page) {
await expect
.poll(() => page.evaluate(() => window.unansweredBackendRequests.size))
.toBe(0);
}
// Makes the backend fail to take a budget's lock, at once rather than after
// the wait a budget that is open in another tab ends with
async function setBudgetLocksRefused(page: Page, isRefused: boolean) {
const [backend] = page.workers();
await backend.evaluate(refused => {
const scope = globalThis as typeof globalThis & {
workingRequest?: LockManager['request'];
};
scope.workingRequest ??= LockManager.prototype.request;
LockManager.prototype.request = refused
? function refusedRequest() {
return Promise.reject(new DOMException('held', 'AbortError'));
}
: scope.workingRequest;
}, isRefused);
}
// The budget locks held anywhere in this origin, the API's worker included
function countHeldBudgetLocks(page: Page) {
return page.evaluate(async () => {
const { held = [] } = await navigator.locks.query();
return held.filter(lock => lock.name?.startsWith('actual-budget:')).length;
});
}
for (const storageBackend of ['opfs', 'memory-idb']) {
test(`gives the budget up when the page leaves and opens it again when it comes back (${storageBackend})`, async ({
page,
}) => {
await page.goto('/e2e/harness.html');
await page.evaluate(async backend => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: backend });
await api.runImport('e2e-lifecycle-budget', async () => {
const accountId = await api.createAccount({ name: 'Checking' }, 0);
await api.addTransactions(accountId, [
{ date: '2024-03-01', amount: -700, notes: 'before leaving' },
]);
});
}, storageBackend);
expect(await countHeldBudgetLocks(page)).toBe(1);
// What a browser fires when the page goes into its back/forward cache.
// WebKit keeps the worker of such a page, locks and all, so the lock
// has to go for the next page to open the same budget
await page.evaluate(() => {
window.dispatchEvent(
new PageTransitionEvent('pagehide', { persisted: true }),
);
});
await expect.poll(() => countHeldBudgetLocks(page)).toBe(0);
// And when it comes back out of it: the budget is open again, with
// what was saved before the page left. What was asked while the page
// was away, or is asked the moment it is back, waits for that instead
// of meeting the closed database
const answers = await page.evaluate(async () => {
const api = await window.apiReady;
const askedWhileAway = api.getAccounts();
window.dispatchEvent(
new PageTransitionEvent('pageshow', { persisted: true }),
);
const askedOnReturn = api.getAccounts();
const [whileAway, onReturn] = await Promise.all([
askedWhileAway,
askedOnReturn,
]);
const transactions = await api.getTransactions(
onReturn[0].id,
'2024-03-01',
'2024-03-01',
);
return {
whileAway: whileAway.map(account => account.name),
onReturn: onReturn.map(account => account.name),
amounts: transactions.map(transaction => transaction.amount),
};
});
expect(answers).toEqual({
whileAway: ['Checking'],
onReturn: ['Checking'],
amounts: [-700],
});
expect(await countHeldBudgetLocks(page)).toBe(1);
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
});
test(`a page that leaves again while it reopens its budget holds nothing, and reopens it on its next return (${storageBackend})`, async ({
page,
}) => {
await trackBackendRequests(page);
await page.goto('/e2e/harness.html');
await page.evaluate(async backend => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: backend });
await api.runImport('e2e-back-and-forth-budget', async () => {
await api.createAccount({ name: 'Checking' }, 0);
});
}, storageBackend);
// Back, forward and back again before the budget is open again, the
// second departure landing at a different step of the reopening each time
for (const delayMs of [0, 10, 30, 60, 120, 250]) {
await page.evaluate(async delay => {
const dispatch = (type: string) =>
window.dispatchEvent(
new PageTransitionEvent(type, { persisted: true }),
);
dispatch('pagehide');
dispatch('pageshow');
await new Promise(resolve => setTimeout(resolve, delay));
dispatch('pagehide');
}, delayMs);
// The reopening that began in between has run its course by now, and
// must not have taken the budget back
await backendToSettle(page);
await expect
.poll(() => countHeldBudgetLocks(page), {
message: `after ${delayMs} ms`,
})
.toBe(0);
const accountNames = await page.evaluate(async () => {
const api = await window.apiReady;
window.dispatchEvent(
new PageTransitionEvent('pageshow', { persisted: true }),
);
const accounts = await api.getAccounts();
return accounts.map(account => account.name);
});
expect(accountNames, `after ${delayMs} ms`).toEqual(['Checking']);
expect(await countHeldBudgetLocks(page), `after ${delayMs} ms`).toBe(1);
}
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
});
test(`a budget that could not be reopened is tried again on the next return (${storageBackend})`, async ({
page,
}) => {
await page.goto('/e2e/harness.html');
await page.evaluate(async backend => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: backend });
await api.runImport('e2e-taken-budget', async () => {
await api.createAccount({ name: 'Checking' }, 0);
});
}, storageBackend);
async function leaveAndReturn() {
return page.evaluate(async () => {
const api = await window.apiReady;
for (const type of ['pagehide', 'pageshow']) {
window.dispatchEvent(
new PageTransitionEvent(type, { persisted: true }),
);
}
return api.getAccounts().then(
accounts => accounts.map(account => account.name),
(error: { message?: string }) => error.message,
);
});
}
// Another tab took the budget while the page was away: what was asked
// meanwhile is answered for the budget being closed
await setBudgetLocksRefused(page, true);
expect(await leaveAndReturn()).toBe('No budget file is open');
expect(await countHeldBudgetLocks(page)).toBe(0);
// The other tab is done by the next return, and the budget opens again
await setBudgetLocksRefused(page, false);
expect(await leaveAndReturn()).toEqual(['Checking']);
expect(await countHeldBudgetLocks(page)).toBe(1);
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
});
test(`holds nothing for a page that leaves while its budget is loading (${storageBackend})`, async ({
page,
}) => {
await page.goto('/e2e/harness.html');
const budgetId = await page.evaluate(async backend => {
const api = await window.apiReady;
await api.init({ dataDir: '/documents', storageBackend: backend });
await api.runImport('e2e-leaving-budget', async () => {
await api.createAccount({ name: 'Checking' }, 0);
});
const [budget] = await api.getBudgets();
// Start again with the budget closed
await api.shutdown();
await api.init({ dataDir: '/documents', storageBackend: backend });
return budget.id;
}, storageBackend);
// The load is cut short, rather than left to take the budget for a
// page that has gone
const interruption = await page.evaluate(async id => {
const api = await window.apiReady;
const loading = api.loadBudget(id).then(
() => null,
(error: { code?: string }) => error.code,
);
window.dispatchEvent(
new PageTransitionEvent('pagehide', { persisted: true }),
);
return loading;
}, budgetId);
expect(interruption).toBe('storage-unavailable');
await expect.poll(() => countHeldBudgetLocks(page)).toBe(0);
// Back out of the cache, the budget loads as usual
const accountNames = await page.evaluate(async id => {
const api = await window.apiReady;
window.dispatchEvent(
new PageTransitionEvent('pageshow', { persisted: true }),
);
await api.loadBudget(id);
const accounts = await api.getAccounts();
return accounts.map(account => account.name);
}, budgetId);
expect(accountNames).toEqual(['Checking']);
expect(await countHeldBudgetLocks(page)).toBe(1);
await page.evaluate(async () => {
const api = await window.apiReady;
await api.shutdown();
});
});
}
+2 -2
View File
@@ -1,7 +1,7 @@
import { expect, test } from '@playwright/test';
// Loads the consumer fixture's production build (built in global-setup.ts,
// served by serve-dist.mjs with COOP/COEP). On master this fails: the consumer
// Loads the consumer fixture's production build (built in global-setup.mjs,
// served by serve-dist.mjs without cross-origin isolation headers). On master this fails: the consumer
// bundler re-bundles the prebuilt worker and `init` throws a structured-clone
// error. With the self-contained build it passes with no consumer config.
test('a consumer production build boots the worker and seeds a budget', async ({
+2 -2
View File
@@ -6,8 +6,8 @@ import { build } from 'vite';
const here = path.dirname(fileURLToPath(import.meta.url));
// Build the consumer fixture (e2e/consumer) for production with Vite, exactly
// as a third-party app would. serve-dist.mjs then serves the output (with the
// COOP/COEP headers) at /e2e/consumer/dist/. This is the coverage the
// as a third-party app would. serve-dist.mjs then serves the output (without
// cross-origin isolation headers) at /e2e/consumer/dist/. This is the coverage the
// verbatim-dist harness can't give: a consumer bundler processing browser.js.
// Playwright's globalSetup contract requires a default export.
// oxlint-disable-next-line import/no-default-export
+3 -5
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env node
// Static server for the package root with the COOP/COEP headers required for
// SharedArrayBuffer (absurd-sql). Used by playwright.config.ts.
// Static server for the package root, used by playwright.config.ts. It sends
// no cross-origin isolation headers on purpose: the browser build must work
// without SharedArrayBuffer, and the tests check that it does.
import fs from 'node:fs';
import http from 'node:http';
@@ -21,9 +22,6 @@ const MIME = {
http
.createServer((req, res) => {
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
const urlPath = decodeURIComponent(req.url.split('?')[0]);
const file = path.resolve(ROOT, urlPath.replace(/^\/+/, ''));
if (
+165 -10
View File
@@ -9,10 +9,19 @@ export * as utils from './utils';
let worker: Worker | null = null;
let ownsSession = false;
let unlistenPageLifecycle: (() => void) | null = null;
export async function init(
config: InitConfig = {},
): Promise<{ send: typeof send }> {
// One backend at a time: a second worker could not open the budget the
// first one holds, and the first would be out of reach, along with any
// changes it has not saved
if (worker) {
throw new Error(
'The API is already running. Call shutdown() before calling init() again.',
);
}
worker = new InlineWorker();
try {
@@ -24,9 +33,152 @@ export async function init(
throw error;
}
unlistenPageLifecycle = listenForPageLifecycle(worker);
return { send };
}
// Where sync access handles are missing, the engine keeps the budget in
// memory and saves it to IndexedDB a moment after each change. A page that
// is hidden asks for that save at once, as the web client does, so a reload
// right after a change does not lose it even without `shutdown()`.
//
// A page that is leaving gives the budget up altogether, saving first:
// WebKit keeps the locks and file handles of a worker whose page went into
// the back/forward cache, where they would keep the next page from opening
// the same budget until the cached page is evicted. A page that comes back
// out of that cache opens its budget again, and what was asked of the API in
// between is held back until then: the worker would run it against the
// database it closed.
//
// Best effort: a page that unloads at once may not give the worker the time
// to finish.
function listenForPageLifecycle(backend: Worker): (() => void) | null {
if (typeof window === 'undefined' || typeof document === 'undefined') {
return null;
}
// Requests kept from the worker while the budget is given up. Each is
// copied as posting it would have: what cannot be sent fails for its
// caller, and later changes to it do not count
let heldRequests: unknown[] | null = null;
let isSendingAhead = false;
const post = backend.postMessage.bind(backend);
backend.postMessage = (message: unknown) => {
if (heldRequests && !isSendingAhead) {
heldRequests.push(structuredClone(message));
} else {
post(message);
}
};
// For the requests that give the budget up and open it again, which go
// ahead of the held ones; `send` posts before it returns
function sendAhead<Result>(request: () => Promise<Result>): Promise<Result> {
isSendingAhead = true;
try {
return request();
} finally {
isSendingAhead = false;
}
}
// What the page that left asked of the worker, until the page that came
// back has waited for it
let leaving: Promise<unknown> | null = null;
let reopening: Promise<void> = Promise.resolve();
// Counts departures, so a reopening can tell that the page left again
let departures = 0;
// The budget to open again. Kept until it is open: a reopening can fail,
// and one the page left in the middle of may already have closed it
let budgetToReopen: string | null = null;
// `left` and `departure` are what the page had at its return
async function reopenBudget(left: Promise<unknown>, departure: number) {
// Once the page has left again, nothing more is asked of the worker:
// the next return starts over
const hasLeftAgain = () => departures !== departure;
try {
// Frozen along with its page, the worker may not have finished
await left;
if (!hasLeftAgain()) {
await sendAhead(() => send('storage-resume'));
// The worker still counts the budget it gave up as open
const loaded = await sendAhead(() => send('load-prefs'));
// A budget that is open now counts over one kept from before
budgetToReopen = loaded?.id ?? budgetToReopen;
}
const budgetId = budgetToReopen;
if (budgetId && !hasLeftAgain()) {
await sendAhead(() => send('close-budget'));
if (!hasLeftAgain()) {
const { error } = await sendAhead(() =>
send('load-budget', { id: budgetId }),
);
if (!error) {
budgetToReopen = null;
}
}
}
} catch {
// The held requests meet the same trouble, and report it
}
// The held requests wait for the next return too
if (hasLeftAgain()) {
return;
}
// Sent whether or not the budget is open again: those that need it are
// told that none is open, and the others run as usual
const requests = heldRequests ?? [];
heldRequests = null;
requests.forEach(request => post(request));
}
const onVisibilityChange = () => {
if (document.visibilityState === 'hidden') {
void send('storage-flush').catch(() => undefined);
}
};
const onPageHide = () => {
departures += 1;
heldRequests ??= [];
leaving = sendAhead(() => send('storage-release')).catch(() => undefined);
};
const onPageShow = (event: PageTransitionEvent) => {
const left = leaving;
if (event.persisted && left) {
leaving = null;
const departure = departures;
reopening = reopening.then(() => reopenBudget(left, departure));
}
};
window.addEventListener('pagehide', onPageHide);
window.addEventListener('pageshow', onPageShow);
document.addEventListener('visibilitychange', onVisibilityChange);
return () => {
window.removeEventListener('pagehide', onPageHide);
window.removeEventListener('pageshow', onPageShow);
document.removeEventListener('visibilitychange', onVisibilityChange);
};
}
// Where the engine saves to IndexedDB, the worker holds the only copy of
// whatever a save has not stored yet, and a save can fail, on a full disk
// for one. The worker is not ended over such changes: the caller hears
// about it and can try again.
async function ensureStored() {
try {
await send('storage-flush');
} catch (cause) {
throw new Error(
"Could not save the last changes to the browser's storage, so the " +
'backend was left running with them. The storage is most likely ' +
'full: make room and call shutdown() again, loading the budget ' +
'again first if it is no longer open.',
{ cause },
);
}
}
export async function shutdown() {
if (worker) {
try {
@@ -34,18 +186,21 @@ export async function shutdown() {
} catch {
// most likely that no budget is loaded, so the sync failed
}
// Before closing, so that a budget whose changes cannot be stored
// stays open, and after, because closing saves once more
await ensureStored();
await send('close-budget');
await ensureStored();
try {
await send('close-budget');
} finally {
try {
if (ownsSession) {
await send('subscribe-sign-out');
}
} finally {
ownsSession = false;
worker.terminate();
worker = null;
if (ownsSession) {
await send('subscribe-sign-out');
}
} finally {
ownsSession = false;
worker.terminate();
worker = null;
unlistenPageLifecycle?.();
unlistenPageLifecycle = null;
}
}
}
+1 -1
View File
@@ -9,7 +9,7 @@ declare module '*?worker&inline' {
// Runtime assets emitted by the `actual-embedded-assets` plugin
// (vite.browser.config.mts), sourced from loot-core's `default-filesystem`.
declare module 'virtual:actual-embedded-assets' {
/** base64 of sql.js' sql-wasm.wasm */
/** base64 of the official SQLite build's sqlite3.wasm */
export const wasmBase64: string;
/** contents of data-file-index.txt */
export const dataIndex: string;
+1 -1
View File
@@ -8,7 +8,7 @@ import { nodePolyfills } from 'vite-plugin-node-polyfills';
const distDir = path.resolve(__dirname, 'dist');
// Embed loot-core's default filesystem (wasm + default DB + migrations) into
// Embed loot-core's default filesystem (SQLite wasm + default DB + migrations) into
// the worker as a virtual module, so the browser build performs no PUBLIC_URL
// asset fetches and consumers serve no extra files. loot-core owns the bytes
// and the wire format; this just inlines them.
+47 -2
View File
@@ -1,5 +1,5 @@
import { test as base, expect as baseExpect } from '@playwright/test';
import type { Browser, Locator, Page } from '@playwright/test';
import type { Browser, BrowserContext, Locator, Page } from '@playwright/test';
/**
* Disable CSS transitions and animations globally in e2e (non-VRT) runs.
@@ -33,6 +33,51 @@ export const disableAnimationsInitScript = () => {
}
};
/**
* Storage engine the backend worker runs on. Set E2E_STORAGE_BACKEND to
* `opfs` (or `memory-idb`) to run the suite on the sqlite-wasm storage;
* unset means the app's default. The key mirrors
* loot-core/src/platform/client/browser-preload/storage-backend.ts.
*/
export const E2E_STORAGE_BACKEND = process.env.E2E_STORAGE_BACKEND;
const STORAGE_BACKEND_KEY = 'actual:storage-backend';
/** A test that drives the flag itself sets this in sessionStorage first. */
export const MANUAL_STORAGE_BACKEND_FLAG = 'e2e-manual-storage-backend';
const storageBackendInitScript = ({
key,
backend,
manualFlag,
}: {
key: string;
backend: string;
manualFlag: string;
}) => {
try {
if (!sessionStorage.getItem(manualFlag)) {
localStorage.setItem(key, backend);
}
} catch {
// Without storage the app uses its default engine
}
};
/**
* Installs the e2e init scripts on a page or context. `browser.newPage`
* gets them automatically; tests that create their own context (several
* tabs sharing storage) call this themselves.
*/
export async function addE2EInitScripts(target: Page | BrowserContext) {
await target.addInitScript(disableAnimationsInitScript);
if (E2E_STORAGE_BACKEND) {
await target.addInitScript(storageBackendInitScript, {
key: STORAGE_BACKEND_KEY,
backend: E2E_STORAGE_BACKEND,
manualFlag: MANUAL_STORAGE_BACKEND_FLAG,
});
}
}
export const test = process.env.VRT
? base
: base.extend<object, { browser: Browser }>({
@@ -41,7 +86,7 @@ export const test = process.env.VRT
const originalNewPage = browser.newPage.bind(browser);
browser.newPage = async options => {
const page = await originalNewPage(options);
await page.addInitScript(disableAnimationsInitScript);
await addE2EInitScripts(page);
return page;
};
await runWithBrowser(browser);
+4 -4
View File
@@ -1,6 +1,6 @@
import type { BrowserContext, Page } from '@playwright/test';
import { disableAnimationsInitScript, expect, test } from './fixtures';
import { addE2EInitScripts, expect, test } from './fixtures';
import { BudgetPage } from './page-models/budget-page';
import { ConfigurationPage } from './page-models/configuration-page';
import { Navigation } from './page-models/navigation';
@@ -62,7 +62,7 @@ test.describe('Multiple tabs on one budget', () => {
test.beforeEach(async ({ browser }) => {
context = await browser.newContext({ userAgent: CHROME_USER_AGENT });
await context.addInitScript(disableAnimationsInitScript);
await addE2EInitScripts(context);
leader = await context.newPage();
await leader.goto('/');
await new ConfigurationPage(leader).createTestFile();
@@ -122,7 +122,7 @@ test.describe('Refreshing a tab', () => {
browser,
}) => {
const context = await browser.newContext({ userAgent: CHROME_USER_AGENT });
await context.addInitScript(disableAnimationsInitScript);
await addE2EInitScripts(context);
const creator = await context.newPage();
await creator.goto('/');
// The second tab opens once the first has booted, as a user would
@@ -157,7 +157,7 @@ test.describe('Refreshing a tab', () => {
browser,
}) => {
const context = await browser.newContext({ userAgent: CHROME_USER_AGENT });
await context.addInitScript(disableAnimationsInitScript);
await addE2EInitScripts(context);
// The first tab creates the budget and goes back to the file list
const closer = await context.newPage();
await closer.goto('/');
@@ -0,0 +1,64 @@
import type { Page } from '@playwright/test';
/** Where the open page's budgets are stored, read straight from the browser. */
export type StorageSnapshot = {
/** IndexedDB database names; absurd-sql keeps one per budget file. */
databases: string[];
/** `files` rows for sqlite paths, with the sqlite-wasm storage marker. */
sqliteRows: Array<{ filepath: string; storage?: string; hasBytes: boolean }>;
/** Files under the origin's private file system (OPFS pools). */
poolFiles: string[];
};
export async function inspectStorage(page: Page): Promise<StorageSnapshot> {
return page.evaluate(async () => {
const databases = (await indexedDB.databases()).map(
database => database.name ?? '',
);
const db = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open('actual');
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
type FileRow = { filepath: string; storage?: string; contents: unknown };
const rows = await new Promise<FileRow[]>((resolve, reject) => {
const request = db
.transaction(['files'], 'readonly')
.objectStore('files')
.getAll();
request.onsuccess = () => resolve(request.result as FileRow[]);
request.onerror = () => reject(request.error);
});
db.close();
const sqliteRows = rows
.filter(row => row.filepath.endsWith('.sqlite'))
.map(row => ({
filepath: row.filepath,
storage: row.storage,
hasBytes:
ArrayBuffer.isView(row.contents) && row.contents.byteLength > 0,
}));
const poolFiles: string[] = [];
type DirectoryHandle = {
entries(): AsyncIterable<[string, { kind: string }]>;
};
async function walk(directory: DirectoryHandle, prefix: string) {
for await (const [name, handle] of directory.entries()) {
const path = `${prefix}/${name}`;
if (handle.kind === 'directory') {
await walk(handle as unknown as DirectoryHandle, path);
} else {
poolFiles.push(path);
}
}
}
try {
const root = await navigator.storage.getDirectory();
await walk(root as unknown as DirectoryHandle, '');
} catch {
// No private file system in this browser
}
return { databases, sqliteRows, poolFiles };
});
}
+277
View File
@@ -0,0 +1,277 @@
import type { Page } from '@playwright/test';
import {
addE2EInitScripts,
E2E_STORAGE_BACKEND,
expect,
MANUAL_STORAGE_BACKEND_FLAG,
test,
} from './fixtures';
import { AccountPage } from './page-models/account-page';
import { ConfigurationPage } from './page-models/configuration-page';
import { Navigation } from './page-models/navigation';
import { inspectStorage } from './page-models/storage-inspector';
import type { StorageSnapshot } from './page-models/storage-inspector';
// Mirrors loot-core/src/platform/client/browser-preload/storage-backend.ts
const STORAGE_BACKEND_KEY = 'actual:storage-backend';
const requestedBackend = E2E_STORAGE_BACKEND ?? 'legacy';
const usesNewStorage = requestedBackend !== 'legacy';
// The IndexedDB fallback persists on a one second debounce
const PERSIST_WAIT_MS = 2000;
function expectStorageLayout(
snapshot: StorageSnapshot,
layout: 'legacy' | 'new',
) {
expect(snapshot.sqliteRows.length).toBeGreaterThan(0);
const legacyDatabases = snapshot.databases.filter(name =>
name.startsWith('documents-'),
);
if (layout === 'legacy') {
expect(legacyDatabases.length).toBeGreaterThan(0);
for (const row of snapshot.sqliteRows) {
expect(row.storage).toBeUndefined();
}
return;
}
expect(legacyDatabases).toEqual([]);
for (const row of snapshot.sqliteRows) {
expect(['opfs', 'memory-idb']).toContain(row.storage);
if (row.storage === 'opfs') {
expect(
snapshot.poolFiles.some(file => file.startsWith('/actual/pools/')),
).toBe(true);
} else {
expect(row.hasBytes).toBe(true);
}
}
}
async function openAllySavings(page: Page) {
const accountPage = await new Navigation(page).goToAccountPage(
'Ally Savings',
);
await accountPage.waitFor();
return accountPage;
}
// The header shows 0.00 until the account's query has answered
async function readAllySavingsBalance(page: Page) {
const accountPage = await openAllySavings(page);
await expect(accountPage.accountBalance).not.toHaveText(/^(0\.00)?$/);
return (await accountPage.accountBalance.textContent())?.trim();
}
async function expectAllySavingsBalance(page: Page, balance?: string) {
const accountPage = await openAllySavings(page);
await expect(accountPage.accountBalance).toHaveText(balance ?? '');
}
test.describe('Browser storage', () => {
let page: Page;
test.afterEach(async () => {
await page?.close();
});
test('keeps the budget across a reload', async ({ browser }) => {
page = await browser.newPage();
await page.goto('/');
await new ConfigurationPage(page).createTestFile();
const balance = await readAllySavingsBalance(page);
expect(balance).toBeTruthy();
await page.waitForTimeout(PERSIST_WAIT_MS);
expectStorageLayout(
await inspectStorage(page),
usesNewStorage ? 'new' : 'legacy',
);
await page.reload();
const accountPage = new AccountPage(page);
await accountPage.waitFor({ timeout: 60_000 });
await expect(accountPage.accountBalance).toHaveText(balance ?? '');
});
test('moves a budget created under the legacy storage', async ({
browser,
}) => {
test.skip(!usesNewStorage, 'only meaningful for the new storage');
page = await browser.newPage();
// The URL flag wins over the stored choice for this first load
await page.goto(`/?storage=legacy`);
await page.evaluate(
flag => sessionStorage.setItem(flag, '1'),
MANUAL_STORAGE_BACKEND_FLAG,
);
await new ConfigurationPage(page).createTestFile();
const balance = await readAllySavingsBalance(page);
expectStorageLayout(await inspectStorage(page), 'legacy');
// Switch the flag and reload: the budget is moved when it is reopened
await page.evaluate(
([key, backend]) => localStorage.setItem(key, backend),
[STORAGE_BACKEND_KEY, requestedBackend],
);
await page.goto('/');
await expect(page.getByTestId('budget-table')).toBeVisible({
timeout: 60_000,
});
await expectAllySavingsBalance(page, balance);
await page.waitForTimeout(PERSIST_WAIT_MS);
expectStorageLayout(await inspectStorage(page), 'new');
});
test('switches back to the legacy storage from the settings', async ({
browser,
}) => {
test.skip(!usesNewStorage, 'only meaningful for the new storage');
page = await browser.newPage();
await page.goto('/');
await new ConfigurationPage(page).createTestFile();
const balance = await readAllySavingsBalance(page);
await page.evaluate(flag => {
sessionStorage.setItem(flag, '1');
localStorage.setItem('devEnableStoragePrefs', 'true');
}, MANUAL_STORAGE_BACKEND_FLAG);
const settingsPage = await new Navigation(page).goToSettingsPage();
await settingsPage.goToSection('Experimental');
const toggle = page.getByRole('checkbox', {
name: 'New browser storage engine',
});
await expect(toggle).toBeChecked();
// Moves every budget back, then reloads into the legacy engine, which
// reopens the budget on the settings page
const reloaded = page.waitForEvent('load', { timeout: 60_000 });
await toggle.click();
await reloaded;
await settingsPage.waitFor({ timeout: 60_000 });
await expectAllySavingsBalance(page, balance);
expectStorageLayout(await inspectStorage(page), 'legacy');
expect(
await page.evaluate(
key => localStorage.getItem(key),
STORAGE_BACKEND_KEY,
),
).toBe('legacy');
});
test('a second tab cannot open the same budget at the same time', async ({
browser,
}) => {
test.skip(!usesNewStorage, 'the legacy storage has no budget lock');
const context = await browser.newContext();
await addE2EInitScripts(context);
page = await context.newPage();
await page.goto('/');
await new ConfigurationPage(page).createTestFile();
// Playwright runs direct workers, so the second tab is a second backend
const secondTab = await context.newPage();
const dialogs: string[] = [];
secondTab.on('dialog', dialog => {
dialogs.push(dialog.message());
void dialog.dismiss();
});
await secondTab.goto('/');
await expect
.poll(() => dialogs.join('\n'), { timeout: 45_000 })
.toMatch(/already open in another tab/);
// Once the first tab is gone its budget can be opened here. The failed
// open forgot the last budget, so it is picked from the file list.
await page.close();
await secondTab.reload();
await secondTab.getByText('Test Budget').click({ timeout: 60_000 });
await expect(secondTab.getByTestId('budget-table')).toBeVisible({
timeout: 60_000,
});
await secondTab.close();
await context.close();
});
test('a change of engine in another tab does not reload a tab over changes it could not save', async ({
browser,
}) => {
test.skip(!usesNewStorage, 'only meaningful for the new storage');
page = await browser.newPage();
// The IndexedDB fallback, which keeps a budget in memory between saves
await page.goto('/?storage=memory-idb');
await page.evaluate(
flag => sessionStorage.setItem(flag, '1'),
MANUAL_STORAGE_BACKEND_FLAG,
);
await new ConfigurationPage(page).createTestFile();
const accountPage = await openAllySavings(page);
await setBackendWritesFailing(page, true);
await accountPage.createSingleTransaction({
payee: '',
notes: 'only in memory',
debit: '10.00',
});
const unsavedTransaction = accountPage.transactionTableRow.filter({
hasText: 'only in memory',
});
await expect(unsavedTransaction).toBeVisible();
// The save and its one retry have failed, and the app has said so
await expect(page.getByText('Unable to save changes')).toBeVisible();
// The tab stays as it is, with the change still in it
await page.evaluate(() => {
document.body.dataset.sameDocument = 'true';
});
await announceEngineChange(page);
// Long enough for a relaunch that was not held back to have begun
await page.waitForTimeout(1500);
await expect(page.locator('body')).toHaveAttribute(
'data-same-document',
'true',
);
await expect(unsavedTransaction).toBeVisible();
// Once the change can be saved, the tab saves it and relaunches
await setBackendWritesFailing(page, false);
const relaunched = page.waitForEvent('load', { timeout: 60_000 });
await announceEngineChange(page);
await relaunched;
await expect(unsavedTransaction).toBeVisible({ timeout: 60_000 });
});
});
// Makes the backend's IndexedDB writes fail the way a full disk does: the
// request is accepted, then its transaction aborts
async function setBackendWritesFailing(page: Page, isFailing: boolean) {
const [backend] = page.workers();
await backend.evaluate(failing => {
const scope = globalThis as typeof globalThis & {
workingPut?: IDBObjectStore['put'];
};
scope.workingPut ??= IDBObjectStore.prototype.put;
const workingPut = scope.workingPut;
IDBObjectStore.prototype.put = failing
? function failingPut(
this: IDBObjectStore,
...args: Parameters<IDBObjectStore['put']>
) {
const request = workingPut.apply(this, args);
this.transaction.abort();
return request;
}
: workingPut;
}, isFailing);
}
// What a tab hears when another one stores a different engine
async function announceEngineChange(page: Page) {
await page.evaluate(key => {
window.dispatchEvent(new StorageEvent('storage', { key }));
}, STORAGE_BACKEND_KEY);
}
+10 -1
View File
@@ -1,4 +1,7 @@
import { startBrowserBackend } from '@actual-app/core/platform/client/browser-preload';
import {
resolveStorageBackend,
startBrowserBackend,
} from '@actual-app/core/platform/client/browser-preload';
import * as Platform from '@actual-app/core/shared/platform';
import { registerSW } from 'virtual:pwa-register';
@@ -36,6 +39,11 @@ const isOpenIdCallback = window.location.pathname
.replace(/\/+$/, '')
.endsWith('/openid-cb');
// Which engine the backend uses for budget files: `?storage=opfs` once, or
// the choice stored by the experimental setting. Resolved on the main
// thread because the worker needs it in its init message.
const storageBackend = resolveStorageBackend();
// *** Start the backend ***
//
// The multi-tab coordinator (leader/follower over SharedWorker), the direct
@@ -50,6 +58,7 @@ const worker = startBrowserBackend({
isDev: IS_DEV,
publicUrl: import.meta.env.BASE_URL.slice(0, -1),
hash: import.meta.env.REACT_APP_BACKEND_WORKER_HASH,
storageBackend,
},
createSharedWorker: () =>
new SharedBrowserServerWorker({ name: 'actual-backend' }),
+18 -20
View File
@@ -64,18 +64,7 @@ self.addEventListener('message', async event => {
const isDev = !!msg.isDev;
// let version = msg.version;
const hash = msg.hash;
if (
!self.SharedArrayBuffer &&
!msg.isSharedArrayBufferOverrideEnabled
) {
appInitFailureInterval = postMessageWithRetry({
type: 'app-init-failure',
SharedArrayBufferMissing: true,
});
return;
}
const storageBackend = msg.storageBackend ?? 'legacy';
// A single failed importScripts bricks the SharedWorker until
// it's evicted, so retry in production too.
@@ -84,15 +73,24 @@ self.addEventListener('message', async event => {
{ maxRetries: isDev ? 5 : 3 },
);
backend.initApp(isDev, self).catch(err => {
console.log(err);
appInitFailureInterval = postMessageWithRetry({
type: 'app-init-failure',
IDBFailure: err.message.includes('indexeddb-failure'),
});
backend
.initApp(isDev, self, {
storageBackend,
isSharedArrayBufferOverrideEnabled:
!!msg.isSharedArrayBufferOverrideEnabled,
})
.catch(err => {
console.log(err);
appInitFailureInterval = postMessageWithRetry({
type: 'app-init-failure',
IDBFailure: err.message.includes('indexeddb-failure'),
SharedArrayBufferMissing: err.message.includes(
'shared-array-buffer-missing',
),
});
throw err;
});
throw err;
});
}
}
@@ -5,6 +5,8 @@ import { Text } from '@actual-app/components/text';
import { theme } from '@actual-app/components/theme';
import { spacing } from '@actual-app/components/tokens';
import { View } from '@actual-app/components/view';
import { STORAGE_PREFS_KEY } from '@actual-app/core/platform/client/browser-preload';
import { isElectron } from '@actual-app/core/shared/environment';
import type { FeatureFlag, ServerPrefs } from '@actual-app/core/types/prefs';
import { useAuth } from '#auth/AuthProvider';
@@ -19,6 +21,7 @@ import { useSyncServerStatus } from '#hooks/useSyncServerStatus';
import { SettingsGroup } from './SettingsGroup';
import { SettingsRow } from './SettingsRow';
import { StorageBackendToggle } from './StorageBackendToggle';
type FeatureToggleProps = {
flag: FeatureFlag;
@@ -188,6 +191,9 @@ export function ExperimentalFeatures() {
const showServerPrefs =
localStorage.getItem('devEnableServerPrefs') === 'true';
// The browser storage engine is hidden until it is ready for testers
const showStoragePrefs =
!isElectron() && localStorage.getItem(STORAGE_PREFS_KEY) === 'true';
return (
<SettingsGroup
@@ -290,6 +296,7 @@ export function ExperimentalFeatures() {
<Trans>Client-Side plugins (soon)</Trans>
</ServerFeatureToggle>
)}
{showStoragePrefs && <StorageBackendToggle />}
</SettingsGroup>
);
}
@@ -0,0 +1,119 @@
import { useEffect, useState } from 'react';
import { Trans, useTranslation } from 'react-i18next';
import { Text } from '@actual-app/components/text';
import { theme } from '@actual-app/components/theme';
import {
getStoredStorageBackend,
setStoredStorageBackend,
} from '@actual-app/core/platform/client/browser-preload';
import type { StorageBackend } from '@actual-app/core/platform/client/browser-preload';
import { send } from '@actual-app/core/platform/client/connection';
import { Checkbox } from '#components/forms';
import { addNotification } from '#notifications/notificationsSlice';
import { useDispatch } from '#redux';
import { SettingsRow } from './SettingsRow';
/**
* Opt-in to the sqlite-wasm storage for budget files (hidden behind a dev
* flag while it is being tried out). Switching on only stores the choice:
* budgets move over when they are next opened. Switching off moves every
* budget back first, so nothing is lost either way. Both relaunch the app.
*/
export function StorageBackendToggle() {
const { t } = useTranslation();
const dispatch = useDispatch();
const [activeBackend, setActiveBackend] = useState<StorageBackend | null>(
null,
);
const [isSwitching, setIsSwitching] = useState(false);
// The engine actually running decides, so the box agrees with the
// description and one click puts the stored choice right when they differ
const isEnabled = (activeBackend ?? getStoredStorageBackend()) !== 'legacy';
useEffect(() => {
let isCancelled = false;
void send('storage-info')
.then(info => {
if (!isCancelled) {
setActiveBackend(info.backend);
}
})
.catch(() => {
// No answer from the backend: the stored choice decides
});
return () => {
isCancelled = true;
};
}, []);
async function toggle() {
setIsSwitching(true);
if (isEnabled) {
const result = await send('storage-revert').catch(() => ({
error: 'revert-failed' as const,
}));
if ('error' in result) {
dispatch(
addNotification({
notification: {
type: 'error',
message:
result.error === 'budget-in-use'
? t('Close Actual in your other tabs first, then try again.')
: t(
'Switching back failed. Your budgets stay on the new storage engine.',
),
},
}),
);
setIsSwitching(false);
return;
}
setStoredStorageBackend('legacy');
} else {
setStoredStorageBackend('opfs');
}
window.Actual.relaunch();
}
const backendDescription =
activeBackend === 'opfs'
? t('private file system (OPFS)')
: activeBackend === 'memory-idb'
? t('in memory, saved to IndexedDB')
: activeBackend === 'legacy'
? t('absurd-sql, IndexedDB blocks (current engine)')
: null;
return (
<SettingsRow
htmlFor="settings-storage-backend"
title={<Trans>New browser storage engine</Trans>}
description={
<Text style={{ color: theme.warningText }}>
<Trans>
Keeps budget files in the browser's private file system using
SQLite's own WebAssembly build. Budgets move over when they are next
opened. Close Actual in all other tabs before changing this; the app
reloads.
</Trans>{' '}
{backendDescription && (
<Trans>Currently active: {{ backendDescription }}.</Trans>
)}
</Text>
}
control={
<Checkbox
id="settings-storage-backend"
checked={isEnabled}
disabled={isSwitching}
onChange={() => void toggle()}
style={{ marginRight: 0 }}
/>
}
/>
);
}
+146 -1
View File
@@ -1,4 +1,8 @@
import { listen } from '@actual-app/core/platform/client/connection';
import {
releaseBackendStorage,
STORAGE_BACKEND_KEY,
} from '@actual-app/core/platform/client/browser-preload';
import { listen, send } from '@actual-app/core/platform/client/connection';
import * as undo from '@actual-app/core/platform/client/undo';
// @ts-strict-ignore
import type { QueryClient } from '@tanstack/react-query';
@@ -166,6 +170,45 @@ export function handleGlobalEvents(store: AppStore, queryClient: QueryClient) {
);
});
// The IndexedDB fallback could not store the budget, retry included; the
// id keeps one notification up however many writes fail. From here on the
// latest changes may be in the backend's memory only
let hasUnsavedChanges = false;
const unlistenStorageWriteFailed = listen('storage-write-failed', () => {
hasUnsavedChanges = true;
store.dispatch(
addNotification({
notification: {
id: 'storage-write-failed',
type: 'error',
title: t('Unable to save changes'),
sticky: true,
message: t(
'Your latest changes could not be stored in this browser, which may be out of space. They are kept in memory for now: free up space, or export your budget before closing this tab.',
),
},
}),
);
});
// The browser dropped the private file system under the open app, which
// some do in private windows: nothing more can be saved from this tab
const unlistenStorageLost = listen('storage-lost', () => {
store.dispatch(
addNotification({
notification: {
id: 'storage-lost',
type: 'error',
title: t('Local data was discarded'),
sticky: true,
message: t(
'This browser has discarded the data Actual keeps on this device, which can happen in private windows. Changes can no longer be saved in this tab. Reload the page; a budget on a sync server can then be downloaded again.',
),
},
}),
);
});
const unlistenIndexeddbQuotaError = listen('indexeddb-quota-error', () => {
alert(
t(
@@ -204,6 +247,83 @@ export function handleGlobalEvents(store: AppStore, queryClient: QueryClient) {
void window.Actual.reload();
});
// Progress while a budget is moved into the new browser storage; the
// budget loader shows it on the loading screen. A move the worker starts
// by itself, such as reopening a budget after a failed revert, has no
// loading screen, and must not leave a stray one behind
const unlistenStorageMigration = listen(
'storage-migration',
({ phase, percent }) => {
if (store.getState().app.loadingText == null) {
return;
}
let loadingText: string;
if (phase === 'copy') {
loadingText = t(
'Moving your budget to the new storage... {{percent}}%',
{ percent: Math.round(percent ?? 0) },
);
} else if (phase === 'verify') {
loadingText = t('Checking the moved budget...');
} else if (phase === 'cleanup') {
loadingText = t('Cleaning up the old storage...');
} else {
loadingText = t('Loading...');
}
store.dispatch(setAppState({ loadingText }));
},
);
// The IndexedDB fallback of the new storage persists lazily; ask for a
// flush whenever the tab is hidden
const flushStorage = () => {
send('storage-flush').catch(() => {
// No backend to flush, or it is already gone
});
};
const unlistenVisibilityChange = listenOnDocument('visibilitychange', () => {
if (document.visibilityState === 'hidden') {
flushStorage();
}
});
// A page that is leaving gives its budget up, flushing first: the browser
// frees a destroyed worker's locks and handles itself, but WebKit keeps
// those of a worker whose page went into the back/forward cache, where
// they would block the next page's open until the cached page is evicted.
// Restored from that cache, the page relaunches, as its worker no longer
// holds the budget
const unlistenPageHide = listenOnWindow('pagehide', () => {
releaseBackendStorage();
});
const unlistenPageShow = listenOnWindow('pageshow', event => {
if (event.persisted) {
window.Actual.relaunch();
}
});
// The storage engine is one choice for the whole origin. When another tab
// changes it, this tab relaunches into the same engine: a tab left on the
// old one would keep the budgets there, or move them straight back. Not
// over changes it was told are only in memory, though: those are saved
// first, and the tab stays as it is for as long as they cannot be
const unlistenStorageBackendChange = listenOnWindow('storage', event => {
if (event.key !== STORAGE_BACKEND_KEY) {
return;
}
if (!hasUnsavedChanges) {
window.Actual.relaunch();
return;
}
void send('storage-flush', undefined, { catchErrors: true }).then(
({ error }) => {
if (!error) {
window.Actual.relaunch();
}
},
);
});
return () => {
unlistenServerError();
unlistenOrphanedPayees();
@@ -211,6 +331,8 @@ export function handleGlobalEvents(store: AppStore, queryClient: QueryClient) {
unlistenSync();
unlistenUndo();
unlistenFallbackWriteError();
unlistenStorageWriteFailed();
unlistenStorageLost();
unlistenIndexeddbQuotaError();
unlistenStartLoad();
unlistenFinishLoad();
@@ -218,5 +340,28 @@ export function handleGlobalEvents(store: AppStore, queryClient: QueryClient) {
unlistenFinishImport();
unlistenShowBudgets();
unlistenApiFetchRedirected();
unlistenStorageMigration();
unlistenVisibilityChange();
unlistenPageHide();
unlistenPageShow();
unlistenStorageBackendChange();
};
}
// Browser events in the shape of the backend's `listen`: registering hands
// back the remover
function listenOnWindow<Type extends keyof WindowEventMap>(
type: Type,
handler: (event: WindowEventMap[Type]) => void,
) {
window.addEventListener(type, handler);
return () => window.removeEventListener(type, handler);
}
function listenOnDocument<Type extends keyof DocumentEventMap>(
type: Type,
handler: (event: DocumentEventMap[Type]) => void,
) {
document.addEventListener(type, handler);
return () => document.removeEventListener(type, handler);
}
@@ -210,6 +210,11 @@ export function getSyncError(error: string, id: string) {
'Budget "{{id}}" is in storage this browser cannot reach right now. Nothing was deleted; reload the page and try again.',
{ id },
);
} else if (error === 'storage-lost') {
return t(
'This browser no longer has the local copy of budget "{{id}}". Browsers can discard it, mostly in private windows. If the budget is on a sync server, delete this copy and download it again; otherwise restore it from a backup.',
{ id },
);
} else {
return t('We had an unknown problem opening "{{id}}".', { id });
}
+10 -4
View File
@@ -122,14 +122,16 @@ console.log(await api.getAccounts());
await api.shutdown();
```
Behind the scenes, `init` starts a Web Worker running the same budget engine the Actual web app uses, backed by SQLite compiled to WebAssembly. Your budget data is stored in the browser's IndexedDB and stays on the device.
Behind the scenes, `init` starts a Web Worker running the same budget engine the Actual web app uses, backed by SQLite compiled to WebAssembly. Your budget data is stored in the browser's origin private file system, or in IndexedDB in browsers without it, and stays on the device. Budgets stored by earlier versions of the package are moved over the first time they are opened.
In the browser, `dataDir` is a path inside the worker's virtual file system rather than a folder on disk. It is optional: it defaults to `/documents`, and if you pass a custom path it is created automatically and persisted to IndexedDB just the same.
In the browser, `dataDir` is a path inside the worker's virtual file system rather than a folder on disk. It is optional: it defaults to `/documents`, and if you pass a custom path it is created automatically and persisted just the same.
The browser build is fully self-contained: the Web Worker and its WebAssembly and data files are inlined into the package. There are no extra files to copy or serve, and no bundler configuration is required (no `optimizeDeps` tweaks, no worker or asset plugins). Importing the package and calling `init()` is all that's needed.
:::caution Cross-origin isolation is required
The engine uses `SharedArrayBuffer`, so the page that runs the API must be served **cross-origin isolated**: over HTTPS, with `Cross-Origin-Opener-Policy: same-origin` and `Cross-Origin-Embedder-Policy: require-corp`. This is a hosting/server requirement (it cannot be bundled away). See [Enabling SharedArrayBuffer Access](../troubleshooting/shared-array-buffer.md). In local development, set the same headers on your dev server (for example via a small Vite middleware plugin).
:::note No special headers
The browser build does not use `SharedArrayBuffer`, so the page running it does not need to be cross-origin isolated. Serve it over HTTPS (or from `localhost`) like any other page that stores data.
A budget can be open in one tab at a time. Opening the same budget from a second tab fails until the first one closes it.
:::
## Handling Errors
@@ -233,12 +235,16 @@ If no `serverURL` is provided, no network connections will be made, and you'll o
You can find your budget id in the "Advanced" section of the settings page.
In the [browser build](#using-the-api-in-a-browser), `init` fails while the API is already running. Call `shutdown` first.
#### `shutdown`
<Method name="shutdown" args={[]} returns="Promise<void>" />
Close the current budget file, and stop any other ongoing processes. It's recommended to call this before exiting your script.
In the [browser build](#using-the-api-in-a-browser), `shutdown` fails when the last changes could not be saved to the browser's storage, for example because the storage is full. Nothing is shut down when that happens, so the changes are not lost. Make room in the storage and call `shutdown` again.
#### `utils.amountToInteger`
<Method name="utils.amountToInteger" args={[{ name: 'amount', type: 'number' }]} returns="number" />
@@ -0,0 +1,35 @@
---
title: Browser Storage Engine (Experimental)
unlisted: true
---
# Browser Storage Engine (Experimental)
Actual is trying out a new way of storing budget files in the browser. Today the web app keeps each budget in IndexedDB through a compatibility layer (absurd-sql) that needs `SharedArrayBuffer`, and every change costs several round trips. The new engine uses SQLite’s own WebAssembly build and keeps budget files in the browser’s private file system ([OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system)), which is faster, uses less memory and does not need the `SharedArrayBuffer` headers.
This page is for people who want to try it before it becomes the default. Everything on it is temporary.
## What it needs
- Chrome or Edge 108 or later, Firefox 111 or later, Safari 16.4 or later, served over HTTPS (or from `localhost`).
- Browsers without private file system access (older versions, private browsing in Safari) get a fallback that keeps the database in memory and saves it to IndexedDB about once a second. It works, but it is slower to open and only one tab can use a budget at a time.
## Turning it on
Open Actual once with `?storage=opfs` added to the address, for example `https://your-server.example/?storage=opfs`. The choice is remembered by the browser; you do not need the parameter again.
Each budget is moved to the new storage the first time it is opened. The loading screen shows the progress. The old copy is deleted only after the moved budget has been checked.
A setting with the same effect appears under **Settings → Experimental** once you run `localStorage.setItem('devEnableStoragePrefs', 'true')` in the browser console and reload.
## Turning it off
Use the setting described above. It moves every budget back to the old layout before the app reloads, so nothing is lost. Opening Actual with `?storage=legacy` only changes the preference: a budget already on the new storage keeps using it.
Close Actual in all other tabs before switching either way.
## Things to know
- A budget can be open in one browser tab at a time. Other tabs on the same budget are served by that tab (this is how Actual already works); a tab that cannot reach it shows an "already open in another tab" message.
- The data stays on your device, exactly as before. Syncing with a server is unchanged.
- If something goes wrong, open an issue with the browser and version you used, what you were doing, and anything the browser console shows.
+4 -2
View File
@@ -4,13 +4,15 @@
export const migrationsDir: string;
/** Absolute path to loot-core's default budget database. */
export const defaultDbPath: string;
/** Absolute path to the sql.js wasm binary loot-core depends on. */
/** Absolute path to the sql.js wasm binary of the legacy engine. */
export const sqlWasmPath: string;
/** Absolute path to the official SQLite wasm binary the sqlite-wasm engine runs on. */
export const sqliteWasmPath: string;
/** The `data-file-index.txt` manifest listing every file under `data/`. */
export function buildDataFileIndex(names?: string[]): string;
/** The base64-encoded assets a self-contained browser worker embeds. */
/** The base64-encoded assets a self-contained browser worker embeds (the wasm is the official SQLite build). */
export function collectEmbeddedAssets(): {
wasmBase64: string;
dataFiles: Record<string, string>;
+13 -5
View File
@@ -1,8 +1,9 @@
// Build-time source of truth for the "default filesystem" that loot-core's
// populateDefaultFilesystem() restores at startup: the sql.js wasm, the default
// budget DB, and the migration files, plus the `data-file-index.txt` wire
// format. loot-core owns these files and the protocol, so consumers should read
// them from here rather than reaching into loot-core's tree themselves.
// populateDefaultFilesystem() restores at startup: the default budget DB and
// the migration files, plus the `data-file-index.txt` wire format, and the
// paths of the SQLite wasm binaries. loot-core owns these files and the
// protocol, so consumers should read them from here rather than reaching into
// loot-core's tree themselves.
//
// Consumers:
// - @actual-app/api embeds them into its self-contained browser worker, and
@@ -19,8 +20,13 @@ const require = createRequire(import.meta.url);
export const migrationsDir = path.join(root, 'migrations');
export const defaultDbPath = path.join(root, 'default-db.sqlite');
// The legacy engine's wasm (sql.js over absurd-sql), still served by
// @actual-app/web until that engine is removed
export const sqlWasmPath =
require.resolve('@jlongster/sql.js/dist/sql-wasm.wasm');
// The official SQLite build the sqlite-wasm engine runs on
export const sqliteWasmPath =
require.resolve('@sqlite.org/sqlite-wasm/sqlite3.wasm');
function migrationFileNames() {
return fs
@@ -40,6 +46,8 @@ export function buildDataFileIndex(names = migrationFileNames()) {
/**
* Everything the browser worker inlines, base64-encoded for embedding as a
* string. `dataFiles` is keyed by the `data/<key>` name fetched at runtime.
* The wasm is the official SQLite build: the browser worker runs the
* sqlite-wasm engine only.
*/
export function collectEmbeddedAssets() {
const names = migrationFileNames();
@@ -52,7 +60,7 @@ export function collectEmbeddedAssets() {
.toString('base64');
}
return {
wasmBase64: fs.readFileSync(sqlWasmPath).toString('base64'),
wasmBase64: fs.readFileSync(sqliteWasmPath).toString('base64'),
dataFiles,
index: buildDataFileIndex(names),
};
@@ -1,15 +1,13 @@
import { initBackend } from 'absurd-sql/dist/indexeddb-main-thread';
import * as connection from '#platform/client/connection';
import type { InitConfig } from '#server/main';
// initBackend installs absurd-sql's main-thread bridge — without it the
// worker's first sqlite write hangs.
// Starts the backend inside an already created Worker. The worker runs the
// sqlite-wasm engine, which needs nothing from the main thread (absurd-sql's
// bridge used to be installed here).
export async function startBackendWorker(
worker: Worker,
config: InitConfig,
): Promise<void> {
initBackend(worker);
await connection.init(worker);
// Worker-local handler, not part of the shared Handlers union.
await (connection.send as (name: string, args?: unknown) => Promise<unknown>)(
@@ -1,7 +1,19 @@
export { WorkerBridge, type WorkerLike } from './worker-bridge';
export {
releaseBackendStorage,
startBrowserBackend,
type StartBackendOptions,
type StartBackendInit,
type StartBackendHandle,
} from './start';
export {
DEFAULT_STORAGE_BACKEND,
STORAGE_BACKEND_KEY,
STORAGE_BACKEND_URL_PARAM,
STORAGE_PREFS_KEY,
getStoredStorageBackend,
isStorageBackend,
resolveStorageBackend,
setStoredStorageBackend,
} from './storage-backend';
export type { StorageBackend } from '#platform/server/sqlite';
@@ -1,15 +1,21 @@
import { initBackend as initSQLBackend } from 'absurd-sql/dist/indexeddb-main-thread';
import { logger } from '#platform/server/log';
import type { StorageBackend } from '#platform/server/sqlite';
import * as Platform from '#shared/platform';
import { WorkerBridge } from './worker-bridge';
import { requestStorageRelease, WorkerBridge } from './worker-bridge';
export type StartBackendInit = {
version: string;
isDev: boolean;
publicUrl?: string;
hash?: string;
/**
* Engine for budget files (see `resolveStorageBackend`). Part of the init
* message because the coordinator replays it to every new leader.
*/
storageBackend?: StorageBackend;
};
export type StartBackendOptions = {
@@ -32,6 +38,9 @@ export type StartBackendOptions = {
export type StartBackendHandle = Worker | WorkerBridge;
const PAGE_RELEASE_REQUEST_ID = '__storage-release-page';
let activeHandle: StartBackendHandle | null = null;
export function startBrowserBackend(
opts: StartBackendOptions,
): StartBackendHandle {
@@ -88,6 +97,7 @@ export function startBrowserBackend(
sharedPort.postMessage({ type: 'tab-closing' });
});
activeHandle = bridge;
return bridge;
} catch (e) {
logger.log('SharedWorker failed, falling back to Worker:', e);
@@ -113,5 +123,25 @@ export function startBrowserBackend(
),
});
activeHandle = worker;
return worker;
}
/**
* Asks the backend worker this page runs, if any, to let go of its storage,
* for a page that is leaving. The browser frees a destroyed worker's locks
* and access handles itself, but WebKit keeps those of a worker whose page
* went into the back/forward cache, where they would block the next page's
* open until the cached page is evicted. The request goes to this page's
* own worker, never through the coordinator, which would hand it to the
* leader; a follower runs no worker and has nothing to release.
*/
export function releaseBackendStorage() {
const worker =
activeHandle instanceof WorkerBridge
? activeHandle.localBackendWorker
: activeHandle;
if (worker) {
requestStorageRelease(worker, PAGE_RELEASE_REQUEST_ID);
}
}
@@ -0,0 +1,75 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
getStoredStorageBackend,
resolveStorageBackend,
setStoredStorageBackend,
STORAGE_BACKEND_KEY,
STORAGE_PREFS_KEY,
} from './storage-backend';
afterEach(() => {
localStorage.clear();
window.history.replaceState(null, '', '/');
vi.restoreAllMocks();
});
describe('storage backend flag', () => {
it('defaults to the legacy engine', () => {
expect(getStoredStorageBackend()).toBe('legacy');
expect(resolveStorageBackend()).toBe('legacy');
});
it('stores and reads the chosen backend', () => {
setStoredStorageBackend('opfs');
expect(localStorage.getItem(STORAGE_BACKEND_KEY)).toBe('opfs');
expect(getStoredStorageBackend()).toBe('opfs');
});
it('ignores values that are not a backend', () => {
localStorage.setItem(STORAGE_BACKEND_KEY, 'floppy');
expect(getStoredStorageBackend()).toBe('legacy');
});
it('takes a one-shot URL parameter, stores it and strips it', () => {
window.history.replaceState(null, '', '/budget?storage=opfs&other=1');
expect(resolveStorageBackend()).toBe('opfs');
expect(localStorage.getItem(STORAGE_BACKEND_KEY)).toBe('opfs');
expect(window.location.search).toBe('?other=1');
expect(window.location.pathname).toBe('/budget');
// Later loads keep the stored choice
expect(resolveStorageBackend()).toBe('opfs');
});
it('uses the URL choice for this load even when it cannot be stored', () => {
vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => {
throw new DOMException('full', 'QuotaExceededError');
});
window.history.replaceState(null, '', '/?storage=opfs');
expect(resolveStorageBackend()).toBe('opfs');
expect(window.location.search).toBe('');
// Nothing was stored, so the next load is back on the default
expect(resolveStorageBackend()).toBe('legacy');
});
it('reveals the engine setting when the URL flag is used', () => {
expect(localStorage.getItem(STORAGE_PREFS_KEY)).toBeNull();
window.history.replaceState(null, '', '/?storage=opfs');
resolveStorageBackend();
expect(localStorage.getItem(STORAGE_PREFS_KEY)).toBe('true');
});
it('drops an unknown URL parameter without changing the stored choice', () => {
setStoredStorageBackend('memory-idb');
window.history.replaceState(null, '', '/?storage=nope');
expect(resolveStorageBackend()).toBe('memory-idb');
expect(window.location.search).toBe('');
});
it('falls back to the default when storage is unavailable', () => {
vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => {
throw new Error('denied');
});
expect(getStoredStorageBackend()).toBe('legacy');
});
});
@@ -0,0 +1,66 @@
import type { StorageBackend } from '#platform/server/sqlite';
// Which engine the backend worker uses for budget files. It has to be known
// before the worker starts, so it travels from the main thread in the init
// message instead of living in the backend's own preferences. A URL
// parameter sets it once; the stored value applies from then on.
export const STORAGE_BACKEND_KEY = 'actual:storage-backend';
export const STORAGE_BACKEND_URL_PARAM = 'storage';
// Reveals the engine setting in Experimental settings while it is hidden
// from everyone else
export const STORAGE_PREFS_KEY = 'devEnableStoragePrefs';
export const DEFAULT_STORAGE_BACKEND: StorageBackend = 'legacy';
const STORAGE_BACKENDS: readonly string[] = ['legacy', 'opfs', 'memory-idb'];
export function isStorageBackend(value: unknown): value is StorageBackend {
return typeof value === 'string' && STORAGE_BACKENDS.includes(value);
}
export function getStoredStorageBackend(): StorageBackend {
try {
const stored = localStorage.getItem(STORAGE_BACKEND_KEY);
return isStorageBackend(stored) ? stored : DEFAULT_STORAGE_BACKEND;
} catch {
// Storage access can throw (private modes, sandboxed frames); the
// default applies
return DEFAULT_STORAGE_BACKEND;
}
}
// Always stored explicitly, so an opt-out survives a change of the default
export function setStoredStorageBackend(backend: StorageBackend): void {
try {
localStorage.setItem(STORAGE_BACKEND_KEY, backend);
} catch {
// Without storage the default applies on the next load
}
}
// Reads `?storage=<backend>` once, stores it and strips it from the URL, so
// a reload or a later settings change is not overridden by a stale link.
// Using the flag also reveals the setting, so the way back is in the app.
export function resolveStorageBackend(): StorageBackend {
const url = new URL(window.location.href);
const requested = url.searchParams.get(STORAGE_BACKEND_URL_PARAM);
if (requested != null) {
if (isStorageBackend(requested)) {
setStoredStorageBackend(requested);
try {
localStorage.setItem(STORAGE_PREFS_KEY, 'true');
} catch {
// Without storage the setting stays hidden
}
}
url.searchParams.delete(STORAGE_BACKEND_URL_PARAM);
try {
window.history.replaceState(window.history.state, '', url);
} catch {
// A stale parameter only matters for the next load
}
}
// A valid link decides this load even where the choice could not be
// stored, with storage full or denied; it then just does not outlast it
return isStorageBackend(requested) ? requested : getStoredStorageBackend();
}
@@ -1,34 +1,116 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { WorkerBridge } from './worker-bridge';
import { releaseBackendStorage, startBrowserBackend } from './start';
import { releaseStorageThenTerminate, WorkerBridge } from './worker-bridge';
import type { TerminatableWorker } from './worker-bridge';
type FakeWorker = TerminatableWorker & {
posted: unknown[];
reply: (message: unknown) => void;
};
function fakeWorker(): FakeWorker {
const listeners: Array<(event: MessageEvent) => void> = [];
const worker: FakeWorker = {
onmessage: () => {
// Replaced by the bridge before the worker is released
},
posted: [],
postMessage: message => worker.posted.push(message),
addEventListener: (_type, handler) => listeners.push(handler),
terminate: vi.fn<() => void>(),
reply: message => {
for (const listener of listeners) {
listener({ data: message } as MessageEvent);
}
},
};
return worker;
}
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
describe('releaseStorageThenTerminate', () => {
it('asks the backend to release its storage and terminates on the reply', () => {
const worker = fakeWorker();
releaseStorageThenTerminate(worker, 300);
expect(worker.onmessage).toBeNull();
expect(worker.posted).toEqual([
expect.objectContaining({ name: 'storage-release' }),
]);
expect(worker.terminate).not.toHaveBeenCalled();
const request = worker.posted[0] as { id: string };
worker.reply({ type: 'reply', id: request.id, result: 'ok' });
expect(worker.terminate).toHaveBeenCalledTimes(1);
// The timeout must not terminate a second time
vi.advanceTimersByTime(1000);
expect(worker.terminate).toHaveBeenCalledTimes(1);
});
it('terminates anyway when the backend does not answer in time', () => {
const worker = fakeWorker();
releaseStorageThenTerminate(worker, 300);
vi.advanceTimersByTime(299);
expect(worker.terminate).not.toHaveBeenCalled();
vi.advanceTimersByTime(1);
expect(worker.terminate).toHaveBeenCalledTimes(1);
// A late reply changes nothing
const request = worker.posted[0] as { id: string };
worker.reply({ type: 'reply', id: request.id, result: 'ok' });
expect(worker.terminate).toHaveBeenCalledTimes(1);
});
it('ignores unrelated messages while waiting', () => {
const worker = fakeWorker();
releaseStorageThenTerminate(worker, 300);
worker.reply({ type: 'push', name: 'sync-event' });
worker.reply({ type: 'reply', id: 'something-else' });
expect(worker.terminate).not.toHaveBeenCalled();
});
});
class FakeBackendWorker {
static created: FakeBackendWorker[] = [];
posted: unknown[] = [];
onmessage: ((event: MessageEvent) => void) | null = null;
constructor() {
FakeBackendWorker.created.push(this);
}
postMessage(message: unknown) {
this.posted.push(message);
}
addEventListener() {
// absurd-sql's main-thread helper listens here; nothing to do
}
terminate() {
// Nothing to release in the fake
}
}
function fakeSharedPort() {
return {
postMessage: vi.fn(),
start: vi.fn(),
addEventListener: vi.fn(),
};
}
describe('WorkerBridge takeover', () => {
class FakeBackendWorker {
static created: FakeBackendWorker[] = [];
posted: unknown[] = [];
onmessage: ((event: MessageEvent) => void) | null = null;
constructor() {
FakeBackendWorker.created.push(this);
}
postMessage(message: unknown) {
this.posted.push(message);
}
addEventListener() {
// absurd-sql's main-thread helper listens here; nothing to do
}
terminate() {
// Nothing to release in the fake
}
}
function createBridge() {
FakeBackendWorker.created = [];
vi.stubGlobal('Worker', FakeBackendWorker);
const sharedPort = {
postMessage: vi.fn(),
start: vi.fn(),
addEventListener: vi.fn(),
};
const sharedPort = fakeSharedPort();
const bridge = new WorkerBridge(
sharedPort as unknown as MessagePort,
new URL('http://localhost/worker.js'),
@@ -92,3 +174,64 @@ describe('WorkerBridge takeover', () => {
]);
});
});
describe('releaseBackendStorage', () => {
const options = {
backendWorkerUrl: new URL('http://localhost/worker.js'),
initPayload: { version: 'test', isDev: false },
};
beforeEach(() => {
FakeBackendWorker.created = [];
vi.stubGlobal('Worker', FakeBackendWorker);
});
afterEach(() => {
vi.unstubAllGlobals();
});
it('asks the direct worker of the page to let go of its storage', () => {
const handle = startBrowserBackend({ ...options, forceDirectWorker: true });
expect(handle).toBe(FakeBackendWorker.created[0]);
releaseBackendStorage();
expect(FakeBackendWorker.created[0].posted).toEqual([
expect.objectContaining({ type: 'init' }),
expect.objectContaining({ name: 'storage-release' }),
]);
});
it('asks the worker a leader runs, and never the coordinator', () => {
vi.stubGlobal('SharedWorker', class {});
const sharedPort = fakeSharedPort();
startBrowserBackend({
...options,
createSharedWorker: () =>
({ port: sharedPort }) as unknown as SharedWorker,
});
const postedToCoordinator = sharedPort.postMessage.mock.calls.length;
// A follower runs no worker, so there is nothing to release
releaseBackendStorage();
expect(FakeBackendWorker.created).toEqual([]);
const onSharedMessage = sharedPort.addEventListener.mock.calls[0][1] as (
event: MessageEvent,
) => void;
onSharedMessage({
data: {
type: '__become-leader',
initMsg: { type: 'init' },
budgetToRestore: null,
pendingMsg: null,
},
} as MessageEvent);
releaseBackendStorage();
expect(FakeBackendWorker.created[0].posted).toEqual([
{ type: 'init' },
expect.objectContaining({ name: 'storage-release' }),
]);
expect(sharedPort.postMessage).toHaveBeenCalledTimes(postedToCoordinator);
});
});
@@ -4,8 +4,70 @@ import { logger } from '#platform/server/log';
type BridgeMessage = { type?: string; [key: string]: unknown };
// How long a leaving leader waits for the backend to let go of its storage
// before the worker is terminated regardless. The IndexedDB fallback writes
// a large budget whole here, which can take a good part of a second.
const STORAGE_RELEASE_TIMEOUT_MS = 3000;
const STORAGE_RELEASE_REQUEST_ID = '__storage-release';
const RESTORE_REQUEST_ID = '__restore-budget';
/** The part of a Worker the release step needs; fakes stand in for tests. */
export type TerminatableWorker = {
onmessage: ((event: MessageEvent) => void) | null;
postMessage: (message: unknown) => void;
addEventListener: (
type: 'message',
handler: (event: MessageEvent) => void,
) => void;
terminate: () => void;
};
/**
* Asks the backend to let go of its storage before the worker is
* terminated: the IndexedDB fallback persists its pending write, the OPFS
* store closes its access handles and releases the budget lock. A worker
* terminated outright releases access handles asynchronously, which makes
* the next leader's pool install retry. Terminates after a short timeout
* either way, so a hung backend cannot block the handover.
*/
export function releaseStorageThenTerminate(
worker: TerminatableWorker,
timeoutMs = STORAGE_RELEASE_TIMEOUT_MS,
) {
// Nothing else may hear from this worker now: its messages would reach a
// coordinator that no longer routes for it
worker.onmessage = null;
let isTerminated = false;
const terminate = () => {
if (!isTerminated) {
isTerminated = true;
clearTimeout(timer);
worker.terminate();
}
};
const timer = setTimeout(terminate, timeoutMs);
worker.addEventListener('message', event => {
const message = event.data as BridgeMessage | null;
if (message?.id === STORAGE_RELEASE_REQUEST_ID) {
terminate();
}
});
requestStorageRelease(worker);
}
/** Asks the backend to persist and let go of its storage; `id` names the reply. */
export function requestStorageRelease(
worker: Pick<TerminatableWorker, 'postMessage'>,
id = STORAGE_RELEASE_REQUEST_ID,
) {
worker.postMessage({
id,
name: 'storage-release',
args: undefined,
catchErrors: true,
});
}
/** The Worker-like surface the connection layer interacts with. */
export type WorkerLike = {
onmessage: ((e: MessageEvent) => void) | null;
@@ -98,10 +160,11 @@ export class WorkerBridge {
}
_terminateLocalBackendWorker() {
const worker = this.localBackendWorker;
this._heldRequests = null;
if (this.localBackendWorker) {
this.localBackendWorker.terminate();
if (worker) {
this.localBackendWorker = null;
releaseStorageThenTerminate(worker);
}
}
@@ -450,7 +450,7 @@ function lockBudget(filepath: string): Promise<() => void> {
BUDGET_LOCK_TIMEOUT_MS,
'shared',
).catch(() => {
throw new BudgetInUseError(filepath);
throw new BudgetInUseError(filepath, 'the storage lock could not be taken');
});
}
@@ -299,6 +299,43 @@ describe('web filesystem', () => {
const { node } = FS.lookupPath('/custom/xyz/db.sqlite', {});
expect(node.link).toBe('/blocked/' + pathToId('/custom/xyz/db.sqlite'));
});
test('releasing the storage closes the database and gives up its lock', async () => {
const dir = '/documents/Actual/leaving';
const { store } = idb.getStore(await idb.openDatabase(), 'files');
await idb.set(store, { filepath: dir + '/metadata.json', contents: '{}' });
await idb.set(store, { filepath: dir + '/db.sqlite', contents: '' });
await sqlite.init();
await init();
let isReleased = false;
const request = vi.fn(
(_name: string, _options: unknown, callback: () => Promise<void>) =>
callback().then(() => {
isReleased = true;
}),
);
Object.defineProperty(navigator, 'locks', {
value: { request },
configurable: true,
});
try {
const db = await sqlite.openDatabase(dir + '/db.sqlite');
expect(request).toHaveBeenCalledTimes(1);
expect(isReleased).toBe(false);
await sqlite.releaseSqliteStorage();
await vi.waitFor(() => expect(isReleased).toBe(true));
// Closed for good, so absurd-sql lets go of its IndexedDB connection
// too; the page relaunches if it comes back
expect(() => sqlite.runQuery(db, 'SELECT 1 AS one', [], true)).toThrow();
} finally {
Object.defineProperty(navigator, 'locks', {
value: undefined,
configurable: true,
});
}
});
});
describe('join', () => {
@@ -178,7 +178,10 @@ export async function openDatabase(pathOrBuffer?: string | Uint8Array) {
BUDGET_LOCK_TIMEOUT_MS,
'shared',
).catch(() => {
throw new BudgetInUseError(path);
throw new BudgetInUseError(
path,
'the storage lock is held exclusively elsewhere',
);
});
try {
// Another worker may have moved the budget to the new storage
@@ -243,6 +246,29 @@ export function closeDatabase(db: Database) {
}
}
// Closes every database that holds a budget lock, for a page that is
// leaving. The browser frees a destroyed worker's locks and connections
// itself, but WebKit keeps those of a worker whose page went into the
// back/forward cache, where they would block the next page's open until the
// cached page is evicted. Releasing the lock alone is not enough: absurd-sql
// keeps its IndexedDB connection in a worker that idles in `Atomics.wait`,
// which never sees the version change the next page uses to close it, so
// that page would wait forever. Closing the database makes absurd-sql close
// the connection over its shared buffer, which works even then. Such a page
// relaunches when it is restored, so a closed database is never used again.
export function closeOpenDatabases() {
for (const db of Array.from(budgetLocks.keys())) {
try {
closeDatabase(db);
} catch (error) {
logger.warn(
'sqlite: could not close a database of a leaving page',
error,
);
}
}
}
export async function exportDatabase(db: Database) {
return db.export();
}
@@ -1,6 +1,6 @@
import { logger } from '#platform/server/log';
import { getSqliteStore } from './storage';
import { checkStorageAfterIoError, getSqliteStore } from './storage';
import type { Database, SqliteEngine, SqlParam, Statement } from './types';
import {
exportBytes,
@@ -45,6 +45,26 @@ function isReadOnly(statement: PreparedStatement): boolean {
);
}
// The text of a statement, for the log: a cached one would otherwise be
// logged as an object
function sqlText(statement: PreparedStatement): string {
return statement.pointer == null
? ''
: getSqlite3().capi.sqlite3_sql(statement.pointer);
}
// Extended result codes keep the primary one in their low byte
function isIoError(error: unknown): boolean {
if (typeof error !== 'object' || error === null || !('resultCode' in error)) {
return false;
}
const { resultCode } = error;
return (
typeof resultCode === 'number' &&
(resultCode & 0xff) === getSqlite3().capi.SQLITE_IOERR
);
}
function verifyParamTypes(sql: string | Statement, params: SqlParam[]) {
for (const value of params) {
if (
@@ -100,7 +120,14 @@ function runQuery<T>(
statement.step();
return { changes: database.changes() };
} catch (error) {
logger.log(sql);
// Only failed reads are logged, as in the legacy engine: sync applies
// messages as insert-then-update, so failed writes are routine
if (fetchAll) {
logger.log(sqlText(statement));
}
if (isIoError(error)) {
checkStorageAfterIoError();
}
throw error;
} finally {
// Told even after a failure: a statement can leave changes behind
@@ -122,6 +149,11 @@ function execQuery(db: Database, sql: string) {
const database = asWasm(db);
try {
database.exec(sql);
} catch (error) {
if (isIoError(error)) {
checkStorageAfterIoError();
}
throw error;
} finally {
// A script can fail after its earlier statements were committed
notifyWrite(database);
@@ -25,6 +25,7 @@ export type {
} from './types';
export {
BudgetInUseError,
StorageLostError,
StorageMigrationError,
StorageUnavailableError,
} from './storage/errors';
@@ -54,10 +55,26 @@ export async function init(_options?: {
return 'legacy';
}
export function getStorageBackend(): StorageBackend {
return 'legacy';
}
export async function flushSqliteStorage(): Promise<void> {
// Native sqlite writes through to disk
}
export async function releaseSqliteStorage(): Promise<void> {
// Nothing to release on native sqlite
}
export function resumeSqliteStorage(): void {
// Nothing was released on native sqlite
}
export async function revertSqliteStorageToLegacy(): Promise<void> {
// Only the browser has a legacy storage layout
}
// Parity with the browser sqlite backend (which instantiates sql.js from an
// embedded wasm binary). better-sqlite3 has no wasm, so this is a no-op; it
// exists so callers can reference `setWasmBinary` regardless of platform.
@@ -2,11 +2,17 @@ import 'fake-indexeddb/auto';
import { beforeAll, describe, expect, it, vi } from 'vitest';
import { patchFetchForSqlJS } from '#mocks/util';
import * as connection from '#platform/server/connection';
import * as storage from './storage';
import { BudgetInUseError, StorageUnavailableError } from './storage/errors';
import { putSqliteFileRow } from './storage/files-store';
import { initWasmEngine } from './storage/test-helpers';
import type { Database } from './types';
import {
initWasmEngine,
readSqliteWasm,
resetIndexedDb,
} from './storage/test-helpers';
import type { Database, Statement } from './types';
import {
closeDatabase,
@@ -15,6 +21,8 @@ import {
init,
openDatabase,
prepare,
releaseSqliteStorage,
resumeSqliteStorage,
runQuery,
transaction,
} from './index';
@@ -38,6 +46,42 @@ function expectConstraintFailure(fn: () => void) {
consoleSpy.mockRestore();
}
// Satisfies the OPFS probe of `detectOpfsSyncAccess`, which jsdom cannot,
// for the duration of `run`. Once the browser has dropped its index of the
// private file system, lookups that create nothing fail.
async function withOpfsSyncAccess(
run: () => Promise<void>,
isIndexDropped: () => boolean = () => false,
): Promise<void> {
const directory = {
getFileHandle: async () => ({
createSyncAccessHandle: async () => ({ close: () => undefined }),
}),
removeEntry: async () => undefined,
};
Object.defineProperty(navigator, 'storage', {
configurable: true,
value: {
getDirectory: async () => ({
getDirectoryHandle: async (
_name: string,
options?: { create?: boolean },
) => {
if (!options?.create && isIndexDropped()) {
throw new DOMException('could not be found', 'NotFoundError');
}
return directory;
},
}),
},
});
try {
await run();
} finally {
Reflect.deleteProperty(navigator, 'storage');
}
}
// The same contract is expected of both web engines; each describe block
// below switches the facade to one of them.
function itBehavesLikeSqlite() {
@@ -422,4 +466,179 @@ describe('sqlite-wasm engine', () => {
}
closeDatabase(db);
});
it('logs the text of a failed read, and nothing for a failed write', async () => {
const db = await openWithSchema();
runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 1)");
const log = vi.spyOn(console, 'log').mockImplementation(() => null);
try {
expect(() =>
runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 2)"),
).toThrow(/constraint failed/i);
expect(log).not.toHaveBeenCalled();
// Fails when stepped rather than prepared, so it can be cached
const statement = prepare(db, 'SELECT abs(-9223372036854775808)');
expect(() => runQuery(db, statement, [], true)).toThrow(/overflow/i);
expect(log).toHaveBeenCalledWith('SELECT abs(-9223372036854775808)');
} finally {
log.mockRestore();
}
closeDatabase(db);
});
it('has the storage looked at when a statement fails with an I/O error', async () => {
const db = await openWithSchema();
runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 1)");
const check = vi
.spyOn(storage, 'checkStorageAfterIoError')
.mockImplementation(() => undefined);
try {
// A routine failure is none of the storage's business
expectConstraintFailure(() =>
runQuery(db, "INSERT INTO numbers (id, number) VALUES ('id1', 2)"),
);
expect(check).not.toHaveBeenCalled();
// SQLITE_IOERR_WRITE, what a write gets once the browser has dropped
// the file under an open database
const ioError = Object.assign(new Error('disk I/O error'), {
resultCode: 778,
});
const failing = {
step: () => {
throw ioError;
},
reset: () => undefined,
};
expect(() =>
runQuery(db, failing as unknown as Statement, [], false),
).toThrow(ioError);
expect(check).toHaveBeenCalledTimes(1);
const exec = vi
.spyOn(db as unknown as { exec: (sql: string) => void }, 'exec')
.mockImplementation(() => {
throw ioError;
});
expect(() => execQuery(db, 'BEGIN')).toThrow(ioError);
exec.mockRestore();
expect(check).toHaveBeenCalledTimes(2);
} finally {
check.mockRestore();
}
closeDatabase(db);
});
it('tells the client when the browser drops the private file system', async () => {
const wasmBinary = readSqliteWasm();
const send = vi
.spyOn(connection, 'send')
.mockImplementation(() => undefined);
const error = vi.spyOn(console, 'error').mockImplementation(() => null);
let isIndexDropped = false;
try {
await withOpfsSyncAccess(
async () => {
await resetIndexedDb();
expect(await init({ storage: 'opfs', wasmBinary })).toBe('opfs');
// An I/O error with the files still there says nothing
storage.checkStorageAfterIoError();
await new Promise(resolve => setTimeout(resolve, 10));
expect(send).not.toHaveBeenCalledWith('storage-lost');
isIndexDropped = true;
storage.checkStorageAfterIoError();
await vi.waitFor(() =>
expect(send).toHaveBeenCalledWith('storage-lost'),
);
},
() => isIndexDropped,
);
} finally {
send.mockRestore();
error.mockRestore();
await initWasmEngine();
}
});
it('opens no stored database once the storage is let go, until it is resumed', async () => {
await resetIndexedDb();
await initWasmEngine();
const path = '/documents/Actual/let-go/db.sqlite';
try {
await releaseSqliteStorage();
await expect(openDatabase(path)).rejects.toBeInstanceOf(
StorageUnavailableError,
);
// A database in memory holds no storage
closeDatabase(await openDatabase());
resumeSqliteStorage();
closeDatabase(await openDatabase(path));
// The new file's first save, so that it does not outlast the test
await storage.getSqliteStore().flush();
} finally {
await initWasmEngine();
}
});
it('gives back a database that was still being opened when the storage was let go', async () => {
await resetIndexedDb();
await initWasmEngine();
let heldLocks = 0;
Object.defineProperty(navigator, 'locks', {
configurable: true,
value: {
request: (
_name: string,
_options: unknown,
hold: () => Promise<void>,
) => {
heldLocks += 1;
return hold().then(() => {
heldLocks -= 1;
});
},
},
});
try {
const opening = openDatabase('/documents/Actual/on-its-way/db.sqlite');
await releaseSqliteStorage();
// Under way, with the budget's lock already taken
expect(heldLocks).toBe(1);
await expect(opening).rejects.toBeInstanceOf(StorageUnavailableError);
await vi.waitFor(() => expect(heldLocks).toBe(0));
} finally {
Reflect.deleteProperty(navigator, 'locks');
await initWasmEngine();
}
});
it('asks for OPFS when a fallback request meets data held in OPFS', async () => {
const wasmBinary = readSqliteWasm();
try {
await withOpfsSyncAccess(async () => {
await resetIndexedDb();
await putSqliteFileRow({
filepath: '/documents/Actual/a/db.sqlite',
contents: new Uint8Array(0),
storage: 'memory-idb',
});
expect(await init({ storage: 'memory-idb', wasmBinary })).toBe(
'memory-idb',
);
await putSqliteFileRow({
filepath: '/documents/Actual/b/db.sqlite',
contents: '',
storage: 'opfs',
});
expect(await init({ storage: 'memory-idb', wasmBinary })).toBe('opfs');
expect(await init({ storage: 'legacy', wasmBinary })).toBe('opfs');
});
} finally {
await initWasmEngine();
}
});
});
@@ -5,7 +5,8 @@ import type {
import * as legacy from './engine.sqljs.legacy';
import { wasmEngine } from './engine.wasm';
import { getSqliteStore, initSqliteStorage } from './storage';
import { getSqliteStore, initSqliteStorage, revertToLegacy } from './storage';
import { StorageUnavailableError } from './storage/errors';
import { hasMigratedSqliteFiles } from './storage/files-store';
import type {
Database,
@@ -25,6 +26,7 @@ export type {
} from './types';
export {
BudgetInUseError,
StorageLostError,
StorageMigrationError,
StorageUnavailableError,
} from './storage/errors';
@@ -72,17 +74,22 @@ const legacyEngine: SqliteEngine = {
};
let activeEngine: SqliteEngine = legacyEngine;
let activeBackend: StorageBackend = 'legacy';
// Opt-in: the browser build of `@actual-app/api` embeds the wasm of the
// engine it ships and hands it over here before `init`, so the engine is
// instantiated from these bytes instead of fetching them.
let wasmBinaryOverride: ArrayBuffer | Uint8Array | undefined;
// The browser build of `@actual-app/api` embeds sql.js's wasm; the
// sqlite-wasm engine fetches its own file.
export function setWasmBinary(binary: ArrayBuffer | Uint8Array) {
legacyEngine.setWasmBinary(binary);
wasmBinaryOverride = binary;
}
// Starts the engine for the requested storage and resolves to the backend
// actually in use: data already moved to the new storage wins over a
// `legacy` request, so a cleared flag never shows an empty app, and `opfs`
// falls back to `memory-idb` where sync access handles are unavailable.
// `legacy` request and data held in OPFS over a `memory-idb` one, so a
// cleared or stale flag never shows an empty app, and `opfs` falls back to
// `memory-idb` where sync access handles are unavailable.
export async function init({
baseURL,
wasmBinary,
@@ -92,29 +99,81 @@ export async function init({
wasmBinary?: ArrayBuffer | Uint8Array;
storage?: StorageBackend;
} = {}): Promise<StorageBackend> {
// Only the OPFS store can read its pools, and it adopts the fallback's
// rows, so any migrated data means asking for OPFS (which itself falls
// back where sync access handles are missing)
isReleased = false;
// The legacy engine reads neither store's data and the fallback cannot
// read OPFS pools, while the OPFS store adopts the fallback's rows, so
// data the requested engine could not read means asking for OPFS instead
// (which itself falls back where sync access handles are missing)
const requested =
storage === 'legacy' && (await hasMigratedSqliteFiles()) ? 'opfs' : storage;
(storage === 'legacy' && (await hasMigratedSqliteFiles())) ||
(storage === 'memory-idb' && (await hasMigratedSqliteFiles('opfs')))
? 'opfs'
: storage;
if (requested === 'legacy') {
activeEngine = legacyEngine;
await legacyEngine.init({ baseURL, wasmBinary });
return 'legacy';
await legacyEngine.init({
baseURL,
wasmBinary: wasmBinary ?? wasmBinaryOverride,
});
activeBackend = 'legacy';
} else {
activeEngine = wasmEngine;
await wasmEngine.init({
baseURL,
wasmBinary: wasmBinary ?? wasmBinaryOverride,
});
activeBackend = await initSqliteStorage(requested);
}
activeEngine = wasmEngine;
await wasmEngine.init({ baseURL, wasmBinary });
return initSqliteStorage(requested);
return activeBackend;
}
// Persist anything pending (the IndexedDB fallback writes lazily); rejects
// when a file could not be written
export function getStorageBackend(): StorageBackend {
return activeBackend;
}
// Persist anything pending (the IndexedDB fallback writes lazily)
export async function flushSqliteStorage(): Promise<void> {
if (activeEngine === wasmEngine) {
if (activeBackend !== 'legacy') {
await getSqliteStore().flush();
}
}
// Set once the storage was let go of. A budget can be on its way in just
// then; it would open its database afterwards and hold the budget from then
// on, in a worker that is about to end or to be frozen along with its page.
// So nothing stored is opened until `resumeSqliteStorage`
let isReleased = false;
function assertNotReleased() {
if (isReleased) {
throw new StorageUnavailableError('this worker has let go of its storage');
}
}
// Let go of storage handles and locks, so another worker can take over or
// so a page that is leaving holds nothing back
export async function releaseSqliteStorage(): Promise<void> {
isReleased = true;
if (activeBackend === 'legacy') {
legacy.closeOpenDatabases();
} else {
await getSqliteStore().releaseAll();
}
}
// For a worker whose page came back out of the back/forward cache
export function resumeSqliteStorage(): void {
isReleased = false;
}
// Move every budget back to absurd-sql's layout (the opt-in phase's way
// back); the next start then picks the legacy engine
export async function revertSqliteStorageToLegacy(): Promise<void> {
if (activeBackend !== 'legacy') {
await revertToLegacy();
}
}
export function prepare(db: Database, sql: string): Statement {
return activeEngine.prepare(db, sql);
}
@@ -215,7 +274,18 @@ export async function asyncTransaction(db: Database, fn: () => Promise<void>) {
export async function openDatabase(
pathOrBuffer?: string | Uint8Array,
): Promise<Database> {
return activeEngine.openDatabase(pathOrBuffer);
if (typeof pathOrBuffer !== 'string' || pathOrBuffer === ':memory:') {
// In memory, which holds no storage
return activeEngine.openDatabase(pathOrBuffer);
}
assertNotReleased();
const db = await activeEngine.openDatabase(pathOrBuffer);
if (isReleased) {
// Let go of while this database was being opened, so it goes back too
activeEngine.closeDatabase(db);
}
assertNotReleased();
return db;
}
export function closeDatabase(db: Database) {
@@ -1,5 +1,5 @@
import 'fake-indexeddb/auto';
import { beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import {
closeDatabase,
@@ -203,6 +203,30 @@ describe('absurd-sql migration helpers', () => {
}
});
it('reports the budget in use when nothing answers the version change', async () => {
await seedLegacyDatabase(NAME, await makeDatabaseBytes(5));
// A connection in a worker that never reaches its event loop never
// acknowledges the version change, and the browser reports `blocked`
// only once every connection has: the open request just stays pending
const realOpen = indexedDB.open.bind(indexedDB);
const open = vi
.spyOn(indexedDB, 'open')
.mockImplementation((name, version) =>
version == null ? realOpen(name) : ({} as unknown as IDBOpenDBRequest),
);
vi.useFakeTimers();
try {
const pending = readLegacyDatabaseBytes(PATH);
pending.catch(() => undefined);
await vi.advanceTimersByTimeAsync(5000);
await expect(pending).rejects.toBeInstanceOf(BudgetInUseError);
} finally {
vi.useRealTimers();
open.mockRestore();
}
expect(await legacyVersion(NAME)).toBe(2);
});
it('refuses the cutover when a connection to the copy appeared meanwhile', async () => {
const bytes = await makeDatabaseBytes(5);
await seedLegacyDatabase(NAME, bytes);
@@ -11,14 +11,18 @@ import type { MigrationProgress } from './types';
// after the file path, with a single object store `data`: key -1 holds
// `{ size }` and keys 0..N hold the file's blocks, one sqlite page each.
// These helpers read that layout so a budget can be moved into the new
// storage, and finish the move the same way for both stores.
// storage, finish the move the same way for both stores, and write it back
// so the opt-in can be reversed.
const LEGACY_STORE = 'data';
// absurd-sql opens its databases at this version
const LEGACY_VERSION = 2;
const BLOCKS_PER_CHUNK = 256;
const DELETE_TIMEOUT_MS = 5000;
// After `blocked`, how long a connection gets to close
const EVICT_TIMEOUT_MS = 1000;
// How long the version change gets when nothing answers it at all
const EVICT_RESPONSE_TIMEOUT_MS = 5000;
const SQLITE_HEADER = 'SQLite format 3\0';
// The sqlite file header, whose change counter moves on every write
const HEADER_BYTES = 100;
@@ -53,7 +57,13 @@ function matchesSnapshot(
// longer matches it: absurd-sql reopens its database on the next
// operation, so a tab on the previous release can write between the copy
// and the cutover. Resolves whether the database exists, and rejects when
// a connection will not close.
// a connection will not close, or when nothing answers: the browser reports
// `blocked` only once every connection has seen the version change, and
// absurd-sql's connection lives in a worker that idles in `Atomics.wait`,
// which never sees it, whether its tab is frozen in the back/forward cache
// or still running. So the request as a whole gets a generous bound, as a
// cold open of a fresh database can be slow, and a `blocked` event a short
// one.
function evictLegacyConnections(
name: string,
path: string,
@@ -63,10 +73,18 @@ function evictLegacyConnections(
const request = indexedDB.open(name, LEGACY_VERSION + 1);
let exists = true;
let changed = false;
const timer = setTimeout(
() => reject(new BudgetInUseError(path)),
EVICT_TIMEOUT_MS,
const fail = (reason: string) => reject(new BudgetInUseError(path, reason));
let timer = setTimeout(
() => fail('a connection to the absurd-sql copy did not answer'),
EVICT_RESPONSE_TIMEOUT_MS,
);
request.onblocked = () => {
clearTimeout(timer);
timer = setTimeout(
() => fail('a connection to the absurd-sql copy would not close'),
EVICT_TIMEOUT_MS,
);
};
request.onupgradeneeded = event => {
exists = event.oldVersion !== 0;
const transaction = request.transaction;
@@ -91,7 +109,12 @@ function evictLegacyConnections(
request.onerror = () => {
clearTimeout(timer);
if (changed) {
reject(new BudgetInUseError(path));
reject(
new BudgetInUseError(
path,
'the absurd-sql copy changed during the move',
),
);
} else {
resolve(exists);
}
@@ -252,6 +275,7 @@ export async function openLegacyDatabaseContents(
onProgress?: (percent: number) => void,
): Promise<LegacyDatabaseContents | null> {
const name = legacyDatabaseName(path);
logger.info('sqlite storage: looking for an absurd-sql copy', path);
const database = await openLegacyDatabase(name, path);
if (database == null) {
return null;
@@ -304,6 +328,51 @@ export async function readLegacyDatabaseBytes(
return bytes;
}
// Writes a sqlite file back into absurd-sql's layout so switching the
// storage flag off again keeps the data. Any previous contents are cleared
// in the same transaction as the new blocks are written, so the store is
// never left with a mix of the two.
export async function writeLegacyDatabaseBytes(
name: string,
bytes: Uint8Array,
): Promise<void> {
const blockSize = pageSizeFromHeader(bytes) ?? 4096;
const database = await new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open(name, LEGACY_VERSION);
request.onupgradeneeded = () => {
if (!request.result.objectStoreNames.contains(LEGACY_STORE)) {
request.result.createObjectStore(LEGACY_STORE);
}
};
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error);
});
try {
const transaction = database.transaction([LEGACY_STORE], 'readwrite');
const store = transaction.objectStore(LEGACY_STORE);
store.clear();
store.put({ size: bytes.length }, -1);
for (
let offset = 0, key = 0;
offset < bytes.length;
offset += blockSize, key++
) {
const block = new Uint8Array(blockSize);
block.set(
bytes.subarray(offset, Math.min(offset + blockSize, bytes.length)),
);
store.put(block.buffer, key);
}
await new Promise<void>((resolve, reject) => {
transaction.oncomplete = () => resolve();
transaction.onerror = () => reject(transaction.error);
transaction.onabort = () => reject(transaction.error);
});
} finally {
database.close();
}
}
// What both stores check on a freshly copied database before trusting
// it: structurally sound, and an Actual budget
export function verifyMigratedDatabase(db: WasmDatabase) {
@@ -332,7 +401,10 @@ export async function completeMigration(
const name = legacyDatabaseName(marker.filepath);
await evictLegacyConnections(name, marker.filepath, snapshot);
if (!(await putSqliteFileRowIfRev(marker, expectedRev))) {
throw new BudgetInUseError(marker.filepath);
throw new BudgetInUseError(
marker.filepath,
'another worker moved it first',
);
}
if (!(await deleteLegacyDatabase(name))) {
logger.warn(
@@ -5,8 +5,9 @@ import { getSqlite3 } from '#platform/server/sqlite/wasm';
// pool install deletes the pool directory, which is never acceptable for
// user data
export const MIN_SQLITE_VERSION = 3054000;
// The parent of the pool directories (see POOL_ROOT in opfs-sahpool.ts)
const PROBE_DIRECTORY = 'actual';
// The parent of the pool directories (see POOL_ROOT in opfs-sahpool.ts),
// which the probe below leaves behind
export const PROBE_DIRECTORY = 'actual';
// Whether this worker can use OPFS sync access handles: present (dedicated
// worker, secure context, Chrome 108, Firefox 111, Safari 16.4) and usable
@@ -1,11 +1,17 @@
import { logger } from '#platform/server/log';
// Errors the storage layer raises while opening a budget's database. The
// budget loader maps them to user-facing error codes. Kept free of wasm
// imports so every platform module can re-export them.
// The user sees one generic message for every cause, so the cause is
// logged here, where the error is raised, for the console to tell them
// apart
export class BudgetInUseError extends Error {
constructor(path: string) {
super(`The database at ${path} is open in another tab or window`);
constructor(path: string, reason: string) {
super(`The database at ${path} is in use: ${reason}`);
this.name = 'BudgetInUseError';
logger.warn('sqlite storage: budget in use', path, reason);
}
}
@@ -23,3 +29,12 @@ export class StorageUnavailableError extends Error {
this.name = 'StorageUnavailableError';
}
}
// The browser no longer has the bytes of a file its marker says it holds.
// Unlike storage that is out of reach, nothing brings them back
export class StorageLostError extends Error {
constructor(path: string) {
super(`The browser no longer holds the database at ${path}`);
this.name = 'StorageLostError';
}
}
@@ -0,0 +1,50 @@
import 'fake-indexeddb/auto';
import { beforeEach, describe, expect, it } from 'vitest';
import {
hasMigratedSqliteFiles,
listSqliteFileRows,
putSqliteFileRow,
} from './files-store';
import { resetIndexedDb } from './test-helpers';
beforeEach(() => resetIndexedDb());
describe('sqlite file rows', () => {
it('finds no migrated storage among legacy markers', async () => {
await putSqliteFileRow({
filepath: '/documents/a/db.sqlite',
contents: '',
});
expect(await listSqliteFileRows()).toEqual([]);
expect(await hasMigratedSqliteFiles()).toBe(false);
});
it('reports migrated data when only fallback rows exist', async () => {
await putSqliteFileRow({
filepath: '/documents/a/db.sqlite',
contents: new Uint8Array(0),
storage: 'memory-idb',
});
expect(await hasMigratedSqliteFiles()).toBe(true);
expect(await hasMigratedSqliteFiles('opfs')).toBe(false);
});
it('reports migrated data when both kinds of rows exist', async () => {
await putSqliteFileRow({
filepath: '/documents/a/db.sqlite',
contents: new Uint8Array(0),
storage: 'memory-idb',
});
await putSqliteFileRow({
filepath: '/documents/b/db.sqlite',
contents: '',
storage: 'opfs',
});
expect(await hasMigratedSqliteFiles()).toBe(true);
expect(await hasMigratedSqliteFiles('opfs')).toBe(true);
expect(
(await listSqliteFileRows()).map(row => row.filepath).sort(),
).toEqual(['/documents/a/db.sqlite', '/documents/b/db.sqlite']);
});
});
@@ -1,6 +1,8 @@
import * as idb from '#platform/server/indexeddb';
import type { Data } from '#platform/server/indexeddb';
import type { FileStoreKind } from './types';
// The IndexedDB `files` store keeps one row per persisted path. For a
// sqlite path the row is the marker the filesystem rebuilds its hierarchy
// from, records which store holds the bytes, and for the IndexedDB
@@ -62,14 +64,43 @@ export async function deleteSqliteFileRow(path: string): Promise<void> {
await Promise.all([idb.del(store, path), committed(trans)]);
}
// Whether any sqlite file already moved off absurd-sql. Used at startup:
// once data lives in the new storage the legacy engine must not be chosen,
// whatever the flag says. One store then serves every budget: one still in
// absurd-sql's layout is migrated when opened, and the OPFS store adopts
// the fallback's rows. Walks the rows with a cursor and stops at the first
// marked one, so the fallback's whole-file rows are not all read into
// memory.
export async function hasMigratedSqliteFiles(): Promise<boolean> {
// Rows of sqlite files held by the sqlite-wasm storage, without their
// contents. Walks the rows with a cursor so the fallback's whole-file rows
// are held one at a time rather than all at once.
export async function listSqliteFileRows(): Promise<
Pick<Data, 'filepath' | 'storage'>[]
> {
const { store } = idb.getStore(await idb.openDatabase(), 'files');
return new Promise((resolve, reject) => {
const rows: Pick<Data, 'filepath' | 'storage'>[] = [];
const request = store.openCursor();
request.onsuccess = () => {
const cursor = request.result;
if (!cursor) {
resolve(rows);
return;
}
const { filepath, storage } = cursor.value as Data;
if (storage != null) {
rows.push({ filepath, storage });
}
cursor.continue();
};
request.onerror = () => reject(request.error);
});
}
// Whether any sqlite file already moved off absurd-sql, or into `kind` when
// given. Used at startup: once data lives in the new storage the legacy
// engine must not be chosen, and once data lives in an OPFS pool the
// fallback must not be, whatever the flag says. One store then serves every
// budget: one still in absurd-sql's layout is migrated when opened, and the
// OPFS store adopts the fallback's rows. Walks the rows with a cursor and
// stops at the first match, so the fallback's whole-file rows are not all
// read into memory.
export async function hasMigratedSqliteFiles(
kind?: FileStoreKind,
): Promise<boolean> {
const { store } = idb.getStore(await idb.openDatabase(), 'files');
return new Promise((resolve, reject) => {
const request = store.openCursor();
@@ -77,7 +108,10 @@ export async function hasMigratedSqliteFiles(): Promise<boolean> {
const cursor = request.result;
if (!cursor) {
resolve(false);
} else if ((cursor.value as Data).storage) {
return;
}
const { storage } = cursor.value as Data;
if (storage && (kind == null || storage === kind)) {
resolve(true);
} else {
cursor.continue();
@@ -509,6 +509,82 @@ describe('IndexedDB in-memory store', () => {
expect(await store.exists(other)).toBe(false);
});
it('leaves the legacy marker and the absurd-sql copy when a file moves back', async () => {
const store = getSqliteStore();
await store.writeFile(PATH, new Uint8Array([1]));
const name = legacyDatabaseName(PATH);
await seedLegacyDatabase(name, new Uint8Array([1]));
await store.revertFile(PATH);
expect(await getSqliteFileRow(PATH)).toEqual({
filepath: PATH,
contents: '',
});
expect(await listIndexedDbNames()).toContain(name);
// A plain removal takes the copy with it
await store.removeFile(PATH);
expect(await getSqliteFileRow(PATH)).toBeUndefined();
expect(await listIndexedDbNames()).not.toContain(name);
});
it('leaves alone a file another worker has already moved back', async () => {
// What a finished revert leaves: the legacy marker and its copy. A
// second revert that listed the file earlier finds them once it has
// the lock
await putSqliteFileRow({ filepath: PATH, contents: '' });
const name = legacyDatabaseName(PATH);
await seedLegacyDatabase(name, new Uint8Array([1]));
await getSqliteStore().revertFile(PATH);
expect(await getSqliteFileRow(PATH)).toEqual({
filepath: PATH,
contents: '',
});
expect(await listIndexedDbNames()).toContain(name);
});
it('waits out a slow hand-over before it reports the budget in use', async () => {
// A lock that is never granted, and gives up only when told to
const request = vi.fn(
(_name: string, options: { signal: AbortSignal }) =>
new Promise((_resolve, reject) => {
options.signal.addEventListener('abort', () =>
reject(new DOMException('aborted', 'AbortError')),
);
}),
);
Object.defineProperty(navigator, 'locks', {
value: { request },
configurable: true,
});
vi.useFakeTimers();
try {
let outcome = 'waiting';
const opening = getSqliteStore().open(PATH);
opening.then(
() => {
outcome = 'opened';
},
() => {
outcome = 'refused';
},
);
// A leaving leader gets three seconds to save and let go
await vi.advanceTimersByTimeAsync(3000);
expect(outcome).toBe('waiting');
await vi.advanceTimersByTimeAsync(7000);
await expect(opening).rejects.toBeInstanceOf(BudgetInUseError);
} finally {
vi.useRealTimers();
Object.defineProperty(navigator, 'locks', {
value: undefined,
configurable: true,
});
}
});
it('writes a row only at the revision it expects', async () => {
await putSqliteFileRow({ filepath: PATH, contents: '', rev: 1 });
const next = { filepath: PATH, contents: new Uint8Array([1]), rev: 2 };
@@ -562,6 +638,36 @@ describe('IndexedDB in-memory store', () => {
closeDatabase(reopened);
});
it('reads and moves back the row another worker wrote, not a snapshot it moved past', async () => {
await strandSnapshot('stranded');
const source = await openDatabase();
execQuery(source, SCHEMA);
runQuery(source, "INSERT INTO t (v) VALUES ('from elsewhere')");
await putSqliteFileRow({
filepath: PATH,
contents: await exportDatabase(source),
storage: 'memory-idb',
rev: 9,
});
closeDatabase(source);
const store = getSqliteStore();
const read = await openDatabase(await store.readFile(PATH));
expect(runQuery(read, 'SELECT v FROM t', [], true)).toEqual([
{ v: 'from elsewhere' },
]);
closeDatabase(read);
await store.revertFile(PATH);
const back = await openDatabase(
(await absurdMigration.readLegacyDatabaseBytes(PATH)) as Uint8Array,
);
expect(runQuery(back, 'SELECT v FROM t', [], true)).toEqual([
{ v: 'from elsewhere' },
]);
closeDatabase(back);
});
it('refuses a new file another worker created meanwhile', async () => {
const lookup = vi
.spyOn(absurdMigration, 'readLegacyDatabaseBytes')
@@ -14,6 +14,7 @@ import {
legacySnapshot,
readLegacyDatabaseBytes,
verifyMigratedDatabase,
writeLegacyDatabaseBytes,
} from './absurd-migration';
import {
BudgetInUseError,
@@ -26,7 +27,11 @@ import {
putSqliteFileRow,
putSqliteFileRowIfRev,
} from './files-store';
import { acquireWebLock, budgetLockName } from './locks';
import {
acquireWebLock,
BUDGET_LOCK_TIMEOUT_MS,
budgetLockName,
} from './locks';
import { legacyDatabaseName } from './paths';
import type { MigrationProgress, SqliteFileStore } from './types';
@@ -40,7 +45,6 @@ import type { MigrationProgress, SqliteFileStore } from './types';
const DEFAULT_DEBOUNCE_MS = 1000;
const DEFAULT_MAX_DIRTY_MS = 5000;
const DEFAULT_LOCK_TIMEOUT_MS = 2000;
export type MemoryIdbStoreOptions = {
onMigrationProgress?: MigrationProgress;
@@ -75,7 +79,7 @@ export function createMemoryIdbStore({
onMigrationProgress,
debounceMs = DEFAULT_DEBOUNCE_MS,
maxDirtyMs = DEFAULT_MAX_DIRTY_MS,
lockTimeoutMs = DEFAULT_LOCK_TIMEOUT_MS,
lockTimeoutMs = BUDGET_LOCK_TIMEOUT_MS,
}: MemoryIdbStoreOptions = {}): SqliteFileStore {
const entries = new Map<string, Entry>();
const byDb = new Map<WasmDatabase, Entry>();
@@ -218,9 +222,21 @@ export function createMemoryIdbStore({
return { bytes, rev: rev + 1 };
}
// A snapshot stranded by a failed close is only current while the row is
// still at the revision it was taken from; a write by another worker
// since supersedes it
function currentStranded(path: string, row: Data | undefined) {
const stranded = unsaved.get(path);
if (stranded && (row?.rev ?? 0) !== stranded.rev) {
unsaved.delete(path);
return undefined;
}
return stranded;
}
function lockBudget(path: string): Promise<() => void> {
return acquireWebLock(budgetLockName(path), lockTimeoutMs).catch(() => {
throw new BudgetInUseError(path);
throw new BudgetInUseError(path, 'the storage lock could not be taken');
});
}
@@ -252,13 +268,7 @@ export function createMemoryIdbStore({
try {
const row = await getSqliteFileRow(path);
assertNotInOpfs(path, row);
// A snapshot stranded by a failed close is only current while the
// row is still at the revision it was taken from
let stranded = unsaved.get(path);
if (stranded && (row?.rev ?? 0) !== stranded.rev) {
unsaved.delete(path);
stranded = undefined;
}
const stranded = currentStranded(path, row);
const { bytes, rev } = stranded ?? (await loadBytes(path, row));
const db = openMemoryDatabase(bytes ?? undefined);
const entry: Entry = {
@@ -288,7 +298,10 @@ export function createMemoryIdbStore({
);
if (!written) {
db.close();
throw new BudgetInUseError(path);
throw new BudgetInUseError(
path,
'another worker created it meanwhile',
);
}
}
if (bytes == null || stranded) {
@@ -376,15 +389,17 @@ export function createMemoryIdbStore({
if (entry) {
return exportBytes(entry.db);
}
const stranded = unsaved.get(path);
if (stranded) {
return stranded.bytes;
}
// Loading may migrate the absurd-sql copy, so hold the budget lock
// as an open does
const releaseLock = await lockBudget(path);
try {
const { bytes } = await loadBytes(path, await getSqliteFileRow(path));
const row = await getSqliteFileRow(path);
assertNotInOpfs(path, row);
const stranded = currentStranded(path, row);
if (stranded) {
return stranded.bytes;
}
const { bytes } = await loadBytes(path, row);
if (bytes == null) {
throw new Error('File does not exist: ' + path);
}
@@ -465,6 +480,39 @@ export function createMemoryIdbStore({
}
},
// Bytes a failed save left in memory count as the file's, as in a read
async revertFile(path) {
await opening.get(path)?.catch(() => undefined);
await pendingWrites.get(path);
if (entries.has(path)) {
throw new Error(`cannot revert an open database: ${path}`);
}
const releaseLock = await lockBudget(path);
try {
const row = await getSqliteFileRow(path);
// Another worker moved it back while this one waited for the lock,
// or it is gone: the legacy marker is not this store's to touch
if (row?.storage == null) {
return;
}
assertNotInOpfs(path, row);
const stranded = currentStranded(path, row);
const bytes =
stranded?.bytes ??
(isByteContents(row.contents) ? row.contents : null);
if (bytes == null) {
// Recorded but never saved: nothing to copy, the row just goes
await deleteSqliteFileRow(path);
return;
}
await writeLegacyDatabaseBytes(legacyDatabaseName(path), bytes);
await putSqliteFileRow({ filepath: path, contents: '' });
unsaved.delete(path);
} finally {
releaseLock();
}
},
async exists(path) {
return (await getSqliteFileRow(path)) != null;
},
@@ -487,6 +535,13 @@ export function createMemoryIdbStore({
}
},
async releaseAll() {
for (const entry of [...entries.values()]) {
store.close(entry.path, entry.db);
}
await Promise.all([...pendingWrites.values()]);
},
onWrite(db) {
const entry = byDb.get(db);
if (!entry) {
@@ -1,13 +1,17 @@
import * as connection from '#platform/server/connection';
import { detectOpfsSyncAccess } from './config';
import { detectOpfsSyncAccess, PROBE_DIRECTORY } from './config';
import { listSqliteFileRows } from './files-store';
import { createMemoryIdbStore } from './idb-memory';
import { startOpfsKeepAlive } from './opfs-keep-alive';
import type { OpfsKeepAlive } from './opfs-keep-alive';
import { createOpfsStore } from './opfs-sahpool';
import type { FileStoreKind, MigrationPhase, SqliteFileStore } from './types';
export type { FileStoreKind, SqliteFileStore } from './types';
let store: SqliteFileStore | null = null;
let keepAlive: OpfsKeepAlive | null = null;
function reportMigration(phase: MigrationPhase, percent?: number) {
connection.send('storage-migration', { phase, percent });
@@ -22,13 +26,29 @@ export async function initSqliteStorage(
requested === 'opfs' && (await detectOpfsSyncAccess())
? 'opfs'
: 'memory-idb';
store =
kind === 'opfs'
? createOpfsStore({ onMigrationProgress: reportMigration })
: createMemoryIdbStore({ onMigrationProgress: reportMigration });
keepAlive?.stop();
keepAlive = null;
if (kind === 'opfs') {
store = createOpfsStore({ onMigrationProgress: reportMigration });
// From here on, not from the first budget: the app can sit on the
// budget list for longer than the browser keeps its index open
keepAlive = await startOpfsKeepAlive({
directoryName: PROBE_DIRECTORY,
onLost: () => connection.send('storage-lost'),
});
} else {
store = createMemoryIdbStore({ onMigrationProgress: reportMigration });
}
return kind;
}
// The engine saw an I/O error. Where the browser dropped the private file
// system that is the first sign of it, so look now rather than at the next
// keep-alive; any other cause leaves the lookup working and changes nothing
export function checkStorageAfterIoError() {
void keepAlive?.check();
}
export function getSqliteStore(): SqliteFileStore {
if (store == null) {
throw new Error('getSqliteStore: sqlite storage is not initialized');
@@ -40,3 +60,19 @@ export function getSqliteStore(): SqliteFileStore {
export function setSqliteStore(next: SqliteFileStore | null) {
store = next;
}
// Move every budget back to absurd-sql's layout and clear the markers, so
// the next start can pick the legacy engine (the opt-in phase's way back).
// Each budget moves under one hold of its lock, so nothing can open it and
// write between the copy and the cutover; a budget open in another tab
// holds that lock and makes this throw. The legacy copy is written before
// the store's file goes, and the row turns into the legacy marker in the
// same step, so an interruption leaves the budget whole in one place or the
// other; a copy left behind by a failure is ignored while the row still
// names the store, and replaced by the next revert.
export async function revertToLegacy(): Promise<void> {
const current = getSqliteStore();
for (const row of await listSqliteFileRows()) {
await current.revertFile(row.filepath);
}
}
@@ -7,6 +7,10 @@ import { splitSqlitePath } from './paths';
// worker that holds them ends. Where the API is missing (insecure contexts,
// tests) they are no-ops.
// How long a store waits for a budget's lock before it reports the budget
// in use. Longer than a leaving leader gets to save and let go
// (STORAGE_RELEASE_TIMEOUT_MS in the tab bridge), so a slow hand-over is
// waited out
export const BUDGET_LOCK_TIMEOUT_MS = 10_000;
export function budgetLockName(path: string): string {
@@ -0,0 +1,124 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { startOpfsKeepAlive } from './opfs-keep-alive';
const INTERVAL_MS = 1000;
// What a lookup rejects with once the browser has dropped its index
function notFound() {
return new DOMException('could not be found', 'NotFoundError');
}
async function start(getDirectoryHandle: (name: string) => Promise<unknown>) {
const getRoot = vi.fn(async () => ({ getDirectoryHandle }));
const onLost = vi.fn();
const keepAlive = await startOpfsKeepAlive({
directoryName: 'actual',
onLost,
getRoot,
intervalMs: INTERVAL_MS,
});
return { keepAlive, getRoot, onLost };
}
describe('OPFS keep-alive', () => {
beforeEach(() => {
vi.useFakeTimers();
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
vi.spyOn(console, 'error').mockImplementation(() => undefined);
});
afterEach(() => {
vi.useRealTimers();
vi.restoreAllMocks();
});
it('looks the directory up on every tick, through the root it took at the start', async () => {
const getDirectoryHandle = vi.fn(async () => ({}));
const { keepAlive, getRoot, onLost } = await start(getDirectoryHandle);
expect(getDirectoryHandle).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(INTERVAL_MS * 3);
expect(getDirectoryHandle).toHaveBeenCalledTimes(3);
// A plain lookup: nothing is created, and the root is never asked for
// again, which is the call that deletes the files once the index is gone
for (const call of getDirectoryHandle.mock.calls) {
expect(call).toEqual(['actual']);
}
expect(getRoot).toHaveBeenCalledTimes(1);
expect(onLost).not.toHaveBeenCalled();
keepAlive.stop();
await vi.advanceTimersByTimeAsync(INTERVAL_MS * 3);
expect(getDirectoryHandle).toHaveBeenCalledTimes(3);
});
it('reports the loss once and stops looking', async () => {
const getDirectoryHandle = vi
.fn<(name: string) => Promise<unknown>>()
.mockResolvedValueOnce({})
.mockRejectedValue(notFound());
const { keepAlive, onLost } = await start(getDirectoryHandle);
await vi.advanceTimersByTimeAsync(INTERVAL_MS);
expect(onLost).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(INTERVAL_MS);
expect(onLost).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(INTERVAL_MS * 5);
expect(getDirectoryHandle).toHaveBeenCalledTimes(2);
expect(onLost).toHaveBeenCalledTimes(1);
// And says so without looking again
expect(await keepAlive.check()).toBe(false);
expect(getDirectoryHandle).toHaveBeenCalledTimes(2);
});
it('concludes nothing from another kind of failure', async () => {
const getDirectoryHandle = vi
.fn<(name: string) => Promise<unknown>>()
.mockRejectedValueOnce(new DOMException('busy', 'InvalidStateError'))
.mockResolvedValue({});
const { keepAlive, onLost } = await start(getDirectoryHandle);
expect(await keepAlive.check()).toBe(true);
await vi.advanceTimersByTimeAsync(INTERVAL_MS * 2);
expect(getDirectoryHandle).toHaveBeenCalledTimes(3);
expect(onLost).not.toHaveBeenCalled();
keepAlive.stop();
});
it('looks at once when asked, and reports a loss it finds', async () => {
const getDirectoryHandle = vi
.fn<(name: string) => Promise<unknown>>()
.mockResolvedValueOnce({})
.mockRejectedValue(notFound());
const { keepAlive, onLost } = await start(getDirectoryHandle);
expect(await keepAlive.check()).toBe(true);
expect(onLost).not.toHaveBeenCalled();
// Two at the same time, as a failed statement and a tick can be
expect(await Promise.all([keepAlive.check(), keepAlive.check()])).toEqual([
false,
false,
]);
expect(onLost).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(INTERVAL_MS * 3);
expect(getDirectoryHandle).toHaveBeenCalledTimes(3);
});
it('does nothing when the root cannot be had', async () => {
const onLost = vi.fn();
const keepAlive = await startOpfsKeepAlive({
directoryName: 'actual',
onLost,
getRoot: () => Promise.reject(new DOMException('no', 'SecurityError')),
intervalMs: INTERVAL_MS,
});
expect(vi.getTimerCount()).toBe(0);
expect(await keepAlive.check()).toBe(true);
expect(onLost).not.toHaveBeenCalled();
keepAlive.stop();
});
});
@@ -0,0 +1,98 @@
import { logger } from '#platform/server/log';
// Chromium keeps an index of every site's private file system and closes it
// after ten minutes without file activity. In a private window the index
// lives in memory, yet reopening it checks for its folder on disk
// (`File System/Origins` in the profile), which is only there when a normal
// window has ever used this storage. Without it the site looks new: access
// handles that are open stop working, and the next
// `navigator.storage.getDirectory()` deletes the site's files as leftovers,
// while IndexedDB carries on. A lookup counts as activity, so one every few
// minutes keeps the index open for as long as this worker runs. Nothing
// helps once the tab is closed or frozen for longer than that: then the
// lookup fails, and all that is left is to say so.
const KEEP_ALIVE_INTERVAL_MS = 3 * 60 * 1000;
type DirectoryLookup = {
getDirectoryHandle(name: string): Promise<unknown>;
};
export type OpfsKeepAliveOptions = {
// A directory that is known to exist, looked up on every tick
directoryName: string;
// The browser dropped the private file system under this worker
onLost: () => void;
getRoot?: () => Promise<DirectoryLookup>;
intervalMs?: number;
};
export type OpfsKeepAlive = {
// Looks now rather than at the next tick, and resolves whether the
// private file system is still there
check(): Promise<boolean>;
stop(): void;
};
function isNotFound(error: unknown): boolean {
return (
typeof error === 'object' &&
error !== null &&
'name' in error &&
error.name === 'NotFoundError'
);
}
// Call while the index is known to be live, straight after the storage
// probe: the root is taken once here and every later lookup goes through
// it, never through `getDirectory()` again, which is the call that deletes
// the files once the index has gone.
export async function startOpfsKeepAlive({
directoryName,
onLost,
getRoot = () => navigator.storage.getDirectory(),
intervalMs = KEEP_ALIVE_INTERVAL_MS,
}: OpfsKeepAliveOptions): Promise<OpfsKeepAlive> {
let root: DirectoryLookup;
try {
root = await getRoot();
} catch (error) {
logger.warn('sqlite storage: no keep-alive for OPFS', error);
return { check: async () => true, stop: () => undefined };
}
let isLost = false;
async function check(): Promise<boolean> {
if (isLost) {
return false;
}
try {
await root.getDirectoryHandle(directoryName);
return true;
} catch (error) {
if (!isNotFound(error)) {
// Not the sign of a dropped index, so nothing to conclude
logger.warn('sqlite storage: OPFS keep-alive lookup failed', error);
return true;
}
if (!isLost) {
isLost = true;
stop();
logger.error(
'sqlite storage: the browser discarded the private file system',
);
onLost();
}
return false;
}
}
const timer = setInterval(() => void check(), intervalMs);
function stop() {
clearInterval(timer);
}
return { check, stop };
}
@@ -14,8 +14,8 @@ import * as wasm from '#platform/server/sqlite/wasm';
import {
BudgetInUseError,
StorageLostError,
StorageMigrationError,
StorageUnavailableError,
} from './errors';
import { getSqliteFileRow, putSqliteFileRow } from './files-store';
import { applyFilePragmas, createOpfsStore, POOL_ROOT } from './opfs-sahpool';
@@ -400,9 +400,7 @@ describe('OPFS pool store', () => {
sleep: noSleep,
});
await expect(store.open(PATH)).rejects.toBeInstanceOf(
StorageUnavailableError,
);
await expect(store.open(PATH)).rejects.toBeInstanceOf(StorageLostError);
expect(pool.importDb).not.toHaveBeenCalled();
expect(pool.files.size).toBe(0);
// The marker and the copy are left as they were
@@ -659,6 +657,32 @@ describe('OPFS pool store', () => {
store.close(PATH, db);
});
it('releases every pool for a hand-over and keeps holds balanced', async () => {
const pool = fakePool();
const store = createOpfsStore({
sqlite3: () => fakeSqlite3(pool),
acquireLock: trackingLock().acquire,
sleep: noSleep,
});
await store.open(PATH);
expect(pool.pauseVfs).not.toHaveBeenCalled();
await store.releaseAll();
expect(pool.pauseVfs).toHaveBeenCalledTimes(1);
// Afterwards an overlap still keeps the pool live: a read ends while
// another file of the directory is being opened
const other = '/documents/Actual/budget-one/other.sqlite';
await seedLegacyDatabase(
legacyDatabaseName(other),
await budgetBytes('slow'),
);
const opening = store.open(other);
await store.readFile(PATH);
expect(pool.pauseVfs).toHaveBeenCalledTimes(1);
store.close(other, await opening);
expect(pool.pauseVfs).toHaveBeenCalledTimes(2);
});
it('deletes with unlink and wipeFiles, never removeVfs', async () => {
const pool = fakePool(
new Map([
@@ -691,6 +715,90 @@ describe('OPFS pool store', () => {
delSpy.mockRestore();
});
it('leaves the legacy marker and the absurd-sql copy when a file moves back', async () => {
const pool = fakePool(new Map([['/db.sqlite', new Uint8Array([1])]]));
const sqlite3 = fakeSqlite3(pool);
const store = createOpfsStore({
sqlite3: () => sqlite3,
acquireLock: trackingLock().acquire,
sleep: noSleep,
});
await putSqliteFileRow({ filepath: PATH, contents: '', storage: 'opfs' });
const name = legacyDatabaseName(PATH);
await seedLegacyDatabase(name, new Uint8Array([1]));
await store.revertFile(PATH);
expect(pool.files.has('/db.sqlite')).toBe(false);
expect(await getSqliteFileRow(PATH)).toEqual({
filepath: PATH,
contents: '',
});
expect(await listIndexedDbNames()).toContain(name);
});
it('leaves alone a file another worker has already moved back', async () => {
// What a finished revert leaves: the legacy marker and its copy, with
// nothing in the pool. A second revert that listed the file earlier
// finds them once it has the pool
const pool = fakePool();
const sqlite3 = fakeSqlite3(pool);
const store = createOpfsStore({
sqlite3: () => sqlite3,
acquireLock: trackingLock().acquire,
sleep: noSleep,
});
await putSqliteFileRow({ filepath: PATH, contents: '' });
const name = legacyDatabaseName(PATH);
await seedLegacyDatabase(name, new Uint8Array([1]));
await store.revertFile(PATH);
expect(await getSqliteFileRow(PATH)).toEqual({
filepath: PATH,
contents: '',
});
expect(await listIndexedDbNames()).toContain(name);
});
it('moves back the fallback row rather than a stale pooled file', async () => {
// An interrupted promotion left a pooled file behind; the row is what
// the app last used
const pool = fakePool(new Map([['/db.sqlite', new Uint8Array([1])]]));
const sqlite3 = fakeSqlite3(pool);
const store = createOpfsStore({
sqlite3: () => sqlite3,
acquireLock: trackingLock().acquire,
sleep: noSleep,
});
await putSqliteFileRow({
filepath: PATH,
contents: new Uint8Array([2]),
storage: 'memory-idb',
rev: 1,
});
await store.revertFile(PATH);
const name = legacyDatabaseName(PATH);
const firstBlock = await new Promise<Uint8Array>((resolve, reject) => {
const request = indexedDB.open(name);
request.onsuccess = () => {
const database = request.result;
const read = database.transaction(['data']).objectStore('data').get(0);
read.onsuccess = () => {
database.close();
resolve(new Uint8Array(read.result as ArrayBuffer));
};
read.onerror = () => reject(read.error);
};
request.onerror = () => reject(request.error);
});
expect(firstBlock[0]).toBe(2);
expect(pool.files.has('/db.sqlite')).toBe(false);
expect(await getSqliteFileRow(PATH)).toEqual({
filepath: PATH,
contents: '',
});
});
it('imports an absurd-sql database into the pool and verifies it', async () => {
const bytes = await budgetBytes('legacy');
const name = legacyDatabaseName(PATH);
@@ -777,4 +885,21 @@ describe('OPFS pool store', () => {
expect(() => applyFilePragmas(db as never)).not.toThrow();
closeDatabase(db);
});
it('does not treat an empty fallback row as data to promote', async () => {
await putSqliteFileRow({
filepath: PATH,
contents: new Uint8Array(0),
storage: 'memory-idb',
});
const pool = fakePool();
const store = createOpfsStore({
sqlite3: () => fakeSqlite3(pool),
acquireLock: trackingLock().acquire,
sleep: noSleep,
});
await store.open(PATH);
expect(pool.importDb).not.toHaveBeenCalled();
expect((await getSqliteFileRow(PATH))?.storage).toBe('opfs');
});
});
@@ -10,11 +10,12 @@ import {
deleteLegacyDatabase,
openLegacyDatabaseContents,
verifyMigratedDatabase,
writeLegacyDatabaseBytes,
} from './absurd-migration';
import {
BudgetInUseError,
StorageLostError,
StorageMigrationError,
StorageUnavailableError,
} from './errors';
import {
deleteSqliteFileRow,
@@ -143,7 +144,7 @@ export function createOpfsStore({
budgetLockName(path),
BUDGET_LOCK_TIMEOUT_MS,
).catch(() => {
throw new BudgetInUseError(path);
throw new BudgetInUseError(path, 'the storage lock could not be taken');
});
try {
if (existing) {
@@ -171,7 +172,10 @@ export function createOpfsStore({
error instanceof Error &&
/lock|NoModificationAllowed/i.test(error.name + error.message)
) {
throw new BudgetInUseError(path);
throw new BudgetInUseError(
path,
'its access handles are still held by another worker',
);
}
throw error;
}
@@ -232,6 +236,22 @@ export function createOpfsStore({
}
}
// A journal left by a crash is rolled back by a connection's first read,
// so the raw file is consistent by the time it is exported
function exportPooled(pool: Pool, fileName: string): Uint8Array {
const db = new (sqlite3().oo1.DB)({
filename: fileName,
flags: 'w',
vfs: pool.util.vfsName,
});
try {
db.selectValue('PRAGMA schema_version');
} finally {
db.close();
}
return pool.util.exportFile(fileName);
}
function discardImport(pool: Pool, fileName: string) {
try {
pool.util.unlink(fileName);
@@ -330,9 +350,7 @@ export function createOpfsStore({
const pooled = pool.util.getFileNames().includes(fileName);
if (row?.storage === 'opfs') {
if (!pooled) {
throw new StorageUnavailableError(
`${path} is missing from its OPFS pool`,
);
throw new StorageLostError(path);
}
return true;
}
@@ -470,19 +488,7 @@ export function createOpfsStore({
if (!(await importExisting(pool, path, fileName))) {
throw new Error('File does not exist: ' + path);
}
// A journal left by a crash is rolled back by a connection's first
// read, so the raw file is consistent by the time it is exported
const db = new (sqlite3().oo1.DB)({
filename: fileName,
flags: 'w',
vfs: pool.util.vfsName,
});
try {
db.selectValue('PRAGMA schema_version');
} finally {
db.close();
}
return pool.util.exportFile(fileName);
return exportPooled(pool, fileName);
} finally {
release(pool);
}
@@ -524,6 +530,13 @@ export function createOpfsStore({
}
files.push({ pool, fileName });
}
// The rows go first, while the budgets are held: a waiting worker
// could otherwise write a replacement whose marker this would
// delete, and a worker killed from here on leaves at worst a file
// nothing names, never a marker for a file that is gone.
for (const path of paths) {
await deleteSqliteFileRow(path);
}
for (const { pool, fileName } of files) {
const open = pool.open.get(fileName);
if (open) {
@@ -542,11 +555,6 @@ export function createOpfsStore({
await pool.util.wipeFiles();
}
}
// While the budgets are still held: a waiting worker could
// otherwise write a replacement whose marker this would delete
for (const path of paths) {
await deleteSqliteFileRow(path);
}
} finally {
for (const pool of held.values()) {
release(pool);
@@ -559,6 +567,58 @@ export function createOpfsStore({
}
},
async revertFile(path) {
const { dirId, fileName } = splitSqlitePath(path);
await opening.get(path)?.catch(() => undefined);
await replacing.get(path)?.catch(() => undefined);
const pool = await acquire(dirId, path);
try {
if (pool.open.has(fileName)) {
throw new Error(`cannot revert an open database: ${path}`);
}
const row = await getSqliteFileRow(path);
// Another worker moved it back while this one waited for the pool,
// or it is gone: the legacy marker is not this store's to touch
if (row?.storage == null) {
return;
}
const pooled = pool.util.getFileNames().includes(fileName);
if (row.storage === 'opfs' && !pooled) {
throw new StorageLostError(path);
}
// The fallback's whole-file row is what the app last used, so as on
// open it wins over a pooled file an interrupted promotion left; an
// empty one was recorded but never saved
const stored = row.contents;
const fallbackBytes =
ArrayBuffer.isView(stored) && stored.byteLength > 0
? new Uint8Array(
stored.buffer,
stored.byteOffset,
stored.byteLength,
)
: null;
const bytes =
fallbackBytes ?? (pooled ? exportPooled(pool, fileName) : null);
if (bytes == null) {
await deleteSqliteFileRow(path);
return;
}
await writeLegacyDatabaseBytes(legacyDatabaseName(path), bytes);
// The marker goes before the file, as in a removal
await putSqliteFileRow({ filepath: path, contents: '' });
if (pooled) {
unlinkJournal(pool, fileName);
pool.util.unlink(fileName);
if (pool.util.getFileCount() === 0) {
await pool.util.wipeFiles();
}
}
} finally {
release(pool);
}
},
async exists(path) {
return (await getSqliteFileRow(path)) != null;
},
@@ -566,6 +626,17 @@ export function createOpfsStore({
async flush() {
// Writes go straight to the access handles and are flushed by sqlite
},
async releaseAll() {
for (const pool of pools.values()) {
for (const db of pool.open.values()) {
db.close();
}
pool.open.clear();
// Nothing of ours is held here: settle rather than give a hold back
settle(pool);
}
},
};
return store;
@@ -0,0 +1,165 @@
import 'fake-indexeddb/auto';
import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import {
closeDatabase,
execQuery,
openDatabase,
runQuery,
} from '#platform/server/sqlite';
import { readLegacyDatabaseBytes } from './absurd-migration';
import { StorageUnavailableError } from './errors';
import {
getSqliteFileRow,
hasMigratedSqliteFiles,
putSqliteFileRow,
} from './files-store';
import { createMemoryIdbStore } from './idb-memory';
import { legacyDatabaseName } from './paths';
import {
initWasmEngine,
listIndexedDbNames,
resetIndexedDb,
} from './test-helpers';
import { getSqliteStore, revertToLegacy, setSqliteStore } from './index';
const sendMock = vi.hoisted(() => vi.fn());
vi.mock('#platform/server/connection', () => ({ send: sendMock }));
const PATH = '/documents/Actual/budget-one/db.sqlite';
const OTHER = '/documents/Actual/budget-two/db.sqlite';
const SCHEMA =
'CREATE TABLE __migrations__ (id INTEGER PRIMARY KEY); CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT);';
beforeAll(() => initWasmEngine());
let previousStore: ReturnType<typeof createMemoryIdbStore> | null = null;
beforeEach(async () => {
// Pending writes of the previous test's store must not land in this one
await previousStore?.releaseAll();
await resetIndexedDb();
sendMock.mockClear();
previousStore = createMemoryIdbStore({ debounceMs: 10, maxDirtyMs: 50 });
setSqliteStore(previousStore);
});
async function createBudget(path: string, value: string) {
const db = await openDatabase(path);
execQuery(db, SCHEMA);
runQuery(db, 'INSERT INTO t (v) VALUES (?)', [value]);
closeDatabase(db);
}
describe('reverting to the legacy storage layout', () => {
it('moves every budget back and clears the markers', async () => {
await createBudget(PATH, 'one');
await createBudget(OTHER, 'two');
const before = await getSqliteStore().readFile(PATH);
expect(await hasMigratedSqliteFiles()).toBe(true);
await revertToLegacy();
expect(await hasMigratedSqliteFiles()).toBe(false);
for (const path of [PATH, OTHER]) {
expect(await getSqliteFileRow(path)).toEqual({
filepath: path,
contents: '',
});
expect(await listIndexedDbNames()).toContain(legacyDatabaseName(path));
}
const back = await readLegacyDatabaseBytes(PATH);
expect(Buffer.from(back as Uint8Array).equals(Buffer.from(before))).toBe(
true,
);
// And the legacy bytes are a working database with the data
const db = await openDatabase(back as Uint8Array);
expect(runQuery(db, 'SELECT v FROM t', [], true)).toEqual([{ v: 'one' }]);
closeDatabase(db);
});
it('refuses while a database is open', async () => {
const db = await openDatabase(PATH);
execQuery(db, SCHEMA);
await expect(revertToLegacy()).rejects.toThrow(/open database/);
closeDatabase(db);
expect(await listIndexedDbNames()).not.toContain(legacyDatabaseName(PATH));
});
it('drops a budget the fallback recorded but never saved', async () => {
// The mark a new database gets before its first save, left by a crash
await putSqliteFileRow({
filepath: PATH,
contents: new Uint8Array(0),
storage: 'memory-idb',
rev: 1,
});
await createBudget(OTHER, 'two');
await revertToLegacy();
expect(await hasMigratedSqliteFiles()).toBe(false);
expect(await getSqliteFileRow(PATH)).toBeUndefined();
expect(await listIndexedDbNames()).not.toContain(legacyDatabaseName(PATH));
// The budget with data still moved back
expect(await getSqliteFileRow(OTHER)).toEqual({
filepath: OTHER,
contents: '',
});
expect(await listIndexedDbNames()).toContain(legacyDatabaseName(OTHER));
});
it('still refuses a budget whose bytes are out of reach', async () => {
await putSqliteFileRow({ filepath: PATH, contents: '', storage: 'opfs' });
await expect(revertToLegacy()).rejects.toBeInstanceOf(
StorageUnavailableError,
);
expect((await getSqliteFileRow(PATH))?.storage).toBe('opfs');
});
it('holds the budget lock from the read to the cutover', async () => {
await createBudget(PATH, 'one');
const name = legacyDatabaseName(PATH);
const held: string[] = [];
const releases: Promise<void>[] = [];
let copyExistedAtRelease: boolean | null = null;
const request = vi.fn(
(lockName: string, _options: unknown, callback: () => Promise<void>) => {
held.push(lockName);
// Looks at the store the moment the lock is given back
const released = callback().then(async () => {
copyExistedAtRelease = (await listIndexedDbNames()).includes(name);
});
releases.push(released);
return released;
},
);
Object.defineProperty(navigator, 'locks', {
value: { request },
configurable: true,
});
try {
await revertToLegacy();
await Promise.all(releases);
} finally {
Object.defineProperty(navigator, 'locks', {
value: undefined,
configurable: true,
});
}
// One hold for the budget, still in place when its copy was complete
expect(
held.filter(
lockName => lockName === 'actual-budget:documents-Actual-budget-one',
),
).toHaveLength(1);
expect(copyExistedAtRelease).toBe(true);
});
it('is a no-op without migrated budgets', async () => {
await revertToLegacy();
expect(await hasMigratedSqliteFiles()).toBe(false);
});
});
@@ -17,13 +17,21 @@ export type SqliteFileStore = {
close(path: string, db: WasmDatabase): void;
readFile(path: string): Promise<Uint8Array>;
writeFile(path: string, bytes: Uint8Array): Promise<void>;
// A removal also drops the file's absurd-sql copy and its row
removeFile(path: string): Promise<void>;
// All or nothing: every budget involved is locked and checked before the
// first file goes, so a directory never ends up half deleted
removeFiles(paths: string[]): Promise<void>;
// Moves the file back to an absurd-sql copy and leaves the legacy
// filesystem's marker, holding the budget from the read to the cutover so
// the copy is never older than the file; a file recorded but never saved
// just loses its row
revertFile(path: string): Promise<void>;
exists(path: string): Promise<boolean>;
// Persist anything pending (the fallback store writes lazily)
flush(): Promise<void>;
// Let go of every handle and lock so another worker can take over
releaseAll(): Promise<void>;
// Called by the engine after a statement that may have written
onWrite?(db: WasmDatabase): void;
};
@@ -10,6 +10,7 @@ import { logger } from '#platform/server/log';
import {
BudgetInUseError,
flushSqliteStorage,
StorageLostError,
StorageMigrationError,
StorageUnavailableError,
} from '#platform/server/sqlite';
@@ -274,7 +275,29 @@ async function syncBudget() {
return result;
}
// The load in progress: a second request for the same budget waits for it.
// Prefs are set before the database opens, so the shortcut in
// `loadBudgetOnce` would otherwise answer a tab that joins through the
// coordinator while the storage is still moving the budget over
let loading: {
id: Budget['id'];
result: ReturnType<typeof loadBudgetOnce>;
} | null = null;
async function loadBudget({ id }: { id: Budget['id'] }) {
if (loading?.id === id) {
return loading.result;
}
const result = loadBudgetOnce(id).finally(() => {
if (loading?.result === result) {
loading = null;
}
});
loading = { id, result };
return result;
}
async function loadBudgetOnce(id: Budget['id']) {
const currentPrefs = prefs.getPrefs();
if (currentPrefs) {
@@ -287,9 +310,7 @@ async function loadBudget({ id }: { id: Budget['id'] }) {
}
}
const res = await _loadBudget(id);
return res;
return _loadBudget(id);
}
async function createDemoBudget() {
@@ -609,7 +630,8 @@ async function _loadBudget(id: Budget['id']): Promise<{
| 'opening-budget'
| 'budget-in-use'
| 'storage-migration-failed'
| 'storage-unavailable';
| 'storage-unavailable'
| 'storage-lost';
}> {
let dir: string;
try {
@@ -647,6 +669,10 @@ async function _loadBudget(id: Budget['id']): Promise<{
logger.error('Storage unavailable', e);
return { error: 'storage-unavailable' };
}
if (e instanceof StorageLostError) {
logger.error('Storage lost', e);
return { error: 'storage-lost' };
}
captureException(e);
return { error: 'opening-budget' };
}
@@ -88,6 +88,47 @@ describe('Budgets', () => {
expect(getClock()).toEqual(deserializeClock(row.clock));
});
test('a second request for the budget being loaded waits for that load', async () => {
await createTestBudget('default-budget-template');
// Prefs are read before the database opens, so the budget looks loaded
// while its open is still running (slow when the storage moves a budget
// over); a tab joining then must not be answered before it is
let finishOpen: (() => void) | undefined;
const openGate = new Promise<void>(resolve => {
finishOpen = resolve;
});
const openDatabase = db.openDatabase;
const openSpy = vi
.spyOn(db, 'openDatabase')
.mockImplementation(async id => {
await openGate;
return openDatabase(id);
});
try {
const first = runHandler(handlers['load-budget'], { id: 'test-budget' });
await vi.waitFor(() => expect(openSpy).toHaveBeenCalledTimes(1));
expect(prefs.getPrefs().id).toBe('test-budget');
let isSecondSettled = false;
const second = runHandler(handlers['load-budget'], {
id: 'test-budget',
}).finally(() => {
isSecondSettled = true;
});
await new Promise(resolve => setTimeout(resolve, 20));
expect(isSecondSettled).toBe(false);
finishOpen?.();
expect((await first).error).toBe(undefined);
expect((await second).error).toBe(undefined);
expect(openSpy).toHaveBeenCalledTimes(1);
expect(db.getDatabase()).not.toBe(null);
} finally {
openSpy.mockRestore();
}
});
test('budget detects out of sync migrations', async () => {
await createTestBudget('default-budget-template');
+26 -2
View File
@@ -37,6 +37,7 @@ import { app as rulesApp } from './rules/app';
import { app as schedulesApp } from './schedules/app';
import { getServer, setServer } from './server-config';
import { app as spreadsheetApp } from './spreadsheet/app';
import { app as storageApp } from './storage/app';
import { fullSync, setSyncingMode } from './sync';
import { app as syncApp } from './sync/app';
import { app as tagsApp } from './tags/app';
@@ -153,6 +154,7 @@ app.combine(
budgetFilesApp,
encryptionApp,
tagsApp,
storageApp,
);
export function getDefaultDocumentDir() {
@@ -215,9 +217,25 @@ async function ensureUsable(dir) {
export async function initApp(
isDev,
socketName,
{ storageBackend = 'legacy' }: { storageBackend?: StorageBackend } = {},
{
storageBackend = 'legacy',
isSharedArrayBufferOverrideEnabled = false,
}: {
storageBackend?: StorageBackend;
isSharedArrayBufferOverrideEnabled?: boolean;
} = {},
) {
const storage = await sqlite.init({ storage: storageBackend });
// Only the legacy engine (absurd-sql) needs SharedArrayBuffer, and
// whether it runs is known only now: data already moved to the new
// storage wins over the flag
if (
storage === 'legacy' &&
typeof SharedArrayBuffer === 'undefined' &&
!isSharedArrayBufferOverrideEnabled
) {
throw new Error('shared-array-buffer-missing');
}
asyncStorage.init();
await fs.init({ storage });
await setupDocumentsDir();
@@ -260,6 +278,12 @@ export async function initApp(
}
type BaseInitConfig = {
/**
* Browser build only: which storage holds budgets. `opfs` uses the
* origin private file system, falling back to `memory-idb` where
* unavailable.
*/
storageBackend?: StorageBackend;
/**
* Directory where budget data is stored. In Node this is a directory on
* disk (defaults to the current working directory) and must already exist.
@@ -313,7 +337,7 @@ export async function init(config: InitConfig) {
serverURL = process.env.ACTUAL_SERVER_URL;
}
const storage = await sqlite.init();
const storage = await sqlite.init({ storage: config?.storageBackend });
asyncStorage.init({ persist: false });
await fs.init({ storage });
fs._setDocumentDir(dataDir || process.cwd());
@@ -0,0 +1,119 @@
import * as asyncStorage from '#platform/server/asyncStorage';
import { logger } from '#platform/server/log';
import * as sqlite from '#platform/server/sqlite';
import { createApp } from '#server/app';
import { app as mainApp } from '#server/main-app';
import { runHandler } from '#server/mutators';
import * as prefs from '#server/prefs';
export type StorageHandlers = {
'storage-flush': typeof flushStorage;
'storage-release': typeof releaseStorage;
'storage-resume': typeof resumeStorage;
'storage-info': typeof getStorageInfo;
'storage-revert': typeof revertStorage;
};
export const app = createApp<StorageHandlers>();
app.method('storage-flush', flushStorage);
app.method('storage-release', releaseStorage);
app.method('storage-resume', resumeStorage);
app.method('storage-info', getStorageInfo);
app.method('storage-revert', revertStorage);
// Persist anything the storage layer is holding. The IndexedDB fallback
// writes lazily, so the client asks for this when the tab is hidden or
// about to unload.
async function flushStorage() {
await sqlite.flushSqliteStorage();
return 'ok' as const;
}
// Let go of storage handles and locks so a worker in another tab can take
// the budget over. The tab bridge asks for this right before terminating
// this worker, and a page that is leaving asks for it so that a frozen copy
// of the page holds nothing back; only the storage matters, the app state
// dies or relaunches with the page. A failed flush still lets go: the next
// worker must not wait on this one. No budget is opened from here on, so one
// that was still loading cannot take itself back.
async function releaseStorage() {
try {
await sqlite.flushSqliteStorage();
} finally {
await sqlite.releaseSqliteStorage();
}
return 'ok' as const;
}
// For a page that came back out of the back/forward cache with this worker,
// which may open budgets again
async function resumeStorage() {
sqlite.resumeSqliteStorage();
return 'ok' as const;
}
// Which storage this worker ended up using
async function getStorageInfo() {
return { backend: sqlite.getStorageBackend() };
}
// Move every budget back to absurd-sql's layout so the next start can use
// the legacy engine. Closes the open budget first; a budget open in another
// tab holds its storage lock and makes this fail.
async function revertStorage(): Promise<
{ ok: true } | { error: 'budget-in-use' | 'revert-failed' }
> {
if (sqlite.getStorageBackend() === 'legacy') {
return { ok: true };
}
// The budget this worker has open; `lastBudget` is whichever budget any
// tab opened last, kept only so the relaunch starts where the user was
const openBudgetId = prefs.getPrefs()?.id;
const lastBudget = await asyncStorage.getItem('lastBudget');
if (openBudgetId) {
await runHandler(mainApp.handlers['close-budget'], undefined);
}
try {
await sqlite.revertSqliteStorageToLegacy();
return { ok: true };
} catch (error) {
// The client still shows the budget this closed, so reopen it; a
// failure to is logged, and the revert's own error still answers
if (openBudgetId) {
try {
const reopened = await runHandler(mainApp.handlers['load-budget'], {
id: openBudgetId,
});
if (reopened.error) {
logger.error(
'storage: could not reopen the budget after a failed revert',
reopened.error,
);
}
} catch (reopenError) {
logger.error(
'storage: could not reopen the budget after a failed revert',
reopenError,
);
}
}
if (error instanceof sqlite.BudgetInUseError) {
return { error: 'budget-in-use' };
}
logger.error('storage: revert to the legacy layout failed', error);
return { error: 'revert-failed' };
} finally {
if (lastBudget) {
try {
await asyncStorage.setItem('lastBudget', lastBudget);
} catch (restoreError) {
// It only decides where the next launch starts, so the result
// above stands
logger.error(
'storage: could not restore the last opened budget',
restoreError,
);
}
}
}
}
@@ -67,6 +67,16 @@ describe('getSyncError', () => {
);
});
it('says when the browser no longer has the local copy, and what to do', () => {
expect(getSyncError('storage-lost', 'budget-id')).toBe(
'This browser no longer has the local copy of budget "budget-id". Browsers can discard it, mostly in private windows. Delete this copy and download the budget again, or restore it from a backup.',
);
// Not the message for storage that is only out of reach
expect(getSyncError('storage-unavailable', 'budget-id')).toContain(
'Nothing was deleted',
);
});
it('falls back to the generic message for unknown errors', () => {
expect(getSyncError('something-else', 'budget-id')).toBe(
'We had an unknown problem opening "budget-id".',
+2
View File
@@ -167,6 +167,8 @@ export function getSyncError(error: string, id: string, meta?: unknown) {
return `Moving budget "${id}" to the new browser storage failed. Nothing was deleted; reload the page and try again.`;
} else if (error === 'storage-unavailable') {
return `Budget "${id}" is in storage this browser cannot reach right now. Nothing was deleted; reload the page and try again.`;
} else if (error === 'storage-lost') {
return `This browser no longer has the local copy of budget "${id}". Browsers can discard it, mostly in private windows. Delete this copy and download the budget again, or restore it from a backup.`;
} else if (error === 'clock-drift') {
return 'Failed to sync because your device time differs too much from the server. Please check your device time settings and ensure they are correct.';
} else if (error === 'out-of-sync') {
+2
View File
@@ -16,6 +16,7 @@ import type { ReportsHandlers } from '#server/reports/app';
import type { RulesHandlers } from '#server/rules/app';
import type { SchedulesHandlers } from '#server/schedules/app';
import type { SpreadsheetHandlers } from '#server/spreadsheet/app';
import type { StorageHandlers } from '#server/storage/app';
import type { SyncHandlers } from '#server/sync/app';
import type { TagsHandlers } from '#server/tags/app';
import type { ToolsHandlers } from '#server/tools/app';
@@ -47,6 +48,7 @@ export type Handlers = {} & ServerHandlers &
BudgetFileHandlers &
EncryptionHandlers &
TagsHandlers &
StorageHandlers &
AuthHandlers;
export type HandlerFunctions = Handlers[keyof Handlers];
@@ -65,6 +65,8 @@ type CellsChangedEvent = Array<{
type FallbackWriteErrorEvent = undefined;
// The IndexedDB fallback could not store a file, retry included
type StorageWriteFailedEvent = { path: string };
// The browser dropped the private file system under the running app
type StorageLostEvent = undefined;
type FinishImportEvent = undefined;
type FinishLoadEvent = undefined;
type IndexeddbQuotaErrorEvent = undefined;
@@ -117,6 +119,7 @@ export type ServerEvents = {
'cells-changed': CellsChangedEvent;
'fallback-write-error': FallbackWriteErrorEvent;
'storage-write-failed': StorageWriteFailedEvent;
'storage-lost': StorageLostEvent;
'finish-import': FinishImportEvent;
'finish-load': FinishLoadEvent;
'import-progress': ImportProgressEvent;
+1
View File
@@ -9,6 +9,7 @@ export default defineConfig({
'src/platform/client/browser-preload/worker-bridge.test.ts',
'src/platform/server/sqlite/index.test.ts',
'src/platform/server/sqlite/storage/*.test.ts',
'src/platform/client/browser-preload/storage-backend.test.ts',
'src/platform/server/fs/index.test.ts',
'src/platform/server/fs/fs.web.test.ts',
'node_modules',
+1
View File
@@ -9,6 +9,7 @@ export default defineConfig({
'src/platform/client/browser-preload/worker-bridge.test.ts',
'src/platform/server/sqlite/index.test.ts',
'src/platform/server/sqlite/storage/*.test.ts',
'src/platform/client/browser-preload/storage-backend.test.ts',
'src/platform/server/fs/index.test.ts',
'src/platform/server/fs/fs.web.test.ts',
],
@@ -0,0 +1,6 @@
---
category: Enhancements
authors: [MikesGlitch]
---
Add a hidden setting to try the new browser storage engine, and move the API's browser build to it so pages using it no longer need cross-origin isolation headers